From 66967470a8f79d9a62fbf65ebbef5167ebc9092b Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Fri, 8 Aug 2025 14:00:27 +0800 Subject: [PATCH 001/119] xxx --- .gitignore | 51 + README.md | 58 + atom/__init__.py | 2 + atom/config.py | 26 + atom/model_engine/block_manager.py | 113 ++ atom/model_engine/llm_engine.py | 93 ++ atom/model_engine/model_runner.py | 251 ++++ atom/model_engine/scheduler.py | 71 ++ atom/model_engine/sequence.py | 83 ++ atom/model_loader/loader.py | 30 + atom/model_ops/activation.py | 14 + atom/model_ops/attention.py | 95 ++ atom/model_ops/embed_head.py | 71 ++ atom/model_ops/layernorm.py | 51 + atom/model_ops/linear.py | 174 +++ atom/model_ops/rotary_embedding.py | 68 ++ atom/model_ops/sampler.py | 18 + atom/models/deepseek_v2.py | 1717 ++++++++++++++++++++++++++++ atom/models/qwen3.py | 220 ++++ atom/sampling_params.py | 8 + atom/utils/context.py | 27 + bench.py | 43 + pyproject.toml | 21 + 23 files changed, 3305 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 atom/__init__.py create mode 100644 atom/config.py create mode 100644 atom/model_engine/block_manager.py create mode 100644 atom/model_engine/llm_engine.py create mode 100644 atom/model_engine/model_runner.py create mode 100644 atom/model_engine/scheduler.py create mode 100644 atom/model_engine/sequence.py create mode 100644 atom/model_loader/loader.py create mode 100644 atom/model_ops/activation.py create mode 100644 atom/model_ops/attention.py create mode 100644 atom/model_ops/embed_head.py create mode 100644 atom/model_ops/layernorm.py create mode 100644 atom/model_ops/linear.py create mode 100644 atom/model_ops/rotary_embedding.py create mode 100644 atom/model_ops/sampler.py create mode 100644 atom/models/deepseek_v2.py create mode 100644 atom/models/qwen3.py create mode 100644 atom/sampling_params.py create mode 100644 atom/utils/context.py create mode 100644 bench.py create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..15b593a6e1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,51 @@ +# aiter version +aiter/_version.py + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll +build +blob + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# python cache +__pycache__ +*.pyc +.eggs +*.egg-info + +# editor +.vscode + +# debug folder +debug +aiter_logs +*.log \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000..fbbff70d33 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# Nano-vLLM + +A lightweight vLLM implementation built from scratch. + +## Key Features + +* πŸš€ **Fast offline inference** - Comparable inference speeds to vLLM +* πŸ“– **Readable codebase** - Clean implementation in ~ 1,200 lines of Python code +* ⚑ **Optimization Suite** - Prefix caching, Tensor Parallelism, Torch compilation, CUDA graph, etc. + +## Installation + +```bash +pip install git+https://github.com/GeeeekExplorer/nano-vllm.git +``` + +## Manual Download + +If you prefer to download the model weights manually, use the following command: +```bash +huggingface-cli download --resume-download Qwen/Qwen3-0.6B \ + --local-dir ~/huggingface/Qwen3-0.6B/ \ + --local-dir-use-symlinks False +``` + +## Quick Start + +See `example.py` for usage. The API mirrors vLLM's interface with minor differences in the `LLM.generate` method: +```python +from atom import LLM, SamplingParams +llm = LLM("/YOUR/MODEL/PATH", enforce_eager=True, tensor_parallel_size=1) +sampling_params = SamplingParams(temperature=0.6, max_tokens=256) +prompts = ["Hello, Nano-vLLM."] +outputs = llm.generate(prompts, sampling_params) +outputs[0]["text"] +``` + +## Benchmark + +See `bench.py` for benchmark. + +**Test Configuration:** +- Hardware: RTX 4070 Laptop (8GB) +- Model: Qwen3-0.6B +- Total Requests: 256 sequences +- Input Length: Randomly sampled between 100–1024 tokens +- Output Length: Randomly sampled between 100–1024 tokens + +**Performance Results:** +| Inference Engine | Output Tokens | Time (s) | Throughput (tokens/s) | +|----------------|-------------|----------|-----------------------| +| vLLM | 133,966 | 98.37 | 1361.84 | +| Nano-vLLM | 133,966 | 93.41 | 1434.13 | + + +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=GeeeekExplorer/nano-vllm&type=Date)](https://www.star-history.com/#GeeeekExplorer/nano-vllm&Date) \ No newline at end of file diff --git a/atom/__init__.py b/atom/__init__.py new file mode 100644 index 0000000000..0ad0f2999a --- /dev/null +++ b/atom/__init__.py @@ -0,0 +1,2 @@ +from atom.model_engine.llm_engine import LLMEngine +from atom.sampling_params import SamplingParams diff --git a/atom/config.py b/atom/config.py new file mode 100644 index 0000000000..959ffb3576 --- /dev/null +++ b/atom/config.py @@ -0,0 +1,26 @@ +import os +from dataclasses import dataclass +from transformers import AutoConfig + + +@dataclass +class Config: + model: str + max_num_batched_tokens: int = 16384 + max_num_seqs: int = 512 + max_model_len: int = 4096 + gpu_memory_utilization: float = 0.9 + tensor_parallel_size: int = 1 + enforce_eager: bool = False + hf_config: AutoConfig | None = None + eos: int = -1 + kvcache_block_size: int = 256 + num_kvcache_blocks: int = -1 + + def __post_init__(self): + assert os.path.isdir(self.model) + assert self.kvcache_block_size % 256 == 0 + assert 1 <= self.tensor_parallel_size <= 8 + self.hf_config = AutoConfig.from_pretrained(self.model) + self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings) + assert self.max_num_batched_tokens >= self.max_model_len diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py new file mode 100644 index 0000000000..43373a0c98 --- /dev/null +++ b/atom/model_engine/block_manager.py @@ -0,0 +1,113 @@ +from collections import deque +import xxhash +import numpy as np + +from atom.model_engine.sequence import Sequence + + +class Block: + + def __init__(self, block_id): + self.block_id = block_id + self.ref_count = 0 + self.hash = -1 + self.token_ids = [] + + def update(self, hash: int, token_ids: list[int]): + self.hash = hash + self.token_ids = token_ids + + def reset(self): + self.ref_count = 1 + self.hash = -1 + self.token_ids = [] + + +class BlockManager: + + def __init__(self, num_blocks: int, block_size: int): + assert num_blocks > 0 + self.block_size = block_size + self.blocks: list[Block] = [Block(i) for i in range(num_blocks)] + self.hash_to_block_id: dict[int, int] = dict() + self.free_block_ids: deque[int] = deque(range(num_blocks)) + self.used_block_ids: set[int] = set() + + @classmethod + def compute_hash(cls, token_ids: list[int], prefix: int = -1): + h = xxhash.xxh64() + if prefix != -1: + h.update(prefix.to_bytes(8, "little")) + h.update(np.array(token_ids).tobytes()) + return h.intdigest() + + def _allocate_block(self, block_id: int) -> Block: + block = self.blocks[block_id] + assert block.ref_count == 0 + block.reset() + self.free_block_ids.remove(block_id) + self.used_block_ids.add(block_id) + return self.blocks[block_id] + + def _deallocate_block(self, block_id: int) -> Block: + assert self.blocks[block_id].ref_count == 0 + self.used_block_ids.remove(block_id) + self.free_block_ids.append(block_id) + + def can_allocate(self, seq: Sequence) -> bool: + return len(self.free_block_ids) >= seq.num_blocks + + def allocate(self, seq: Sequence): + assert not seq.block_table + h = -1 + cache_miss = False + for i in range(seq.num_blocks): + token_ids = seq.block(i) + h = self.compute_hash(token_ids, h) if len(token_ids) == self.block_size else -1 + block_id = self.hash_to_block_id.get(h, -1) + if block_id == -1 or self.blocks[block_id].token_ids != token_ids: + cache_miss = True + if cache_miss: + block_id = self.free_block_ids[0] + block = self._allocate_block(block_id) + else: + seq.num_cached_tokens += self.block_size + if block_id in self.used_block_ids: + block = self.blocks[block_id] + block.ref_count += 1 + else: + block = self._allocate_block(block_id) + if h != -1: + block.update(h, token_ids) + self.hash_to_block_id[h] = block_id + seq.block_table.append(block_id) + + def deallocate(self, seq: Sequence): + for block_id in reversed(seq.block_table): + block = self.blocks[block_id] + block.ref_count -= 1 + if block.ref_count == 0: + self._deallocate_block(block_id) + seq.num_cached_tokens = 0 + seq.block_table.clear() + + def can_append(self, seq: Sequence) -> bool: + return len(self.free_block_ids) >= (len(seq) % self.block_size == 1) + + def may_append(self, seq: Sequence): + block_table = seq.block_table + last_block = self.blocks[block_table[-1]] + if len(seq) % self.block_size == 1: + assert last_block.hash != -1 + block_id = self.free_block_ids[0] + self._allocate_block(block_id) + block_table.append(block_id) + elif len(seq) % self.block_size == 0: + assert last_block.hash == -1 + token_ids = seq.block(seq.num_blocks-1) + prefix = self.blocks[block_table[-2]].hash if len(block_table) > 1 else -1 + h = self.compute_hash(token_ids, prefix) + last_block.update(h, token_ids) + self.hash_to_block_id[h] = last_block.block_id + else: + assert last_block.hash == -1 diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py new file mode 100644 index 0000000000..b34a0529c9 --- /dev/null +++ b/atom/model_engine/llm_engine.py @@ -0,0 +1,93 @@ +import atexit +from dataclasses import fields +from time import perf_counter +from tqdm.auto import tqdm +from transformers import AutoTokenizer +import torch.multiprocessing as mp + +from atom.config import Config +from atom.sampling_params import SamplingParams +from atom.model_engine.sequence import Sequence +from atom.model_engine.scheduler import Scheduler +from atom.model_engine.model_runner import ModelRunner + + +class LLMEngine: + + def __init__(self, model, **kwargs): + config_fields = {field.name for field in fields(Config)} + config_kwargs = {k: v for k, v in kwargs.items() if k in config_fields} + config = Config(model, **config_kwargs) + self.ps = [] + self.events = [] + ctx = mp.get_context("spawn") + for i in range(1, config.tensor_parallel_size): + event = ctx.Event() + process = ctx.Process(target=ModelRunner, args=(config, i, event)) + process.start() + self.ps.append(process) + self.events.append(event) + self.model_runner = ModelRunner(config, 0, self.events) + self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True) + config.eos = self.tokenizer.eos_token_id + self.scheduler = Scheduler(config) + atexit.register(self.exit) + + def exit(self): + self.model_runner.call("exit") + del self.model_runner + for p in self.ps: + p.join() + + def add_request(self, prompt: str | list[int], sampling_params: SamplingParams): + if isinstance(prompt, str): + prompt = self.tokenizer.encode(prompt) + seq = Sequence(prompt, sampling_params) + self.scheduler.add(seq) + + def step(self): + seqs, is_prefill = self.scheduler.schedule() + token_ids = self.model_runner.call("run", seqs, is_prefill) + self.scheduler.postprocess(seqs, token_ids) + outputs = [(seq.seq_id, seq.completion_token_ids) for seq in seqs if seq.is_finished] + num_tokens = sum(len(seq) for seq in seqs) if is_prefill else -len(seqs) + return outputs, num_tokens + + def is_finished(self): + return self.scheduler.is_finished() + + def generate( + self, + prompts: list[str] | list[list[int]], + sampling_params: SamplingParams | list[SamplingParams], + use_tqdm: bool = True, + ) -> list[str]: + if use_tqdm: + pbar = tqdm(total=len(prompts), desc="Generating", dynamic_ncols=True) + if not isinstance(sampling_params, list): + sampling_params = [sampling_params] * len(prompts) + for prompt, sp in zip(prompts, sampling_params): + self.add_request(prompt, sp) + outputs = {} + prefill_throughput = decode_throughput = 0. + while not self.is_finished(): + t = perf_counter() + output, num_tokens = self.step() + if use_tqdm: + if num_tokens > 0: + prefill_throughput = num_tokens / (perf_counter() - t) + else: + decode_throughput = -num_tokens / (perf_counter() - t) + pbar.set_postfix({ + "Prefill": f"{int(prefill_throughput)}tok/s", + "Decode": f"{int(decode_throughput)}tok/s", + }) + for seq_id, token_ids in output: + outputs[seq_id] = token_ids + if use_tqdm: + pbar.update(1) + outputs = [outputs[seq_id] for seq_id in sorted(outputs)] + outputs = [{"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} for token_ids in outputs] + if use_tqdm: + pbar.close() + return outputs diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py new file mode 100644 index 0000000000..e1d3fa4515 --- /dev/null +++ b/atom/model_engine/model_runner.py @@ -0,0 +1,251 @@ +import pickle +import torch +import torch.distributed as dist +from multiprocessing.synchronize import Event +from multiprocessing.shared_memory import SharedMemory + +from atom.config import Config +from atom.model_engine.sequence import Sequence +from atom.model_ops.sampler import Sampler +from atom.models.qwen3 import Qwen3ForCausalLM +from atom.utils.context import set_context, get_context, reset_context +from atom.model_loader.loader import load_model + + +class ModelRunner: + + def __init__(self, config: Config, rank: int, event: Event | list[Event]): + self.config = config + hf_config = config.hf_config + self.block_size = config.kvcache_block_size + self.enforce_eager = config.enforce_eager + self.world_size = config.tensor_parallel_size + self.rank = rank + self.event = event + + dist.init_process_group("nccl", "tcp://localhost:2333", world_size=self.world_size, rank=rank) + torch.cuda.set_device(rank) + default_dtype = torch.get_default_dtype() + torch.set_default_dtype(hf_config.torch_dtype) + torch.set_default_device("cuda") + self.model = Qwen3ForCausalLM(hf_config) + load_model(self.model, config.model) + self.sampler = Sampler() + self.warmup_model() + self.allocate_kv_cache() + if not self.enforce_eager: + self.capture_cudagraph() + torch.set_default_device("cpu") + torch.set_default_dtype(default_dtype) + + if self.world_size > 1: + if rank == 0: + self.shm = SharedMemory(name="atom", create=True, size=2**20) + dist.barrier() + else: + dist.barrier() + self.shm = SharedMemory(name="atom") + self.loop() + + def exit(self): + if self.world_size > 1: + self.shm.close() + dist.barrier() + if self.rank == 0: + self.shm.unlink() + if not self.enforce_eager: + del self.graphs, self.graph_pool + torch.cuda.synchronize() + dist.destroy_process_group() + + def loop(self): + while True: + method_name, args = self.read_shm() + self.call(method_name, *args) + if method_name == "exit": + break + + def read_shm(self): + assert self.world_size > 1 and self.rank + self.event.wait() + n = int.from_bytes(self.shm.buf[0:4], "little") + method_name, *args = pickle.loads(self.shm.buf[4:n+4]) + self.event.clear() + return method_name, args + + def write_shm(self, method_name, *args): + assert self.world_size > 1 and not self.rank + data = pickle.dumps([method_name, *args]) + n = len(data) + self.shm.buf[0:4] = n.to_bytes(4, "little") + self.shm.buf[4:n+4] = data + for event in self.event: + event.set() + + def call(self, method_name, *args): + if self.world_size > 1 and self.rank == 0: + self.write_shm(method_name, *args) + method = getattr(self, method_name, None) + return method(*args) + + def warmup_model(self): + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + max_num_batched_tokens, max_model_len = self.config.max_num_batched_tokens, self.config.max_model_len + num_seqs = min(max_num_batched_tokens // max_model_len, self.config.max_num_seqs) + seqs = [Sequence([0] * max_model_len) for _ in range(num_seqs)] + self.run(seqs, True) + torch.cuda.empty_cache() + + def allocate_kv_cache(self): + config = self.config + hf_config = config.hf_config + free, total = torch.cuda.mem_get_info() + used = total - free + peak = torch.cuda.memory_stats()["allocated_bytes.all.peak"] + current = torch.cuda.memory_stats()["allocated_bytes.all.current"] + num_kv_heads = hf_config.num_key_value_heads // self.world_size + block_bytes = 2 * hf_config.num_hidden_layers * self.block_size * num_kv_heads * hf_config.head_dim * hf_config.torch_dtype.itemsize + config.num_kvcache_blocks = int(total * config.gpu_memory_utilization - used - peak + current) // block_bytes + assert config.num_kvcache_blocks > 0 + self.kv_cache = torch.zeros(2, hf_config.num_hidden_layers, config.num_kvcache_blocks, self.block_size, num_kv_heads, hf_config.head_dim) + layer_id = 0 + for module in self.model.modules(): + if hasattr(module, "k_cache") and hasattr(module, "v_cache"): + module.k_cache = self.kv_cache[0, layer_id] + module.v_cache = self.kv_cache[1, layer_id] + layer_id += 1 + + def prepare_block_tables(self, seqs: list[Sequence]): + max_len = max(len(seq.block_table) for seq in seqs) + block_tables = [seq.block_table + [-1] * (max_len - len(seq.block_table)) for seq in seqs] + block_tables = torch.tensor(block_tables, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + return block_tables + + def prepare_prefill(self, seqs: list[Sequence]): + input_ids = [] + positions = [] + cu_seqlens_q = [0] + cu_seqlens_k = [0] + max_seqlen_q = 0 + max_seqlen_k = 0 + slot_mapping = [] + block_tables = None + for seq in seqs: + seqlen = len(seq) + input_ids.extend(seq[seq.num_cached_tokens:]) + positions.extend(list(range(seq.num_cached_tokens, seqlen))) + seqlen_q = seqlen - seq.num_cached_tokens + seqlen_k = seqlen + cu_seqlens_q.append(cu_seqlens_q[-1] + seqlen_q) + cu_seqlens_k.append(cu_seqlens_k[-1] + seqlen_k) + max_seqlen_q = max(seqlen_q, max_seqlen_q) + max_seqlen_k = max(seqlen_k, max_seqlen_k) + if not seq.block_table: + continue + for i in range(seq.num_cached_blocks, seq.num_blocks): + start = seq.block_table[i] * self.block_size + if i != seq.num_blocks - 1: + end = start + self.block_size + else: + end = start + seq.last_block_num_tokens + slot_mapping.extend(list(range(start, end))) + if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache + block_tables = self.prepare_block_tables(seqs) + input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) + positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) + cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + set_context(True, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, slot_mapping, None, block_tables) + return input_ids, positions + + def prepare_decode(self, seqs: list[Sequence]): + input_ids = [] + positions = [] + slot_mapping = [] + context_lens = [] + for seq in seqs: + input_ids.append(seq.last_token) + positions.append(len(seq)) + context_lens.append(len(seq)) + slot_mapping.append(seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1) + input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) + positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + block_tables = self.prepare_block_tables(seqs) + set_context(False, slot_mapping=slot_mapping, context_lens=context_lens, block_tables=block_tables) + return input_ids, positions + + def prepare_sample(self, seqs: list[Sequence]): + temperatures = [] + for seq in seqs: + temperatures.append(seq.temperature) + temperatures = torch.tensor(temperatures, dtype=torch.float32, pin_memory=True).cuda(non_blocking=True) + return temperatures + + @torch.inference_mode() + def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool): + if is_prefill or self.enforce_eager or input_ids.size(0) > 512: + return self.model.compute_logits(self.model(input_ids, positions)) + else: + bs = input_ids.size(0) + context = get_context() + graph = self.graphs[next(x for x in self.graph_bs if x >= bs)] + graph_vars = self.graph_vars + for k, v in graph_vars.items(): + if k != "outputs": + v.zero_() + graph_vars["input_ids"][:bs] = input_ids + graph_vars["positions"][:bs] = positions + graph_vars["slot_mapping"][:bs] = context.slot_mapping + graph_vars["context_lens"][:bs] = context.context_lens + graph_vars["block_tables"][:bs, :context.block_tables.size(1)] = context.block_tables + graph.replay() + return self.model.compute_logits(graph_vars["outputs"][:bs]) + + def run(self, seqs: list[Sequence], is_prefill: bool) -> list[int]: + input_ids, positions = self.prepare_prefill(seqs) if is_prefill else self.prepare_decode(seqs) + temperatures = self.prepare_sample(seqs) if self.rank == 0 else None + logits = self.run_model(input_ids, positions, is_prefill) + token_ids = self.sampler(logits, temperatures).tolist() if self.rank == 0 else None + reset_context() + return token_ids + + @torch.inference_mode() + def capture_cudagraph(self): + config = self.config + hf_config = config.hf_config + max_bs = min(self.config.max_num_seqs, 512) + max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size + input_ids = torch.zeros(max_bs, dtype=torch.int64) + positions = torch.zeros(max_bs, dtype=torch.int64) + slot_mapping = torch.zeros(max_bs, dtype=torch.int32) + context_lens = torch.zeros(max_bs, dtype=torch.int32) + block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32) + outputs = torch.zeros(max_bs, hf_config.hidden_size) + self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) + self.graphs = {} + self.graph_pool = None + + for bs in reversed(self.graph_bs): + graph = torch.cuda.CUDAGraph() + set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup + with torch.cuda.graph(graph, self.graph_pool): + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture + if self.graph_pool is None: + self.graph_pool = graph.pool() + self.graphs[bs] = graph + torch.cuda.synchronize() + reset_context() + + self.graph_vars = dict( + input_ids=input_ids, + positions=positions, + slot_mapping=slot_mapping, + context_lens=context_lens, + block_tables=block_tables, + outputs=outputs, + ) diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py new file mode 100644 index 0000000000..6bdc537a64 --- /dev/null +++ b/atom/model_engine/scheduler.py @@ -0,0 +1,71 @@ +from collections import deque + +from atom.config import Config +from atom.model_engine.sequence import Sequence, SequenceStatus +from atom.model_engine.block_manager import BlockManager + + +class Scheduler: + + def __init__(self, config: Config): + self.max_num_seqs = config.max_num_seqs + self.max_num_batched_tokens = config.max_num_batched_tokens + self.eos = config.eos + self.block_manager = BlockManager(config.num_kvcache_blocks, config.kvcache_block_size) + self.waiting: deque[Sequence] = deque() + self.running: deque[Sequence] = deque() + + def is_finished(self): + return not self.waiting and not self.running + + def add(self, seq: Sequence): + self.waiting.append(seq) + + def schedule(self) -> tuple[list[Sequence], bool]: + # prefill + scheduled_seqs = [] + num_seqs = 0 + num_batched_tokens = 0 + while self.waiting and num_seqs < self.max_num_seqs: + seq = self.waiting[0] + if num_batched_tokens + len(seq) > self.max_num_batched_tokens or not self.block_manager.can_allocate(seq): + break + num_seqs += 1 + self.block_manager.allocate(seq) + num_batched_tokens += len(seq) - seq.num_cached_tokens + seq.status = SequenceStatus.RUNNING + self.waiting.popleft() + self.running.append(seq) + scheduled_seqs.append(seq) + if scheduled_seqs: + return scheduled_seqs, True + + # decode + while self.running and num_seqs < self.max_num_seqs: + seq = self.running.popleft() + while not self.block_manager.can_append(seq): + if self.running: + self.preempt(self.running.pop()) + else: + self.preempt(seq) + break + else: + num_seqs += 1 + self.block_manager.may_append(seq) + scheduled_seqs.append(seq) + assert scheduled_seqs + self.running.extendleft(reversed(scheduled_seqs)) + return scheduled_seqs, False + + def preempt(self, seq: Sequence): + seq.status = SequenceStatus.WAITING + self.block_manager.deallocate(seq) + self.waiting.appendleft(seq) + + def postprocess(self, seqs: list[Sequence], token_ids: list[int]) -> list[bool]: + for seq, token_id in zip(seqs, token_ids): + seq.append_token(token_id) + if (not seq.ignore_eos and token_id == self.eos) or seq.num_completion_tokens == seq.max_tokens: + seq.status = SequenceStatus.FINISHED + self.block_manager.deallocate(seq) + self.running.remove(seq) diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py new file mode 100644 index 0000000000..5fb996a304 --- /dev/null +++ b/atom/model_engine/sequence.py @@ -0,0 +1,83 @@ +from copy import copy +from enum import Enum, auto +from itertools import count + +from atom.sampling_params import SamplingParams + + +class SequenceStatus(Enum): + WAITING = auto() + RUNNING = auto() + FINISHED = auto() + + +class Sequence: + block_size = 256 + counter = count() + + def __init__(self, token_ids: list[int], sampling_params = SamplingParams()): + self.seq_id = next(Sequence.counter) + self.status = SequenceStatus.WAITING + self.token_ids = copy(token_ids) + self.last_token = token_ids[-1] + self.num_tokens = len(self.token_ids) + self.num_prompt_tokens = len(token_ids) + self.num_cached_tokens = 0 + self.block_table = [] + self.temperature = sampling_params.temperature + self.max_tokens = sampling_params.max_tokens + self.ignore_eos = sampling_params.ignore_eos + + def __len__(self): + return self.num_tokens + + def __getitem__(self, key): + return self.token_ids[key] + + @property + def is_finished(self): + return self.status == SequenceStatus.FINISHED + + @property + def num_completion_tokens(self): + return self.num_tokens - self.num_prompt_tokens + + @property + def prompt_token_ids(self): + return self.token_ids[:self.num_prompt_tokens] + + @property + def completion_token_ids(self): + return self.token_ids[self.num_prompt_tokens:] + + @property + def num_cached_blocks(self): + return self.num_cached_tokens // self.block_size + + @property + def num_blocks(self): + return (self.num_tokens + self.block_size - 1) // self.block_size + + @property + def last_block_num_tokens(self): + return self.num_tokens - (self.num_blocks - 1) * self.block_size + + def block(self, i): + assert 0 <= i < self.num_blocks + return self.token_ids[i*self.block_size: (i+1)*self.block_size] + + def append_token(self, token_id: int): + self.token_ids.append(token_id) + self.last_token = token_id + self.num_tokens += 1 + + def __getstate__(self): + return (self.num_tokens, self.num_prompt_tokens, self.num_cached_tokens, self.block_table, + self.token_ids if self.num_completion_tokens == 0 else self.last_token) + + def __setstate__(self, state): + self.num_tokens, self.num_prompt_tokens, self.num_cached_tokens, self.block_table = state[:-1] + if self.num_completion_tokens == 0: + self.token_ids = state[-1] + else: + self.last_token = state[-1] diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py new file mode 100644 index 0000000000..74a6264381 --- /dev/null +++ b/atom/model_loader/loader.py @@ -0,0 +1,30 @@ +import os +from glob import glob +import torch +from torch import nn +from safetensors import safe_open + + +def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor): + param.data.copy_(loaded_weight) + + +def load_model(model: nn.Module, path: str): + packed_modules_mapping = getattr(model, "packed_modules_mapping", {}) + for file in glob(os.path.join(path, "*.safetensors")): + with safe_open(file, "pt", "cpu") as f: + for weight_name in f.keys(): + for k in packed_modules_mapping: + if k in weight_name: + v, shard_id = packed_modules_mapping[k] + param_name = weight_name.replace(k, v) + param = model.get_parameter(param_name) + weight_loader = getattr(param, "weight_loader") + weight_loader(param, f.get_tensor(weight_name), shard_id) + break + else: + param = model.get_parameter(weight_name) + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, f.get_tensor(weight_name)) diff --git a/atom/model_ops/activation.py b/atom/model_ops/activation.py new file mode 100644 index 0000000000..041ee20084 --- /dev/null +++ b/atom/model_ops/activation.py @@ -0,0 +1,14 @@ +import torch +from torch import nn +import torch.nn.functional as F + + +class SiluAndMul(nn.Module): + + def __init__(self): + super().__init__() + + @torch.compile + def forward(self, x: torch.Tensor) -> torch.Tensor: + x, y = x.chunk(2, -1) + return F.silu(x) * y diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py new file mode 100644 index 0000000000..c61125cb4d --- /dev/null +++ b/atom/model_ops/attention.py @@ -0,0 +1,95 @@ +import torch +from torch import nn +import triton +import triton.language as tl + +import aiter +from atom.utils.context import get_context + + +@triton.jit +def store_kvcache_kernel( + key_ptr, + key_stride, + value_ptr, + value_stride, + k_cache_ptr, + v_cache_ptr, + slot_mapping_ptr, + D: tl.constexpr, +): + idx = tl.program_id(0) + key_offsets = idx * key_stride + tl.arange(0, D) + value_offsets = idx * value_stride + tl.arange(0, D) + key = tl.load(key_ptr + key_offsets) + value = tl.load(value_ptr + value_offsets) + slot = tl.load(slot_mapping_ptr + idx) + cache_offsets = slot * D + tl.arange(0, D) + tl.store(k_cache_ptr + cache_offsets, key) + tl.store(v_cache_ptr + cache_offsets, value) + + +def store_kvcache( + key: torch.Tensor, + value: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + slot_mapping: torch.Tensor, +): + N, num_heads, head_dim = key.shape + D = num_heads * head_dim + assert key.stride(-1) == 1 and value.stride(-1) == 1 + assert key.stride(1) == head_dim and value.stride(1) == head_dim + assert k_cache.stride(1) == D and v_cache.stride(1) == D + assert slot_mapping.numel() == N + store_kvcache_kernel[(N,)]( + key, key.stride(0), value, value.stride(0), k_cache, v_cache, slot_mapping, D + ) + + +class Attention(nn.Module): + + def __init__( + self, + num_heads, + head_dim, + scale, + num_kv_heads, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = scale + self.num_kv_heads = num_kv_heads + self.k_cache = self.v_cache = torch.tensor([]) + + def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): + o: torch.Tensor + q = q.view(-1, self.num_heads, self.head_dim) + k = k.view(-1, self.num_kv_heads, self.head_dim) + v = v.view(-1, self.num_kv_heads, self.head_dim) + context = get_context() + k_cache, v_cache = self.k_cache, self.v_cache + if k_cache.numel() and v_cache.numel(): + store_kvcache(k, v, k_cache, v_cache, context.slot_mapping) + if context.is_prefill: + if context.block_tables is not None: # prefix cache + k, v = k_cache, v_cache + o = aiter.flash_attn_varlen_func( + q, + k, + v, + max_seqlen_q=context.max_seqlen_q, + cu_seqlens_q=context.cu_seqlens_q, + max_seqlen_k=context.max_seqlen_k, + cu_seqlens_k=context.cu_seqlens_k, + softmax_scale=self.scale, + causal=True, + ) + else: # decode + o = torch.empty_like(q, dtype=q.dtype, device=q.device) + # o = flash_attn_with_kvcache(q.unsqueeze(1), k_cache, v_cache, + # cache_seqlens=context.context_lens, block_table=context.block_tables, + # softmax_scale=self.scale, causal=True) + o = o.view(-1, self.num_heads * self.head_dim) + return o diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py new file mode 100644 index 0000000000..24a6f3e4f7 --- /dev/null +++ b/atom/model_ops/embed_head.py @@ -0,0 +1,71 @@ +import torch +from torch import nn +import torch.nn.functional as F +import torch.distributed as dist + +from atom.utils.context import get_context + + +class VocabParallelEmbedding(nn.Module): + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + ): + super().__init__() + self.tp_rank = dist.get_rank() + self.tp_size = dist.get_world_size() + assert num_embeddings % self.tp_size == 0 + self.num_embeddings = num_embeddings + self.num_embeddings_per_partition = self.num_embeddings // self.tp_size + self.vocab_start_idx = self.num_embeddings_per_partition * self.tp_rank + self.vocab_end_idx = self.vocab_start_idx + self.num_embeddings_per_partition + self.weight = nn.Parameter(torch.empty(self.num_embeddings_per_partition, embedding_dim)) + self.weight.weight_loader = self.weight_loader + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + shard_size = param_data.size(0) + start_idx = self.tp_rank * shard_size + loaded_weight = loaded_weight.narrow(0, start_idx, shard_size) + assert param_data.size() == loaded_weight.size() + param_data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor): + if self.tp_size > 1: + mask = (x >= self.vocab_start_idx) & (x < self.vocab_end_idx) + x = mask * (x - self.vocab_start_idx) + y = F.embedding(x, self.weight) + if self.tp_size > 1: + y = mask.unsqueeze(1) * y + dist.all_reduce(y) + return y + + +class ParallelLMHead(VocabParallelEmbedding): + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + bias: bool = False, + ): + super().__init__(num_embeddings, embedding_dim) + if bias: + self.bias = nn.Parameter(torch.empty(self.num_embeddings_per_partition)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) + + def forward(self, x: torch.Tensor): + context = get_context() + if context.is_prefill: + last_indices = context.cu_seqlens_q[1:] - 1 + x = x[last_indices].contiguous() + logits = F.linear(x, self.weight, self.bias) + if self.tp_size > 1: + all_logits = [torch.empty_like(logits) for _ in range(self.tp_size)] if self.tp_rank == 0 else None + dist.gather(logits, all_logits, 0) + logits = torch.cat(all_logits, -1) if self.tp_rank == 0 else None + return logits diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py new file mode 100644 index 0000000000..32dcfa2cb8 --- /dev/null +++ b/atom/model_ops/layernorm.py @@ -0,0 +1,51 @@ +import torch +from torch import nn + + +class RMSNorm(nn.Module): + + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.eps = eps + self.weight = nn.Parameter(torch.ones(hidden_size)) + + @torch.compile + def rms_forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + orig_dtype = x.dtype + x = x.to(torch.float32) + var = x.pow(2).mean(dim=-1, keepdim=True) + x.mul_(torch.rsqrt(var + self.eps)) + x = x.to(orig_dtype).mul_(self.weight) + return x + + @torch.compile + def add_rms_forward( + self, + x: torch.Tensor, + residual: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + orig_dtype = x.dtype + x = x.to(torch.float32).add_(residual.to(torch.float32)) + residual = x.to(orig_dtype) + var = x.pow(2).mean(dim=-1, keepdim=True) + x.mul_(torch.rsqrt(var + self.eps)) + x = x.to(orig_dtype).mul_(self.weight) + return x, residual + + def forward( + self, + x: torch.Tensor, + residual: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if residual is None: + return self.rms_forward(x) + else: + return self.add_rms_forward(x, residual) diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py new file mode 100644 index 0000000000..c625a9ebd0 --- /dev/null +++ b/atom/model_ops/linear.py @@ -0,0 +1,174 @@ +import torch +from torch import nn +import torch.nn.functional as F +import torch.distributed as dist + + +def divide(numerator, denominator): + assert numerator % denominator == 0 + return numerator // denominator + + +class LinearBase(nn.Module): + + def __init__( + self, + input_size: int, + output_size: int, + tp_dim: int | None = None, + ): + super().__init__() + self.input_size = input_size + self.output_size = output_size + self.tp_dim = tp_dim + self.tp_rank = dist.get_rank() + self.tp_size = dist.get_world_size() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + raise NotImplementedError + + +class ReplicatedLinear(LinearBase): + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = False, + ): + super().__init__(input_size, output_size) + self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) + + +class ColumnParallelLinear(LinearBase): + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = False, + ): + super().__init__(input_size, output_size, 0) + self.input_size_per_partition = input_size + self.output_size_per_partition = divide(output_size, self.tp_size) + + self.weight = nn.Parameter(torch.empty(self.output_size_per_partition, self.input_size)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size_per_partition)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + shard_size = param_data.size(self.tp_dim) + start_idx = self.tp_rank * shard_size + loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) + param_data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) + + +class MergedColumnParallelLinear(ColumnParallelLinear): + + def __init__( + self, + input_size: int, + output_sizes: list[int], + bias: bool = False, + ): + self.output_sizes = output_sizes + super().__init__(input_size, sum(output_sizes), bias=bias) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int): + param_data = param.data + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size + shard_size = self.output_sizes[loaded_shard_id] // self.tp_size + param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) + loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] + param_data.copy_(loaded_weight) + + +class QKVParallelLinear(ColumnParallelLinear): + + def __init__( + self, + hidden_size: int, + head_size: int, + total_num_heads: int, + total_num_kv_heads: int | None = None, + bias: bool = False, + ): + self.head_size = head_size + self.total_num_heads = total_num_heads + self.total_num_kv_heads = total_num_kv_heads or total_num_heads + tp_size = dist.get_world_size() + self.num_heads = divide(self.total_num_heads, tp_size) + self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) + input_size = hidden_size + output_size = (self.total_num_heads + 2 * self.total_num_kv_heads) * self.head_size + super().__init__(input_size, output_size, bias) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str): + param_data = param.data + assert loaded_shard_id in ["q", "k", "v"] + if loaded_shard_id == "q": + shard_size = self.num_heads * self.head_size + shard_offset = 0 + elif loaded_shard_id == "k": + shard_size = self.num_kv_heads * self.head_size + shard_offset = self.num_heads * self.head_size + else: + shard_size = self.num_kv_heads * self.head_size + shard_offset = self.num_heads * self.head_size + self.num_kv_heads * self.head_size + param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) + loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] + param_data.copy_(loaded_weight) + + +class RowParallelLinear(LinearBase): + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = False, + ): + super().__init__(input_size, output_size, 1) + self.input_size_per_partition = divide(input_size, self.tp_size) + self.output_size_per_partition = output_size + + self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size_per_partition)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + shard_size = param_data.size(self.tp_dim) + start_idx = self.tp_rank * shard_size + loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) + param_data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None) + if self.tp_size > 1: + dist.all_reduce(y) + return y diff --git a/atom/model_ops/rotary_embedding.py b/atom/model_ops/rotary_embedding.py new file mode 100644 index 0000000000..c473420f56 --- /dev/null +++ b/atom/model_ops/rotary_embedding.py @@ -0,0 +1,68 @@ +from functools import lru_cache +import torch +from torch import nn + + +def apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, +) -> torch.Tensor: + cos = cos.unsqueeze(-2) + sin = sin.unsqueeze(-2) + x1, x2 = torch.chunk(x.to(torch.float32), 2, dim=-1) + y1 = x1 * cos - x2 * sin + y2 = x2 * cos + x1 * sin + return torch.cat((y1, y2), dim=-1).to(x.dtype) + + +class RotaryEmbedding(nn.Module): + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: float, + ) -> None: + super().__init__() + self.head_size = head_size + assert rotary_dim == head_size + inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim)) + t = torch.arange(max_position_embeddings, dtype=torch.float) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + self.register_buffer("cos_sin_cache", cache, persistent=False) + + @torch.compile + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + num_tokens = positions.size(0) + cos_sin = self.cos_sin_cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query = apply_rotary_emb(query, cos, sin).view(query_shape) + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key = apply_rotary_emb(key, cos, sin).view(key_shape) + return query, key + + +@lru_cache(1) +def get_rope( + head_size: int, + rotary_dim: int, + max_position: int, + base: float, + rope_scaling: dict | None = None, +): + assert rope_scaling is None + rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base) + return rotary_emb diff --git a/atom/model_ops/sampler.py b/atom/model_ops/sampler.py new file mode 100644 index 0000000000..e4b9816e70 --- /dev/null +++ b/atom/model_ops/sampler.py @@ -0,0 +1,18 @@ +import torch +from torch import nn + + +class Sampler(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, logits: torch.Tensor, temperatures: torch.Tensor): + logits = logits.to(torch.float) + greedy_tokens = logits.argmax(dim=-1) + logits.div_(temperatures.unsqueeze(dim=1)) + probs = torch.softmax(logits, dim=-1, dtype=torch.float) + # logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float) + epsilon = 1e-10 + sample_tokens = probs.div_(torch.empty_like(probs).exponential_(1) + epsilon).argmax(dim=-1) + return torch.where(temperatures == 0, greedy_tokens, sample_tokens) diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py new file mode 100644 index 0000000000..dff1ff7648 --- /dev/null +++ b/atom/models/deepseek_v2.py @@ -0,0 +1,1717 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Adapted from: +# https://github.com/vllm-project/vllm/blob/fb6af8bc086328ca6659e72d11ffd4309ce4de22/vllm/model_executor/models/deepseek_v2.py +"""Inference-only DeepseekV2 model.""" + +import logging +import os +from dataclasses import dataclass +from enum import Enum, IntEnum, auto +from typing import Any, Dict, Iterable, Optional, Tuple + +import torch +import torch.nn.functional as F +from torch import nn +from tqdm import tqdm +from transformers import PretrainedConfig + +from sglang.srt.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + parallel_state, + tensor_model_parallel_all_reduce, +) +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.dp_attention import ( + dp_gather_partial, + dp_scatter, + get_attention_dp_size, + get_attention_tp_rank, + get_attention_tp_size, + tp_all_gather, + tp_reduce_scatter, +) +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + ReplicatedLinear, + RowParallelLinear, +) +from sglang.srt.layers.logits_processor import LogitsProcessor +from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, EPMoE +from sglang.srt.layers.moe.ep_moe.token_dispatcher import DeepEPDispatcher +from sglang.srt.layers.moe.fused_moe_triton import FusedMoE +from sglang.srt.layers.moe.topk import select_experts +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.fp8_kernel import ( + _enable_jit_deepgemm_bmm, + per_tensor_quant_mla_deep_gemm_masked_fp8, + per_tensor_quant_mla_fp8, +) +from sglang.srt.layers.quantization.fp8_utils import ( + block_quant_to_tensor_quant, + channel_quant_to_tensor_quant, + normalize_e4m3fn_to_e4m3fnuz, +) +from sglang.srt.layers.quantization.int8_utils import ( + block_dequant as int8_block_dequant, +) +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.layers.rotary_embedding import get_rope, get_rope_wrapper +from sglang.srt.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from sglang.srt.managers.expert_distribution import ExpertDistributionRecorder +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_loader.weight_utils import default_weight_loader +from sglang.srt.utils import BumpAllocator, DeepEPMode, add_prefix, is_cuda, is_hip + +_is_hip = is_hip() +_is_cuda = is_cuda() + +if _is_cuda: + from deep_gemm import m_grouped_gemm_fp8_fp8_bf16_nt_masked + from sgl_kernel import awq_dequantize, bmm_fp8, merge_state_v2 +else: + from vllm._custom_ops import awq_dequantize + +if _is_hip: + from sglang.srt.layers.attention.triton_ops.rocm_mla_decode_rope import ( + decode_attention_fwd_grouped_rope, + ) + +expert_distribution_recorder = ExpertDistributionRecorder() + +logger = logging.getLogger(__name__) + + +class AttnForwardMethod(IntEnum): + # Use multi-head attention + MHA = auto() + + # Use absorbed multi-latent attention + MLA = auto() + + # Use multi-head attention, but with KV cache chunked. + # This method can avoid OOM when prefix lengths are long. + MHA_CHUNKED_KV = auto() + + +class DeepseekV2MLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + prefix: str = "", + tp_rank: Optional[int] = None, + tp_size: Optional[int] = None, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=add_prefix("gate_up_proj", prefix), + tp_rank=tp_rank, + tp_size=tp_size, + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=add_prefix("down_proj", prefix), + tp_rank=tp_rank, + tp_size=tp_size, + ) + if hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now." + ) + self.act_fn = SiluAndMul() + + def forward(self, x, forward_mode: Optional[ForwardMode] = None): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class MoEGate(nn.Module): + def __init__( + self, + config, + prefix: str = "", + ): + super().__init__() + self.weight = nn.Parameter( + torch.empty((config.n_routed_experts, config.hidden_size)) + ) + if config.topk_method == "noaux_tc": + self.e_score_correction_bias = nn.Parameter( + torch.empty((config.n_routed_experts)) + ) + else: + self.e_score_correction_bias = None + + def forward(self, hidden_states): + logits = F.linear(hidden_states, self.weight, None) + return logits + + +class DeepseekV2MoE(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.routed_scaling_factor = config.routed_scaling_factor + self.n_shared_experts = config.n_shared_experts + self.n_share_experts_fusion = global_server_args_dict["n_share_experts_fusion"] + + if self.tp_size > config.n_routed_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.n_routed_experts}." + ) + + if config.hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {config.hidden_act}. " + "Only silu is supported for now." + ) + + self.gate = MoEGate(config=config, prefix=add_prefix("gate", prefix)) + + MoEImpl = ( + DeepEPMoE + if global_server_args_dict["enable_deepep_moe"] + else (EPMoE if global_server_args_dict["enable_ep_moe"] else FusedMoE) + ) + + self.experts = MoEImpl( + num_experts=config.n_routed_experts + self.n_share_experts_fusion, + top_k=config.num_experts_per_tok + min(self.n_share_experts_fusion, 1), + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + renormalize=config.norm_topk_prob, + quant_config=quant_config, + use_grouped_topk=True, + num_expert_group=config.n_group, + topk_group=config.topk_group, + correction_bias=self.gate.e_score_correction_bias, + routed_scaling_factor=self.routed_scaling_factor, + prefix=add_prefix("experts", prefix), + **( + dict(deepep_mode=DeepEPMode[global_server_args_dict["deepep_mode"]]) + if global_server_args_dict["enable_deepep_moe"] + else {} + ), + ) + + if config.n_shared_experts is not None and self.n_share_experts_fusion == 0: + intermediate_size = config.moe_intermediate_size * config.n_shared_experts + # disable tp for shared experts when enable deepep moe + if not global_server_args_dict["enable_deepep_moe"]: + self.shared_experts = DeepseekV2MLP( + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + prefix=add_prefix("shared_experts", prefix), + ) + else: + self.shared_experts = DeepseekV2MLP( + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + prefix=add_prefix("shared_experts", prefix), + tp_rank=0, + tp_size=1, + ) + + if global_server_args_dict["enable_deepep_moe"]: + # TODO: we will support tp < ep in the future + self.ep_size = get_tensor_model_parallel_world_size() + self.num_experts = config.n_routed_experts + self.top_k = config.num_experts_per_tok + self.renormalize = config.norm_topk_prob + self.topk_group = config.topk_group + self.num_expert_group = config.n_group + self.correction_bias = ( + self.gate.e_score_correction_bias.data + if self.gate.e_score_correction_bias is not None + else None + ) + + self.deepep_dispatcher = DeepEPDispatcher( + group=parallel_state.get_tp_group().device_group, + router_topk=self.top_k, + permute_fusion=True, + num_experts=config.n_routed_experts, + num_local_experts=config.n_routed_experts // self.tp_size, + hidden_size=config.hidden_size, + params_dtype=config.torch_dtype, + deepep_mode=DeepEPMode[global_server_args_dict["deepep_mode"]], + async_finish=True, # TODO + return_recv_hook=True, + ) + + def forward( + self, hidden_states: torch.Tensor, forward_mode: Optional[ForwardMode] = None + ) -> torch.Tensor: + if not global_server_args_dict["enable_deepep_moe"]: + return self.forward_normal(hidden_states) + else: + return self.forward_deepep(hidden_states, forward_mode) + + def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor: + shared_output = self._forward_shared_experts(hidden_states) + # router_logits: (num_tokens, n_experts) + router_logits = self.gate(hidden_states) + final_hidden_states = ( + self.experts(hidden_states=hidden_states, router_logits=router_logits) + * self.routed_scaling_factor + ) + if shared_output is not None: + final_hidden_states = final_hidden_states + shared_output + if self.tp_size > 1: + final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + return final_hidden_states + + def forward_deepep( + self, hidden_states: torch.Tensor, forward_mode: ForwardMode + ) -> torch.Tensor: + shared_output = None + topk_idx = torch.full( + (0, self.top_k), -1, dtype=torch.int, device=hidden_states.device + ) + topk_weights = torch.empty( + (0, self.top_k), dtype=torch.float32, device=hidden_states.device + ) + if ( + forward_mode is not None + and not forward_mode.is_idle() + and hidden_states.shape[0] > 0 + ): + # router_logits: (num_tokens, n_experts) + router_logits = self.gate(hidden_states) + shared_output = self._forward_shared_experts(hidden_states) + topk_weights, topk_idx = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + top_k=self.top_k, + use_grouped_topk=True, + renormalize=self.renormalize, + topk_group=self.topk_group, + num_expert_group=self.num_expert_group, + correction_bias=self.correction_bias, + routed_scaling_factor=self.routed_scaling_factor, + ) + if self.ep_size > 1: + # TODO(ch-wan): allow users to set num_max_dispatch_tokens_per_rank value + ( + hidden_states, + topk_idx, + topk_weights, + reorder_topk_ids, + seg_indptr, + masked_m, + expected_m, + ) = self.deepep_dispatcher.dispatch( + hidden_states, + topk_idx, + topk_weights, + forward_mode=forward_mode, + ) + final_hidden_states = self.experts( + hidden_states=hidden_states, + reorder_topk_ids=reorder_topk_ids, + seg_indptr=seg_indptr, + masked_m=masked_m, + expected_m=expected_m, + forward_mode=forward_mode, + ) + if self.ep_size > 1: + final_hidden_states = self.deepep_dispatcher.combine( + final_hidden_states, + topk_idx, + topk_weights, + forward_mode, + ) + final_hidden_states *= self.routed_scaling_factor + + if shared_output is not None: + final_hidden_states = final_hidden_states + shared_output + + return final_hidden_states + + def _forward_shared_experts(self, hidden_states): + if self.n_share_experts_fusion == 0: + return self.shared_experts(hidden_states) + else: + return None + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + import math + + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +class DeepseekV2AttentionMLA(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + q_lora_rank: int, + kv_lora_rank: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + layer_id: int = None, + prefix: str = "", + ) -> None: + super().__init__() + self.layer_id = layer_id + self.hidden_size = hidden_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.dp_size = get_attention_dp_size() + attn_tp_rank = get_attention_tp_rank() + attn_tp_size = get_attention_tp_size() + + self.num_heads = num_heads + assert num_heads % attn_tp_size == 0 + self.num_local_heads = num_heads // attn_tp_size + self.scaling = self.qk_head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + # For tensor parallel attention + if self.q_lora_rank is not None: + self.q_a_proj = ReplicatedLinear( + self.hidden_size, + self.q_lora_rank, + bias=False, + quant_config=quant_config, + prefix=add_prefix("q_a_proj", prefix), + ) + self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) + self.q_b_proj = ColumnParallelLinear( + q_lora_rank, + self.num_heads * self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=add_prefix("q_b_proj", prefix), + tp_rank=attn_tp_rank, + tp_size=attn_tp_size, + ) + else: + self.q_proj = ColumnParallelLinear( + self.hidden_size, + self.num_heads * self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=add_prefix("q_proj", prefix), + tp_rank=attn_tp_rank, + tp_size=attn_tp_size, + ) + self.kv_b_proj = ColumnParallelLinear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config, + prefix=add_prefix("kv_b_proj", prefix), + tp_rank=attn_tp_rank, + tp_size=attn_tp_size, + ) + # O projection. + self.o_proj = RowParallelLinear( + self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=add_prefix("o_proj", prefix), + tp_rank=attn_tp_rank, + tp_size=attn_tp_size, + ) + + self.kv_a_proj_with_mqa = ReplicatedLinear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config, + prefix=add_prefix("kv_a_proj_with_mqa", prefix), + ) + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps) + + if rope_scaling: + rope_scaling["rope_type"] = "deepseek_yarn" + + self.rotary_emb = get_rope( + qk_rope_head_dim, + rotary_dim=qk_rope_head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=False, + ) + + if rope_scaling: + mscale_all_dim = rope_scaling.get("mscale_all_dim", False) + scaling_factor = rope_scaling["factor"] + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) + self.scaling = self.scaling * mscale * mscale + else: + self.rotary_emb.forward = self.rotary_emb.forward_native + + self.attn_mqa = RadixAttention( + self.num_local_heads, + self.kv_lora_rank + self.qk_rope_head_dim, + self.scaling, + num_kv_heads=1, + layer_id=layer_id, + v_head_dim=self.kv_lora_rank, + quant_config=quant_config, + prefix=add_prefix("attn_mqa", prefix), + ) + + self.attn_mha = RadixAttention( + self.num_local_heads, + self.qk_nope_head_dim + self.qk_rope_head_dim, + self.scaling, + num_kv_heads=self.num_local_heads, + layer_id=layer_id, + v_head_dim=self.v_head_dim, + quant_config=quant_config, + prefix=add_prefix("attn_mha", prefix), + ) + + self.w_kc = None + self.w_vc = None + self.w_scale = None + + self.w_scale_k = None + self.w_scale_v = None + self.use_deep_gemm_bmm = False + + self.flashinfer_mla_disable_ragged = global_server_args_dict[ + "flashinfer_mla_disable_ragged" + ] + self.disable_chunked_prefix_cache = global_server_args_dict[ + "disable_chunked_prefix_cache" + ] + self.attention_backend = global_server_args_dict["attention_backend"] + self.rocm_fused_decode_mla = os.getenv("SGLANG_ROCM_FUSED_DECODE_MLA") == "1" + + # TODO: Design a finer way to determine the threshold + self.chunked_prefix_cache_threshold = 8192 + + def dispatch_attn_forward_method( + self, forward_batch: ForwardBatch + ) -> AttnForwardMethod: + if self.attention_backend == "flashinfer": + # Flashinfer MLA: Do not absorb when enabling ragged prefill + if ( + not self.flashinfer_mla_disable_ragged + and forward_batch.forward_mode.is_extend() + and not forward_batch.forward_mode.is_target_verify() + and not forward_batch.forward_mode.is_draft_extend() + and sum(forward_batch.extend_prefix_lens_cpu) == 0 + ): + return AttnForwardMethod.MHA + else: + return AttnForwardMethod.MLA + elif self.attention_backend == "fa3": + # Flash Attention: Use MHA with chunked KV cache when prefilling on long sequences. + if ( + forward_batch.forward_mode.is_extend() + and not self.disable_chunked_prefix_cache + and not forward_batch.forward_mode.is_target_verify() + and not forward_batch.forward_mode.is_draft_extend() + and sum(forward_batch.extend_prefix_lens_cpu) + >= self.chunked_prefix_cache_threshold + ): + return AttnForwardMethod.MHA_CHUNKED_KV + else: + return AttnForwardMethod.MLA + else: + # Triton: Use normal computation for prefill and use weight absorption for extend/decode + if ( + forward_batch.forward_mode.is_extend() + and not forward_batch.forward_mode.is_target_verify() + and not forward_batch.forward_mode.is_draft_extend() + and sum(forward_batch.extend_prefix_lens_cpu) == 0 + ): + return AttnForwardMethod.MHA + else: + return AttnForwardMethod.MLA + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + if hidden_states.shape[0] == 0: + assert ( + not self.o_proj.reduce_results + ), "short-circuiting allreduce will lead to hangs" + return hidden_states + + attn_forward_method = self.dispatch_attn_forward_method(forward_batch) + + if attn_forward_method == AttnForwardMethod.MHA: + return self.forward_normal(positions, hidden_states, forward_batch) + elif attn_forward_method == AttnForwardMethod.MHA_CHUNKED_KV: + return self.forward_normal_chunked_kv( + positions, hidden_states, forward_batch + ) + else: + return self.forward_absorb( + positions, hidden_states, forward_batch, zero_allocator + ) + + def forward_normal( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + if self.q_lora_rank is not None: + q = self.q_a_proj(hidden_states)[0] + q = self.q_a_layernorm(q) + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) + else: + q = self.q_proj(hidden_states)[0].view( + -1, self.num_local_heads, self.qk_head_dim + ) + _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] + kv_a, _ = latent_cache.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + latent_cache = latent_cache.unsqueeze(1) + kv_a = self.kv_a_layernorm(kv_a.contiguous()) + kv = self.kv_b_proj(kv_a)[0] + kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim) + k_nope = kv[..., : self.qk_nope_head_dim] + v = kv[..., self.qk_nope_head_dim :] + k_pe = latent_cache[:, :, self.kv_lora_rank :] + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) + q[..., self.qk_nope_head_dim :] = q_pe + k = torch.empty_like(q) + k[..., : self.qk_nope_head_dim] = k_nope + k[..., self.qk_nope_head_dim :] = k_pe + + latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1) + latent_cache[:, :, self.kv_lora_rank :] = k_pe + + # Save latent cache + forward_batch.token_to_kv_pool.set_kv_buffer( + self.attn_mha, forward_batch.out_cache_loc, latent_cache, None + ) + attn_output = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False) + attn_output = attn_output.reshape(-1, self.num_local_heads * self.v_head_dim) + output, _ = self.o_proj(attn_output) + return output + + def forward_absorb( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + q_len = hidden_states.shape[0] + q_input = hidden_states.new_empty( + q_len, self.num_local_heads, self.kv_lora_rank + self.qk_rope_head_dim + ) + if self.q_lora_rank is not None: + q = self.q_a_proj(hidden_states)[0] + q = self.q_a_layernorm(q) + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) + else: + q = self.q_proj(hidden_states)[0].view( + -1, self.num_local_heads, self.qk_head_dim + ) + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + + if self.use_deep_gemm_bmm: + q_nope_val, q_nope_scale, masked_m, expected_m, aligned_m = ( + per_tensor_quant_mla_deep_gemm_masked_fp8( + q_nope.transpose(0, 1), dtype=torch.float8_e4m3fn + ) + ) + q_nope_out = q_nope.new_empty( + (self.num_local_heads, aligned_m, self.kv_lora_rank) + ) + m_grouped_gemm_fp8_fp8_bf16_nt_masked( + (q_nope_val, q_nope_scale), + (self.w_kc, self.w_scale_k), + q_nope_out, + masked_m, + expected_m, + ) + q_nope_out = q_nope_out[:, :expected_m, :] + elif self.w_kc.dtype == torch.float8_e4m3fnuz: + # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz + q_nope_out = torch.bmm( + q_nope.to(torch.bfloat16).transpose(0, 1), + self.w_kc.to(torch.bfloat16) * self.w_scale, + ) + elif self.w_kc.dtype == torch.float8_e4m3fn: + q_nope_val, q_nope_scale = per_tensor_quant_mla_fp8( + q_nope.transpose(0, 1), + zero_allocator.allocate(1), + ) + q_nope_out = bmm_fp8( + q_nope_val, self.w_kc, q_nope_scale, self.w_scale, torch.bfloat16 + ) + else: + q_nope_out = torch.bmm(q_nope.transpose(0, 1), self.w_kc) + q_input[..., : self.kv_lora_rank] = q_nope_out.transpose(0, 1) + + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] + v_input = latent_cache[..., : self.kv_lora_rank] + v_input = self.kv_a_layernorm(v_input.contiguous()).unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., : self.kv_lora_rank] = v_input + k_pe = k_input[..., self.kv_lora_rank :] + + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) + q_input[..., self.kv_lora_rank :] = q_pe + k_input[..., self.kv_lora_rank :] = k_pe + + attn_output = self.attn_mqa(q_input, k_input, v_input, forward_batch) + attn_output = attn_output.view(-1, self.num_local_heads, self.kv_lora_rank) + + if self.use_deep_gemm_bmm: + attn_output_val, attn_output_scale, masked_m, expected_m, aligned_m = ( + per_tensor_quant_mla_deep_gemm_masked_fp8( + attn_output.transpose(0, 1), dtype=torch.float8_e4m3fn + ) + ) + attn_bmm_output = attn_output.new_empty( + (self.num_local_heads, aligned_m, self.v_head_dim) + ) + m_grouped_gemm_fp8_fp8_bf16_nt_masked( + (attn_output_val, attn_output_scale), + (self.w_vc, self.w_scale_v), + attn_bmm_output, + masked_m, + expected_m, + ) + attn_bmm_output = attn_bmm_output[:, :expected_m, :] + elif self.w_vc.dtype == torch.float8_e4m3fnuz: + # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz + attn_bmm_output = torch.bmm( + attn_output.to(torch.bfloat16).transpose(0, 1), + self.w_vc.to(torch.bfloat16) * self.w_scale, + ) + elif self.w_vc.dtype == torch.float8_e4m3fn: + attn_output_val, attn_output_scale = per_tensor_quant_mla_fp8( + attn_output.transpose(0, 1), + zero_allocator.allocate(1), + ) + attn_bmm_output = bmm_fp8( + attn_output_val, + self.w_vc, + attn_output_scale, + self.w_scale, + torch.bfloat16, + ) + else: + attn_bmm_output = torch.bmm(attn_output.transpose(0, 1), self.w_vc) + attn_output = attn_bmm_output.transpose(0, 1).flatten(1, 2) + output, _ = self.o_proj(attn_output) + + return output + + def forward_absorb_fused_mla_rope( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + enable_rope_fusion = ( + os.getenv("SGLANG_FUSED_MLA_ENABLE_ROPE_FUSION", "1") == "1" + ) + q_len = hidden_states.shape[0] + q_input = hidden_states.new_empty( + q_len, self.num_local_heads, self.kv_lora_rank + self.qk_rope_head_dim + ) + if self.q_lora_rank is not None: + q = self.q_a_proj(hidden_states)[0] + q = self.q_a_layernorm(q) + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) + else: + q = self.q_proj(hidden_states)[0].view( + -1, self.num_local_heads, self.qk_head_dim + ) + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + + if self.w_kc.dtype == torch.float8_e4m3fnuz: + # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz + q_nope_out = torch.bmm( + q_nope.to(torch.bfloat16).transpose(0, 1), + self.w_kc.to(torch.bfloat16) * self.w_scale, + ) + elif self.w_kc.dtype == torch.float8_e4m3fn: + q_nope_val, q_nope_scale = per_tensor_quant_mla_fp8( + q_nope.transpose(0, 1), + zero_allocator.allocate(1), + dtype=torch.float8_e4m3fn, + ) + q_nope_out = bmm_fp8( + q_nope_val, self.w_kc, q_nope_scale, self.w_scale, torch.bfloat16 + ) + else: + q_nope_out = torch.bmm(q_nope.transpose(0, 1), self.w_kc) + q_input[..., : self.kv_lora_rank] = q_nope_out.transpose(0, 1) + + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] + v_input = latent_cache[..., : self.kv_lora_rank] + v_input = self.kv_a_layernorm(v_input.contiguous()).unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., : self.kv_lora_rank] = v_input + + if not enable_rope_fusion: + k_pe = k_input[..., self.kv_lora_rank :] + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) + q_input[..., self.kv_lora_rank :] = q_pe + k_input[..., self.kv_lora_rank :] = k_pe + k_pe_output = None + else: + k_pe_output = torch.empty_like(k_input[..., self.kv_lora_rank :]) + + q_input[..., self.kv_lora_rank :] = q_pe + + # attn_output = self.attn_mqa(q_input, k_input, v_input, forward_batch) + # Use Fused ROPE with use_rope=OFF. + attn_output = torch.empty( + (q_len, self.num_local_heads, self.kv_lora_rank), + dtype=q.dtype, + device=q.device, + ) + attn_logits, _, kv_indptr, kv_indices, _, _, _ = ( + forward_batch.attn_backend.forward_metadata + ) + cos_sin_cache = self.rotary_emb.cos_sin_cache + num_kv_split = forward_batch.attn_backend.num_kv_splits + sm_scale = self.attn_mqa.scaling + if attn_logits is None: + attn_logits = torch.empty( + ( + forward_batch.batch_size, + self.num_local_heads, + num_kv_split, + self.kv_lora_rank + 1, + ), + dtype=torch.float32, + device=q.device, + ) + + # save current latent cache. + forward_batch.token_to_kv_pool.set_kv_buffer( + self.attn_mqa, forward_batch.out_cache_loc, k_input, None + ) + key_cache_buf = forward_batch.token_to_kv_pool.get_key_buffer( + self.attn_mqa.layer_id + ) + val_cache_buf = key_cache_buf[..., : self.kv_lora_rank] + + decode_attention_fwd_grouped_rope( + q_input, + key_cache_buf, + val_cache_buf, + attn_output, + kv_indptr, + kv_indices, + k_pe_output, + self.kv_lora_rank, + self.rotary_emb.rotary_dim, + cos_sin_cache, + positions, + attn_logits, + num_kv_split, + sm_scale, + logit_cap=self.attn_mqa.logit_cap, + use_rope=enable_rope_fusion, + is_neox_style=self.rotary_emb.is_neox_style, + ) + + if enable_rope_fusion: + k_input[..., self.kv_lora_rank :] = k_pe_output + forward_batch.token_to_kv_pool.set_kv_buffer( + self.attn_mqa, forward_batch.out_cache_loc, k_input, None + ) + + attn_output = attn_output.view(-1, self.num_local_heads, self.kv_lora_rank) + + if self.w_vc.dtype == torch.float8_e4m3fnuz: + # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz + attn_bmm_output = torch.bmm( + attn_output.to(torch.bfloat16).transpose(0, 1), + self.w_vc.to(torch.bfloat16) * self.w_scale, + ) + elif self.w_vc.dtype == torch.float8_e4m3fn: + attn_output_val, attn_output_scale = per_tensor_quant_mla_fp8( + attn_output.transpose(0, 1), + zero_allocator.allocate(1), + dtype=torch.float8_e4m3fn, + ) + attn_bmm_output = bmm_fp8( + attn_output_val, + self.w_vc, + attn_output_scale, + self.w_scale, + torch.bfloat16, + ) + else: + attn_bmm_output = torch.bmm(attn_output.transpose(0, 1), self.w_vc) + attn_output = attn_bmm_output.transpose(0, 1).flatten(1, 2) + output, _ = self.o_proj(attn_output) + + return output + + def _chunked_prefix_attn_mha( + self, + q: torch.Tensor, + accum_output: torch.Tensor, + accum_lse: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + + assert forward_batch.num_prefix_chunks is not None + for i in range(forward_batch.num_prefix_chunks): + forward_batch.set_prefix_chunk_idx(i) + + # Fetch latent cache from memory pool with precomputed chunked kv indices + latent_cache_buf = forward_batch.token_to_kv_pool.get_key_buffer( + self.attn_mha.layer_id + ) + latent_cache = latent_cache_buf[ + forward_batch.prefix_chunk_kv_indices[i] + ].contiguous() + + kv_a_normed, k_pe = latent_cache.split( + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 + ) + kv_a_normed = kv_a_normed.squeeze(1).contiguous() + kv = self.kv_b_proj(kv_a_normed)[0] + kv = kv.view( + -1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim + ) + v = kv[..., self.qk_nope_head_dim :] + k_nope = kv[..., : self.qk_nope_head_dim] + + k = torch.empty( + ( + k_nope.shape[0], + self.num_local_heads, + self.qk_nope_head_dim + self.qk_rope_head_dim, + ), + dtype=v.dtype, + device=v.device, + ) + k[..., : self.qk_nope_head_dim] = k_nope + k[..., self.qk_nope_head_dim :] = k_pe + + output, lse = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False) + lse = torch.transpose(lse, 0, 1).contiguous() + tmp_output = torch.empty_like(accum_output) + tmp_lse = torch.empty_like(accum_lse) + merge_state_v2(output, lse, accum_output, accum_lse, tmp_output, tmp_lse) + accum_output, accum_lse = tmp_output, tmp_lse + + return accum_output + + def forward_normal_chunked_kv( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> torch.Tensor: + # In normal mha, the k and v tensors will become overly large when the prefix length is long. + # To avoid this, we split the kv cache into chunks and process them one after another. + # Since mha is compute friendly, the for loop induced here will not introduce significant overhead. + # The top comments in https://github.com/vllm-project/vllm/blob/main/vllm/v1/attention/backends/mla/common.py + # will be helpful for understanding the purpose of this function. + + # First do normal mha forward to get output for extended part + if self.q_lora_rank is not None: + q = self.q_a_proj(hidden_states)[0] + q = self.q_a_layernorm(q) + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) + else: + q = self.q_proj(hidden_states)[0].view( + -1, self.num_local_heads, self.qk_head_dim + ) + _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] + kv_a, _ = latent_cache.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + latent_cache = latent_cache.unsqueeze(1) + kv_a = self.kv_a_layernorm(kv_a.contiguous()) + kv = self.kv_b_proj(kv_a)[0] + kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim) + k_nope = kv[..., : self.qk_nope_head_dim] + v = kv[..., self.qk_nope_head_dim :] + k_pe = latent_cache[:, :, self.kv_lora_rank :] + + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) + q[..., self.qk_nope_head_dim :] = q_pe + k = torch.empty_like(q) + k[..., : self.qk_nope_head_dim] = k_nope + k[..., self.qk_nope_head_dim :] = k_pe + + latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1) + latent_cache[:, :, self.kv_lora_rank :] = k_pe + + # Save latent cache + forward_batch.token_to_kv_pool.set_kv_buffer( + self.attn_mha, forward_batch.out_cache_loc, latent_cache, None + ) + + # Do mha for extended part without prefix + forward_batch.set_attn_attend_prefix_cache(False) + attn_output, lse = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False) + lse = torch.transpose(lse, 0, 1).contiguous() + + # Do mha attention with chunked prefix cache if there are any sequence with prefix + if any(forward_batch.extend_prefix_lens_cpu): + # Only initialize the info once + if forward_batch.num_prefix_chunks is None: + forward_batch.prepare_chunked_prefix_cache_info(q.device) + + forward_batch.set_attn_attend_prefix_cache(True) + attn_output = self._chunked_prefix_attn_mha( + q=q, + accum_output=attn_output, + accum_lse=lse, + forward_batch=forward_batch, + ) + + attn_output = attn_output.reshape(-1, self.num_local_heads * self.v_head_dim) + output, _ = self.o_proj(attn_output) + return output + + +class _FFNInputMode(Enum): + # The MLP sublayer requires 1/tp_size tokens as input + SCATTERED = auto() + # The MLP sublayer requires all tokens as input + FULL = auto() + + +@dataclass +class _DecoderLayerInfo: + is_sparse: bool + ffn_input_mode: _FFNInputMode + + +class DeepseekV2DecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + layer_id: int, + quant_config: Optional[QuantizationConfig] = None, + is_nextn: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", 8192) + self.enable_dp_attention = global_server_args_dict["enable_dp_attention"] + self.layer_id = layer_id + self.dp_size = get_attention_dp_size() + self.attn_tp_size = get_attention_tp_size() + self.attn_tp_rank = get_attention_tp_rank() + self.self_attn = DeepseekV2AttentionMLA( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + qk_nope_head_dim=config.qk_nope_head_dim, + qk_rope_head_dim=config.qk_rope_head_dim, + v_head_dim=config.v_head_dim, + q_lora_rank=( + config.q_lora_rank if hasattr(config, "q_lora_rank") else None + ), + kv_lora_rank=config.kv_lora_rank, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + layer_id=layer_id, + reduce_results=False, + prefix=add_prefix("self_attn", prefix), + ) + + self.info = self._compute_info(config, layer_id=layer_id, is_nextn=is_nextn) + previous_layer_info = self._compute_info( + config, layer_id=layer_id - 1, is_nextn=False + ) + + if self.info.is_sparse: + self.mlp = DeepseekV2MoE( + config=config, + quant_config=quant_config, + prefix=add_prefix("mlp", prefix), + ) + else: + if self._enable_moe_dense_fully_dp(): + mlp_tp_rank, mlp_tp_size = 0, 1 + else: + mlp_tp_rank, mlp_tp_size = None, None + self.mlp = DeepseekV2MLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=add_prefix("mlp", prefix), + tp_rank=mlp_tp_rank, + tp_size=mlp_tp_size, + ) + + self.input_is_scattered = ( + previous_layer_info.ffn_input_mode == _FFNInputMode.SCATTERED + ) + self.is_last_layer = self.layer_id == config.num_hidden_layers - 1 + + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + @staticmethod + def _enable_moe_dense_fully_dp(): + return global_server_args_dict["moe_dense_tp_size"] == 1 + + @staticmethod + def _compute_info(config: PretrainedConfig, layer_id: int, is_nextn: bool): + is_sparse = is_nextn or ( + config.n_routed_experts is not None + and layer_id >= config.first_k_dense_replace + and layer_id % config.moe_layer_freq == 0 + ) + ffn_input_mode = ( + _FFNInputMode.SCATTERED + if (global_server_args_dict["enable_deepep_moe"] and is_sparse) + or (DeepseekV2DecoderLayer._enable_moe_dense_fully_dp() and not is_sparse) + else _FFNInputMode.FULL + ) + return _DecoderLayerInfo(is_sparse=is_sparse, ffn_input_mode=ffn_input_mode) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + residual: Optional[torch.Tensor], + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + if self.info.ffn_input_mode == _FFNInputMode.SCATTERED: + return self.forward_ffn_with_scattered_input( + positions, hidden_states, forward_batch, residual, zero_allocator + ) + elif self.info.ffn_input_mode == _FFNInputMode.FULL: + return self.forward_ffn_with_full_input( + positions, hidden_states, forward_batch, residual, zero_allocator + ) + else: + raise NotImplementedError + + def forward_ffn_with_full_input( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + residual: Optional[torch.Tensor], + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + + if hidden_states.shape[0] == 0: + residual = hidden_states + else: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + assert not ( + self.attn_tp_size != 1 and self.input_is_scattered + ), "moe_layer_freq > 1 is not supported when attn_tp_size > 1" + + # Self Attention + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + zero_allocator=zero_allocator, + ) + + # Gather + if get_tensor_model_parallel_world_size() > 1: + # all gather and all reduce + if self.dp_size != 1: + if self.attn_tp_rank == 0: + hidden_states += residual + hidden_states, local_hidden_states = ( + forward_batch.gathered_buffer, + hidden_states, + ) + dp_gather_partial(hidden_states, local_hidden_states, forward_batch) + dp_scatter(residual, hidden_states, forward_batch) + hidden_states = self.post_attention_layernorm(hidden_states) + else: + hidden_states = tensor_model_parallel_all_reduce(hidden_states) + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual + ) + else: + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual + ) + + # Fully Connected + hidden_states = self.mlp(hidden_states) + + # TODO(ch-wan): ues reduce-scatter in MLP to avoid this scatter + # Scatter + if self.dp_size != 1: + # important: forward batch.gathered_buffer is used both after scatter and after gather. + # be careful about this! + hidden_states, global_hidden_states = ( + forward_batch.gathered_buffer[: forward_batch.input_ids.shape[0]], + hidden_states, + ) + dp_scatter(hidden_states, global_hidden_states, forward_batch) + + return hidden_states, residual + + def forward_ffn_with_scattered_input( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + residual: Optional[torch.Tensor], + zero_allocator: BumpAllocator, + ) -> torch.Tensor: + + if hidden_states.shape[0] == 0: + residual = hidden_states + else: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + if self.attn_tp_size != 1 and self.input_is_scattered: + hidden_states, local_hidden_states = ( + forward_batch.gathered_buffer[: forward_batch.input_ids.shape[0]], + hidden_states, + ) + tp_all_gather( + list(hidden_states.tensor_split(self.attn_tp_size)), local_hidden_states + ) + + # Self Attention + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + forward_batch=forward_batch, + zero_allocator=zero_allocator, + ) + + if self.attn_tp_size != 1: + if self.input_is_scattered: + tensor_list = list(hidden_states.tensor_split(self.attn_tp_size)) + hidden_states = tensor_list[self.attn_tp_rank] + tp_reduce_scatter(hidden_states, tensor_list) + if hidden_states.shape[0] != 0: + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual + ) + else: + if self.attn_tp_rank == 0: + hidden_states += residual + tensor_list = list(hidden_states.tensor_split(self.attn_tp_size)) + hidden_states = tensor_list[self.attn_tp_rank] + tp_reduce_scatter(hidden_states, tensor_list) + residual = hidden_states + if hidden_states.shape[0] != 0: + hidden_states = self.post_attention_layernorm(hidden_states) + else: + if hidden_states.shape[0] != 0: + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual + ) + + if not ( + self._enable_moe_dense_fully_dp() + and (not self.info.is_sparse) + and hidden_states.shape[0] == 0 + ): + hidden_states = self.mlp(hidden_states, forward_batch.forward_mode) + + if self.is_last_layer and self.attn_tp_size != 1: + hidden_states += residual + residual = None + hidden_states, local_hidden_states = ( + forward_batch.gathered_buffer[: forward_batch.input_ids.shape[0]], + hidden_states, + ) + tp_all_gather( + list(hidden_states.tensor_split(self.attn_tp_size)), local_hidden_states + ) + + return hidden_states, residual + + +class DeepseekV2Model(nn.Module): + fall_back_to_pt_during_load = False + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.padding_id = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + enable_tp=not global_server_args_dict["enable_dp_attention"], + ) + self.layers = nn.ModuleList( + [ + DeepseekV2DecoderLayer( + config, + layer_id, + quant_config=quant_config, + prefix=add_prefix(f"layers.{layer_id}", prefix), + ) + for layer_id in range(config.num_hidden_layers) + ] + ) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + self.dp_size = get_attention_dp_size() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor = None, + ) -> torch.Tensor: + zero_allocator = BumpAllocator( + # TODO for two-batch-overlap, we need a larger buffer size + buffer_size=len(self.layers) * 2, + dtype=torch.float32, + device=( + input_embeds.device if input_embeds is not None else input_ids.device + ), + ) + + if input_embeds is None: + hidden_states = self.embed_tokens(input_ids) + else: + hidden_states = input_embeds + + residual = None + for i in range(len(self.layers)): + expert_distribution_recorder.set_current_layer(i) + layer = self.layers[i] + hidden_states, residual = layer( + positions, hidden_states, forward_batch, residual, zero_allocator + ) + if not forward_batch.forward_mode.is_idle(): + if residual is None: + hidden_states = self.norm(hidden_states) + else: + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class DeepseekV2ForCausalLM(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.tp_size = get_tensor_model_parallel_world_size() + self.quant_config = quant_config + self.n_share_experts_fusion = global_server_args_dict["n_share_experts_fusion"] + if self.n_share_experts_fusion > 0: + # Only Deepseek V3/R1 can use shared experts fusion optimization now. + if ( + self.config.architectures[0] != "DeepseekV3ForCausalLM" + or self.config.n_routed_experts != 256 + ): + self.n_share_experts_fusion = 0 + global_server_args_dict["n_share_experts_fusion"] = 0 + logger.info( + "Only Deepseek V3/R1 can use shared experts fusion optimization. Shared experts fusion optimization is disabled." + ) + else: + assert ( + self.n_share_experts_fusion == self.tp_size + ), f"Shared experts fusion optimization is enabled in DeepSeek V3/R1, set it to {self.tp_size} can get best optimized performace." + elif self.n_share_experts_fusion == 0: + if ( + torch.cuda.get_device_capability("cuda") >= (9, 0) + and self.config.architectures[0] == "DeepseekV3ForCausalLM" + and self.config.n_routed_experts == 256 + and (not global_server_args_dict["enable_deepep_moe"]) + ): + self.n_share_experts_fusion = self.tp_size + global_server_args_dict["n_share_experts_fusion"] = self.tp_size + logger.info( + "Deepseek V3/R1 with fp8 can use shared experts fusion optimization when SM version >=90. Shared experts fusion optimization is enabled." + ) + + self.model = DeepseekV2Model( + config, quant_config, prefix=add_prefix("model", prefix) + ) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=add_prefix("lm_head", prefix), + ) + self.logits_processor = LogitsProcessor(config) + self.dp_size = get_attention_dp_size() + + def get_input_embeddings(self) -> nn.Embedding: + return self.model.embed_tokens + + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + input_embeds: torch.Tensor = None, + ) -> torch.Tensor: + + hidden_states = self.model(input_ids, positions, forward_batch, input_embeds) + + return self.logits_processor( + input_ids, hidden_states, self.lm_head, forward_batch + ) + + def post_load_weights(self): + + # Perform post-processing after loading weights + for layer_id in range(self.config.num_hidden_layers): + self_attn = self.model.layers[layer_id].self_attn + if hasattr(self_attn.kv_b_proj, "qweight"): + # AWQ compatible + if _is_cuda: + w = awq_dequantize( + self_attn.kv_b_proj.qweight, + self_attn.kv_b_proj.scales, + self_attn.kv_b_proj.qzeros, + ).T + else: + w = awq_dequantize( + self_attn.kv_b_proj.qweight, + self_attn.kv_b_proj.scales, + self_attn.kv_b_proj.qzeros, + 0, + 0, + 0, + ).T + else: + w = self_attn.kv_b_proj.weight + # NOTE(HandH1998): Since `bmm_fp8` only supports per-tensor scale, we have to requantize `self_attn.kv_b_proj`. + # This may affect the accuracy of fp8 model. + # Fix deepseek v3 blockwise bmm by using deep_gemm + use_deep_gemm_bmm = False + model_dtype = torch.get_default_dtype() + + if w.dtype in ( + torch.float8_e4m3fn, + torch.float8_e4m3fnuz, + ): + if hasattr(self.quant_config, "weight_block_size"): + weight_block_size = self.quant_config.weight_block_size + if weight_block_size is not None: + assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") + if _is_hip: + weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=w, + weight_scale=self_attn.kv_b_proj.weight_scale_inv, + input_scale=None, + ) + else: + weight = w + weight_scale = self_attn.kv_b_proj.weight_scale_inv + + if ( + _is_cuda + and _enable_jit_deepgemm_bmm + and weight_block_size[0] == 128 + and weight_block_size[1] == 128 + and model_dtype == torch.bfloat16 + ): + block_scale = weight_scale + use_deep_gemm_bmm = True + else: + w, scale = block_quant_to_tensor_quant( + weight, weight_scale, weight_block_size + ) + self_attn.w_scale = scale + else: + weight = w + weight_scale = self_attn.kv_b_proj.weight_scale + w, scale = channel_quant_to_tensor_quant(weight, weight_scale) + self_attn.w_scale = scale + + if w.dtype == torch.int8: + if hasattr(self.quant_config, "weight_block_size"): + # block-wise int8 need it + weight_block_size = self.quant_config.weight_block_size + if weight_block_size is not None: + assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") + weight = w + weight_scale = self_attn.kv_b_proj.weight_scale_inv + w = int8_block_dequant( + weight, weight_scale, weight_block_size + ).to(torch.bfloat16) + else: + # channel-wise int8 need it + w = w.to(torch.bfloat16) * self_attn.kv_b_proj.weight_scale.to( + torch.bfloat16 + ) + + w_kc, w_vc = w.unflatten( + 0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim) + ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) + if not use_deep_gemm_bmm: + self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2) + self_attn.w_vc = w_vc.contiguous().transpose(1, 2) + if ( + hasattr(self_attn.kv_b_proj, "weight_scale") + and self_attn.w_scale is None + ): + self_attn.w_scale = self_attn.kv_b_proj.weight_scale + if _is_hip: + self_attn.w_scale *= 2.0 + else: + num_tiles_k = self_attn.qk_nope_head_dim // weight_block_size[1] + num_tiles_n = self_attn.v_head_dim // weight_block_size[0] + ws_kc, ws_vc = block_scale.unflatten( + 0, (-1, (num_tiles_k + num_tiles_n)) + ).split([num_tiles_k, num_tiles_n], dim=1) + self_attn.w_scale_k = ws_kc.transpose(1, 2).contiguous() + self_attn.w_scale_v = ws_vc.contiguous() + self_attn.w_kc = w_kc.transpose(1, 2).contiguous() + self_attn.w_vc = w_vc.contiguous() + self_attn.use_deep_gemm_bmm = True + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + if self.n_share_experts_fusion > 0: + weights_list = list(weights) + weights_dict = dict(weights_list) + if self.quant_config.get_name() == "w8a8_int8": + suffix_list = [ + "down_proj.weight", + "down_proj.weight_scale", + "gate_proj.weight", + "gate_proj.weight_scale", + "up_proj.weight", + "up_proj.weight_scale", + ] + else: + suffix_list = [ + "down_proj.weight", + "down_proj.weight_scale_inv", + "gate_proj.weight", + "gate_proj.weight_scale_inv", + "up_proj.weight", + "up_proj.weight_scale_inv", + ] + names_to_remove = [] + for moe_layer in tqdm( + range( + self.config.first_k_dense_replace, + self.config.num_hidden_layers, + self.config.moe_layer_freq, + ), + desc=f"Cloning {self.n_share_experts_fusion} " + "replicas of the shared expert into MoE", + ): + for num_repeat in range(self.n_share_experts_fusion): + for suffix in suffix_list: + shared_expert_weight_name = ( + f"model.layers.{moe_layer}.mlp.shared_experts.{suffix}" + ) + weights_list.append( + ( + f"model.layers.{moe_layer}." + f"mlp.experts." + f"{self.config.n_routed_experts + num_repeat}" + f".{suffix}", + weights_dict[shared_expert_weight_name], + ) + ) + names_to_remove += [shared_expert_weight_name] + weights = [w for w in weights_list if w[0] not in names_to_remove] + + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + MoEImpl = ( + DeepEPMoE + if global_server_args_dict["enable_deepep_moe"] + else (EPMoE if global_server_args_dict["enable_ep_moe"] else FusedMoE) + ) + expert_params_mapping = MoEImpl.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.n_routed_experts + self.n_share_experts_fusion, + ) + + params_dict = dict(self.named_parameters()) + for name, loaded_weight in weights: + # TODO(HandH1998): Modify it when nextn is supported. + if hasattr(self.config, "num_nextn_predict_layers"): + num_nextn_layers = self.config.num_nextn_predict_layers + if num_nextn_layers > 0 and name.startswith("model.layers"): + name_list = name.split(".") + if ( + len(name_list) >= 3 + and int(name_list[2]) >= self.config.num_hidden_layers + ): + continue + if "rotary_emb.inv_freq" in name: + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + # Skip non-stacked layers and experts (experts handled below). + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if ("mlp.experts." in name) and name not in params_dict: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader( + param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id, + ) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + param = params_dict[name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + + self.post_load_weights() + + def get_embed_and_head(self): + return self.model.embed_tokens.weight, self.lm_head.weight + + def set_embed_and_head(self, embed, head): + del self.model.embed_tokens.weight + del self.lm_head.weight + self.model.embed_tokens.weight = embed + self.lm_head.weight = head + torch.cuda.empty_cache() + torch.cuda.synchronize() + + +class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM): + pass + + +EntryClass = [DeepseekV2ForCausalLM, DeepseekV3ForCausalLM] diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py new file mode 100644 index 0000000000..a6536ca0db --- /dev/null +++ b/atom/models/qwen3.py @@ -0,0 +1,220 @@ +import torch +from torch import nn +import torch.distributed as dist +from transformers import Qwen3Config + +from atom.model_ops.activation import SiluAndMul +from atom.model_ops.attention import Attention +from atom.model_ops.layernorm import RMSNorm +from atom.model_ops.linear import ( + QKVParallelLinear, + MergedColumnParallelLinear, + RowParallelLinear, +) +from atom.model_ops.rotary_embedding import get_rope +from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead + + +class Qwen3Attention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + head_dim: int | None = None, + rms_norm_eps: float = 1e-06, + qkv_bias: bool = False, + rope_theta: float = 10000, + rope_scaling: tuple | None = None, + ) -> None: + super().__init__() + tp_size = dist.get_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + assert self.total_num_kv_heads % tp_size == 0 + self.num_kv_heads = self.total_num_kv_heads // tp_size + self.head_dim = head_dim or hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=qkv_bias, + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + self.num_kv_heads, + ) + self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q_by_head = q.view(-1, self.num_heads, self.head_dim) + q_by_head = self.q_norm(q_by_head) + q = q_by_head.view(q.shape) + k_by_head = k.view(-1, self.num_kv_heads, self.head_dim) + k_by_head = self.k_norm(k_by_head) + k = k_by_head.view(k.shape) + q, k = self.rotary_emb(positions, q, k) + o = self.attn(q, k, v) + output = self.o_proj(o) + return output + + +class Qwen3MLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + ) + assert hidden_act == "silu" + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x = self.down_proj(x) + return x + + +class Qwen3DecoderLayer(nn.Module): + + def __init__( + self, + config: Qwen3Config, + ) -> None: + super().__init__() + self.self_attn = Qwen3Attention( + hidden_size=config.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + max_position=config.max_position_embeddings, + rms_norm_eps=config.rms_norm_eps, + qkv_bias=getattr(config, 'attention_bias', False), + head_dim=getattr(config, 'head_dim', None), + rope_theta=getattr(config, "rope_theta", 1000000), + rope_scaling=getattr(config, "rope_scaling", None), + ) + self.mlp = Qwen3MLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + ) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn(positions, hidden_states) + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +class Qwen3Model(nn.Module): + + def __init__( + self, + config: Qwen3Config, + ) -> None: + super().__init__() + self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size) + self.layers = nn.ModuleList([Qwen3DecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.embed_tokens(input_ids) + residual = None + for layer in self.layers: + hidden_states, residual = layer(positions, hidden_states, residual) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class Qwen3ForCausalLM(nn.Module): + packed_modules_mapping = { + "q_proj": ("qkv_proj", "q"), + "k_proj": ("qkv_proj", "k"), + "v_proj": ("qkv_proj", "v"), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + def __init__( + self, + config: Qwen3Config + ) -> None: + super().__init__() + self.model = Qwen3Model(config) + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size) + if config.tie_word_embeddings: + self.lm_head.weight.data = self.model.embed_tokens.weight.data + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + ) -> torch.Tensor: + hidden_states = self.model(input_ids, positions) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + logits = self.lm_head(hidden_states) + return logits diff --git a/atom/sampling_params.py b/atom/sampling_params.py new file mode 100644 index 0000000000..67d60bb465 --- /dev/null +++ b/atom/sampling_params.py @@ -0,0 +1,8 @@ +from dataclasses import dataclass + + +@dataclass +class SamplingParams: + temperature: float = 1.0 + max_tokens: int = 64 + ignore_eos: bool = False diff --git a/atom/utils/context.py b/atom/utils/context.py new file mode 100644 index 0000000000..2281888f80 --- /dev/null +++ b/atom/utils/context.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass +import torch + + +@dataclass +class Context: + is_prefill: bool = False + cu_seqlens_q: torch.Tensor | None = None + cu_seqlens_k: torch.Tensor | None = None + max_seqlen_q: int = 0 + max_seqlen_k: int = 0 + slot_mapping: torch.Tensor | None = None + context_lens: torch.Tensor | None = None + block_tables: torch.Tensor | None = None + +_CONTEXT = Context() + +def get_context(): + return _CONTEXT + +def set_context(is_prefill, cu_seqlens_q=None, cu_seqlens_k=None, max_seqlen_q=0, max_seqlen_k=0, slot_mapping=None, context_lens=None, block_tables=None): + global _CONTEXT + _CONTEXT = Context(is_prefill, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, slot_mapping, context_lens, block_tables) + +def reset_context(): + global _CONTEXT + _CONTEXT = Context() diff --git a/bench.py b/bench.py new file mode 100644 index 0000000000..f67cec3566 --- /dev/null +++ b/bench.py @@ -0,0 +1,43 @@ +import os +import time +from random import randint, seed +from atom import LLMEngine, SamplingParams + +# from vllm import LLM, SamplingParams + + +def main(): + seed(0) + num_seqs = 256 + max_input_len = 1024 + max_ouput_len = 1024 + + path = os.path.expanduser("/home/ljin1/model_zoo/Qwen3-0.6B/") + llm = LLMEngine(path, enforce_eager=False, max_model_len=4096) + + prompt_token_ids = [ + [randint(0, 10000) for _ in range(randint(100, max_input_len))] + for _ in range(num_seqs) + ] + sampling_params = [ + SamplingParams( + temperature=0.6, ignore_eos=True, max_tokens=randint(100, max_ouput_len) + ) + for _ in range(num_seqs) + ] + # uncomment the following line for vllm + # prompt_token_ids = [dict(prompt_token_ids=p) for p in prompt_token_ids] + + llm.generate(["Benchmark: "], SamplingParams()) + t = time.time() + llm.generate(prompt_token_ids, sampling_params, use_tqdm=False) + t = time.time() - t + total_tokens = sum(sp.max_tokens for sp in sampling_params) + throughput = total_tokens / t + print( + f"Total: {total_tokens}tok, Time: {t:.2f}s, Throughput: {throughput:.2f}tok/s" + ) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..ac8cdfc342 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "atom" +version = "0.1.0" +authors = [{ name = "ValarLip" }] +license = "MIT" +readme = "README.md" +description = "a lightweight vLLM implementation built from scratch" +requires-python = ">=3.10,<3.13" +dependencies = [ +] + +[project.urls] +Homepage="https://github.com/GeeeekExplorer/nano-vllm" + +[tool.setuptools.packages.find] +where = ["."] +include = ["python*"] \ No newline at end of file From aaf83fafd94a55c80e649f75ae6883cf88ef4393 Mon Sep 17 00:00:00 2001 From: chenjun Date: Tue, 12 Aug 2025 06:42:16 +0000 Subject: [PATCH 002/119] update1 --- atom/model_ops/activation.py | 9 +- atom/model_ops/layernorm.py | 8 +- atom/model_ops/linear.py | 75 +++-- atom/model_ops/moe.py | 582 +++++++++++++++++++++++++++++++++++ atom/model_ops/topK.py | 240 +++++++++++++++ atom/models/deepseek_v2.py | 67 ---- 6 files changed, 892 insertions(+), 89 deletions(-) create mode 100644 atom/model_ops/moe.py create mode 100644 atom/model_ops/topK.py diff --git a/atom/model_ops/activation.py b/atom/model_ops/activation.py index 041ee20084..d46f5299f6 100644 --- a/atom/model_ops/activation.py +++ b/atom/model_ops/activation.py @@ -1,6 +1,7 @@ import torch from torch import nn import torch.nn.functional as F +from aiter import silu_and_mul class SiluAndMul(nn.Module): @@ -9,6 +10,12 @@ def __init__(self): super().__init__() @torch.compile - def forward(self, x: torch.Tensor) -> torch.Tensor: + def forward_native(self, x: torch.Tensor) -> torch.Tensor: x, y = x.chunk(2, -1) return F.silu(x) * y + + def forward(self, x: torch.Tensor) -> torch.Tensor: + out = torch.empty([*x.shape[:-1], x.shape[-1] // 2], device=x.device, dtype=x.dtype) + silu_and_mul(out, x) + return out + diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py index 32dcfa2cb8..9c6019d6d1 100644 --- a/atom/model_ops/layernorm.py +++ b/atom/model_ops/layernorm.py @@ -1,5 +1,6 @@ import torch from torch import nn +from aiter import rmsnorm2d_fwd, rmsnorm2d_fwd_with_add class RMSNorm(nn.Module): @@ -46,6 +47,9 @@ def forward( residual: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: if residual is None: - return self.rms_forward(x) + return rmsnorm2d_fwd(x, self.weight, self.eps) else: - return self.add_rms_forward(x, residual) + out = torch.empty_like(out) + residual_out = torch.empty_like(residual) + rmsnorm2d_fwd_with_add(out, x, residual, residual_out, self.weight, self.eps) + return out, residual_out diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index c625a9ebd0..4e33e0bad0 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -2,6 +2,11 @@ from torch import nn import torch.nn.functional as F import torch.distributed as dist +from aiter import QuantType, dtypes, gemm_a8w8_bpreshuffle, gemm_a8w8_blockscale, gemm_a4w4 +from aiter.ops.shuffle import shuffle_weight +from aiter.tuned_gemm import tgemm +from typing import Optional, Callable +from functools import partial as functools_partial def divide(numerator, denominator): @@ -15,17 +20,55 @@ def __init__( self, input_size: int, output_size: int, - tp_dim: int | None = None, + bias: bool = False, + params_dtype = dtypes.bf16, + quant_type = Optional.No, ): super().__init__() self.input_size = input_size self.output_size = output_size - self.tp_dim = tp_dim - self.tp_rank = dist.get_rank() - self.tp_size = dist.get_world_size() + self.weight = nn.Parameter(torch.empty((self.output_size, self.input_size), dtype=params_dtype)) + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size, dtype=params_dtype)) + self.bias.weight_loader_process = self.weight_loader_process + else: + self.register_parameter("bias", None) + self.quant_type = quant_type + if quant_type != Optional.No: + if quant_type == QuantType.per_Tensor: + self.weight.weight_loader_process = functools_partial(self.weight_loader_process, post_process_func=shuffle_weight(16, 16)) + self.weight_scale = nn.Parameter(torch.empty(1, dtype=dtypes.fp32)) + elif quant_type == QuantType.per_Token: + self.weight.weight_loader_process = self.weight_loader_process + self.weight_scale = nn.Parameter(torch.empty(self.output_size, dtype=dtypes.fp32)) + elif quant_type == QuantType.per_128x128: + self.weight.weight_loader_process = self.weight_loader_process + self.weight_scale = nn.Parameter(torch.empty(divide(self.output_size, 128), (self.input_size + 127) // 128, dtype=dtypes.fp32)) + elif quant_type == QuantType.per_1x32: + self.weight.weight_loader_process = functools_partial(self.weight_loader_process, post_process_func=shuffle_weight(16, 16)) + self.weight_scale = nn.Parameter(torch.empty(self.output_size, (self.input_size + 31) // 32, dtype=dtypes.fp8_e8m0)) + self.weight_scale.weight_loader_process = self.weight_loader_process + else: + self.weight.weight_loader_process = self.weight_loader_process + self.register_parameter("weight_scale", None) - def forward(self, x: torch.Tensor) -> torch.Tensor: - raise NotImplementedError + @staticmethod + def weight_loader_process(param: nn.Parameter, loaded_weight: torch.Tensor, post_process_func: Callable=lambda a : a): + param.data.copy_(post_process_func(loaded_weight)) + + def forward(self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16) -> torch.Tensor: + if self.quant_type == Optional.No: + return tgemm.mm(x, self.weight, self.bias) + else: + assert x_scale is not None + if self.quant_type == QuantType.per_Tensor: + return tgemm.mm(x, self.weight, self.bias, otype=otype, scale_a=x_scale, scale_b=self.weight_scale) + elif self.quant_type == QuantType.per_Token: + return gemm_a8w8_bpreshuffle(x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype) + elif self.quant_type == QuantType.per_128x128: + return gemm_a8w8_blockscale(x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype) + elif self.quant_type == QuantType.per_1x32: + return gemm_a4w4(x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype) class ReplicatedLinear(LinearBase): @@ -59,28 +102,22 @@ def __init__( input_size: int, output_size: int, bias: bool = False, + params_dtype = dtypes.bf16, + quant_type = Optional.No, ): - super().__init__(input_size, output_size, 0) + self.tp_dim = 0 + self.tp_rank = dist.get_rank() + self.tp_size = dist.get_world_size() self.input_size_per_partition = input_size self.output_size_per_partition = divide(output_size, self.tp_size) - - self.weight = nn.Parameter(torch.empty(self.output_size_per_partition, self.input_size)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size_per_partition)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) + super().__init__(input_size, self.output_size_per_partition, bias, params_dtype, quant_type) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param_data.copy_(loaded_weight) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return F.linear(x, self.weight, self.bias) + param_data.weight_loader_process(param_data, loaded_weight) class MergedColumnParallelLinear(ColumnParallelLinear): diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py new file mode 100644 index 0000000000..7121420dcd --- /dev/null +++ b/atom/model_ops/moe.py @@ -0,0 +1,582 @@ +from functools import lru_cache +import torch +from torch import nn +from typing import Callable, List, Optional, Tuple +from aiter.dist.parallel_state import get_tensor_model_parallel_world_size, get_tensor_model_parallel_rank +from aiter.dist.communication_op import tensor_model_parallel_all_reduce +from transformers import PretrainedConfig +from aiter import QuantType, dtypes + + +def determine_expert_map( + ep_size: int, ep_rank: int, + global_num_experts: int) -> Tuple[int, Optional[torch.Tensor]]: + """ + Calculates how many experts should be assigned to each rank for EP and + creates a mapping from global to local expert index. Experts are + distributed evenly across ranks. Any remaining are assigned to the + last rank. + + Args: + ep_size (int): The size of the expert parallel group + global_num_experts (int): The total number of experts in the model. + + Returns: + Tuple[int, Optional[torch.Tensor]]: A tuple containing: + - local_num_experts (int): The number of experts assigned + to the current rank. + - expert_map (Optional[torch.Tensor]): A tensor of shape + (global_num_experts,) mapping from global to local index. + Contains -1 for experts not assigned to the current rank. + Returns None if ep_size is 1. + """ + assert ep_size > 0 + if ep_size == 1: + return (global_num_experts, None) + + local_num_experts = global_num_experts // ep_size + + # Create a tensor of size num_experts filled with -1 + expert_map = torch.full((global_num_experts, ), -1, dtype=torch.int32) + # Create a expert map for the local experts + if ep_rank < (ep_size - 1): + # Each non-last rank gets local_num_experts experts. + expert_map[ep_rank * local_num_experts: + (ep_rank + 1) * local_num_experts] = \ + torch.arange(0, local_num_experts, dtype=torch.int32) + else: + # All remaining experts are assigned to the last rank. + local_num_experts = (global_num_experts - ep_rank * local_num_experts) + + expert_map[-local_num_experts:] = \ + torch.arange(0, local_num_experts, dtype=torch.int32) + return (local_num_experts, expert_map) + + +class FusedMoE(torch.nn.Module): + """FusedMoE layer for MoE models. + + This layer contains both MergedColumnParallel weights (gate_up_proj / + w13) and RowParallelLinear weights (down_proj/ w2). + + Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We + copy that naming convention here and handle any remapping in the + load_weights function in each model implementation. + + Args: + num_experts: Number of experts in the model + top_k: Number of experts selected for each token + hidden_size: Input hidden state size of the transformer + intermediate_size: Intermediate size of the experts + params_dtype: Data type for the parameters. + reduce_results: Whether to all all_reduce on the output of the layer + renomalize: Whether to renormalize the logits in the fused_moe kernel + quant_config: Quantization configure. + """ + + def __init__( + self, + num_experts: int, # Global number of experts + top_k: int, + hidden_size: int, + intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, + reduce_results: bool = False, + renormalize: bool = True, + use_grouped_topk: bool = False, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + quant_type = QuantType.No, + tp_size: Optional[int] = None, + ep_size: Optional[int] = None, + dp_size: Optional[int] = None, + prefix: str = "", + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, + activation: str = "silu", + config: Optional[PretrainedConfig] = None, + ): + super().__init__() + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + + # Note: here we guard against accessing the TP and DP groups when + # uninitialized (this happens when testing) + self.tp_size = (tp_size if tp_size is not None else + get_tensor_model_parallel_world_size()) + tp_rank = 0 if self.tp_size == 1 else get_tensor_model_parallel_rank() + self.global_num_experts = num_experts + + # Use expert parallelism instead of tensor parallelism? + vllm_config = get_current_vllm_config() + use_ep = (vllm_config.parallel_config.enable_expert_parallel + and self.tp_size > 1) + + if use_ep: + # Set TP size to 1 to adjust for EP and adjust EP size and rank + # for DP attention. + self.ep_rank = tp_rank + self.tp_size * self.dp_rank + self.tp_rank = 0 + self.ep_size = self.tp_size * self.dp_size + self.tp_size = 1 + + self.local_num_experts, self.expert_map = determine_expert_map( + ep_size=self.ep_size, + ep_rank=self.ep_rank, + global_num_experts=self.global_num_experts) + else: + # Adjust TP size for DP attention + self.tp_rank = tp_rank + self.tp_size * self.dp_rank + self.ep_rank = 0 + self.tp_size = self.tp_size * self.dp_size + self.ep_size = 1 + self.local_num_experts = self.global_num_experts + self.expert_map = None + self.top_k = top_k + self.global_num_experts = num_experts + + self.num_fused_shared_experts = config.n_shared_experts if config is not None and is_rocm_aiter_fusion_shared_expert_enabled() else 0 + self.routed_scaling_factor = config.routed_scaling_factor if config is not None and config.torch_dtype != torch.float16 else 1.0 + self.expert_mask = None + if use_ep and is_rocm_aiter_moe_enabled(): + expert_mask = torch.ones((self.global_num_experts + self.num_fused_shared_experts + 1,), dtype=torch.int32) + expert_mask[-1] = 0 + expert_mask[:self.global_num_experts] = self.expert_map > -1 + self.expert_mask = expert_mask + self.expert_map = torch.cat((self.expert_map, torch.tensor([self.local_num_experts + i for i in range(self.num_fused_shared_experts)],dtype=torch.int32)), dim=0) + if is_rocm_aiter_fusion_shared_expert_enabled() and self.num_fused_shared_experts > 0: + init_aiter_topK_meta_data( + n_routed_experts=self.global_num_experts, + n_shared_experts= self.num_fused_shared_experts, + top_k=self.top_k, + tp_rank=self.ep_rank if use_ep else tp_rank, + tp_size=self.ep_size if use_ep else tp_size, + shared_experts_score=1.0, + max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens, + is_EP=use_ep, + ) + if is_rocm_aiter_fusion_shared_expert_enabled(): + self.local_num_experts += self.num_fused_shared_experts + + assert intermediate_size % self.tp_size == 0 + self.hidden_size = hidden_size + self.intermediate_size_per_partition = intermediate_size // self.tp_size + self.reduce_results = reduce_results + self.renormalize = renormalize + self.use_grouped_topk = use_grouped_topk + if self.use_grouped_topk: + assert num_expert_group is not None and topk_group is not None + self.num_expert_group = num_expert_group + self.topk_group = topk_group + self.custom_routing_function = custom_routing_function + self.scoring_func = scoring_func + self.e_score_correction_bias = e_score_correction_bias + self.activation = activation + + if self.scoring_func != "softmax" and not self.use_grouped_topk: + raise ValueError("Only softmax scoring function is supported for " + "non-grouped topk.") + + self.apply_router_weight_on_input = apply_router_weight_on_input + moe_quant_params = { + "num_experts": self.local_num_experts, + "hidden_size": hidden_size, + "intermediate_size_per_partition": + self.intermediate_size_per_partition, + "params_dtype": params_dtype, + "weight_loader": self.weight_loader, + } + self.quant_method.create_weights(layer=self, **moe_quant_params) + + def _load_per_tensor_weight_scale(self, shard_id: str, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + expert_id: int): + param_data = param.data + # for per tensor weight quantization + if shard_id in ("w1", "w3"): + # We have to keep the weight scales of w1 and w3 because + # we need to re-quantize w1/w3 weights after weight loading. + idx = 0 if shard_id == "w1" else 1 + param_data[expert_id][idx] = loaded_weight + # If we are in the row parallel case (down_proj) + elif shard_id == "w2": + param_data[expert_id] = loaded_weight + + def _load_model_weight_or_group_weight_scale(self, + shard_dim: int, + expert_data: torch.Tensor, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + load_full_w2: bool = False): + """ + Load grouped weight scales for group quantization or model weights + :param shard_dim: dimension to shard + :param expert_data: parameter for a particular expert + :param shard_id: either w1, w2, or w3 + :param loaded_weight: checkpoint weight to load into the param + :param tp_rank: tensor parallel rank + :param load_full_w2: whether or not the w2 loaded should be sharded. + """ + if shard_id == "w2": + # In the case where we have actorder/g_idx, we do not partition the + # w2 scales, as indicated by `load_full` argument, for all tp cases + self._load_w2(shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + load_full=load_full_w2) + elif shard_id in ("w1", "w3"): + self._load_w13(shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + + def _load_per_channel_weight_scale(self, expert_data: torch.Tensor, + shard_dim: int, shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int): + # for per channel weight quantization + if shard_id == "w2": + expert_data.copy_(loaded_weight) + elif shard_id in ("w1", "w3"): + self._load_w13(shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + + def _load_w13(self, expert_data: torch.Tensor, shard_dim: int, + shard_id: str, loaded_weight: torch.Tensor, tp_rank: int): + + # Index the loaded weight for tp sharding. + # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim + shard_size = expert_data.shape[shard_dim] // 2 + loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank, + shard_size) + # Narrow parameter and load. + # w1, gate_proj: Load into first logical weight of w13. + if shard_id == "w1": + expert_data = expert_data.narrow(shard_dim, 0, shard_size) + # w3, up_proj: Load into second logical weight of w13. + else: + assert shard_id == "w3" + expert_data = expert_data.narrow(shard_dim, shard_size, shard_size) + expert_data.copy_(loaded_weight) + + def _load_w2(self, + expert_data: torch.Tensor, + shard_dim: int, + loaded_weight: torch.Tensor, + tp_rank: int, + load_full: bool = False): + + # Index the loaded weight for tp sharding. + # down_proj: "RowParallel" so tp sharding on input_dim + # Narrow parameter and load. + shard_size = expert_data.shape[shard_dim] + if not load_full: + loaded_weight = loaded_weight.narrow(shard_dim, + shard_size * tp_rank, + shard_size) + # w2, down_proj: Load into only logical weight of w2. + expert_data.copy_(loaded_weight) + + def _load_single_value(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor, expert_id: int): + param_data = param.data + + # Input scales can be loaded directly and should be equal. + param_data[expert_id] = loaded_weight + + def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor, + shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int): + + if shard_id == "w2": + self._load_w2(shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank) + else: + assert shard_id in ("w1", "w3") + expert_data.copy_(loaded_weight) + + def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int: + if self.expert_map is None: + return expert_id + return self.expert_map[expert_id].item() + + def weight_loader(self, param: torch.nn.Parameter, + loaded_weight: torch.Tensor, weight_name: str, + shard_id: str, expert_id: int) -> None: + + expert_id = self._map_global_expert_id_to_local_expert_id(expert_id) + if expert_id == -1: + return + + # compressed-tensors checkpoints with packed weights are stored flipped + # TODO (mgoin): check self.quant_method.quant_config.quant_format + # against known CompressionFormat enum values that have this quality + if self.quant_method.__class__.__name__ in ( + "CompressedTensorsWNA16MarlinMoEMethod", + "CompressedTensorsWNA16MoEMethod"): + loaded_weight = loaded_weight.t().contiguous() + + if shard_id not in ("w1", "w2", "w3"): + raise ValueError(f"shard_id must be ['w1','w2','w3'] but " + f"got {shard_id}.") + + WEIGHT_SCALE_SUPPORTED = [ + e.value for e in FusedMoeWeightScaleSupported + ] + # Fetch the dim to shard the parameter/loaded weight + # based on the shard id. This will be whatever + # dimension intermediate_size_per_partition is used. + SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0} + + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.weight_type = loaded_weight.item() + param.data.copy_(loaded_weight) + return + + # is_transposed: if the dim to shard the weight + # should be flipped. Required by GPTQ, compressed-tensors + # should be whatever dimension intermediate_size_per_partition is + is_transposed = getattr(param, "is_transposed", False) + shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id] + if is_transposed: + shard_dim = int(not shard_dim) + + full_load = len(loaded_weight.shape) == 3 + if full_load: + shard_dim += 1 + + # Materialize GGUF UninitializedParameter + if is_gguf_weight and isinstance(param, UninitializedParameter): + final_shape = list(loaded_weight.shape) + if shard_id in ["w1", "w3"]: + final_shape[1] *= 2 + final_shape[shard_dim] = final_shape[shard_dim] // self.tp_size + param.materialize(final_shape, dtype=loaded_weight.dtype) + + expert_data = param.data if full_load else param.data[expert_id] + # Case input scale: input_scale loading is only supported for fp8 + if "input_scale" in weight_name: + # this is needed for compressed-tensors only + loaded_weight = loaded_weight.to(param.data.device) + + if param.data[expert_id] != 1 and (param.data[expert_id] - + loaded_weight).abs() > 1e-5: + raise ValueError( + "input_scales of w1 and w3 of a layer " + f"must be equal. But got {param.data[expert_id]} " + f"vs. {loaded_weight}") + + self._load_single_value(param=param, + loaded_weight=loaded_weight, + expert_id=expert_id) + return + + # Case g_idx + if "g_idx" in weight_name: + self._load_g_idx(shard_dim=0, + shard_id=shard_id, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=self.tp_rank) + return + + # Case weight scales, zero_points and offset + if ("scale" in weight_name or "zero" in weight_name + or "offset" in weight_name): + # load the weight scales and zp based on the quantization scheme + # supported weight scales/zp can be found in + # FusedMoeWeightScaleSupported + # TODO @dsikka: once hardened, refactor to use vLLM Parameters + # specific to each case + quant_method = getattr(param, "quant_method", None) + if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value: + self._load_per_channel_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=self.tp_rank) + elif quant_method in [ + FusedMoeWeightScaleSupported.GROUP.value, + FusedMoeWeightScaleSupported.BLOCK.value, + ]: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=self.tp_rank, + load_full_w2=getattr(param, "load_full_w2", False)) + elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: + self._load_per_tensor_weight_scale(shard_id=shard_id, + param=param, + loaded_weight=loaded_weight, + expert_id=expert_id) + else: + raise ValueError( + f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}") + return + + # Case weight_shape + if "weight_shape" in weight_name: + # only required by compressed-tensors + self._load_single_value(param=param, + loaded_weight=loaded_weight, + expert_id=expert_id) + return + + # Case model weights + if "weight" in weight_name: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=self.tp_rank) + return + + @staticmethod + def select_experts(hidden_states: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + use_grouped_topk: bool, + renormalize: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: float = 1.0): + from aiter.fused_moe import fused_topk + + # DeekSeekv2 uses grouped_top_k + if use_grouped_topk: + assert topk_group is not None + assert num_expert_group is not None + topk_weights, topk_ids = grouped_topk( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + num_expert_group=num_expert_group, + topk_group=topk_group, + scoring_func=scoring_func, + e_score_correction_bias=e_score_correction_bias, + if is_rocm_aiter_moe_enabled() else {}), + **({"num_fused_shared_experts":num_fused_shared_experts} if is_rocm_aiter_fusion_shared_expert_enabled() else {})) + elif custom_routing_function is None: + topk_weights, topk_ids = fused_topk(hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize) + + return topk_weights, topk_ids + + def naive_multicast(self, x: torch.Tensor, + cu_tokens_across_dp_cpu: torch.Tensor): + assert (len(x.shape) == 2) + buffer = torch.empty((cu_tokens_across_dp_cpu[-1], x.size(1)), + device=x.device, + dtype=x.dtype) + + start = 0 if self.dp_rank == 0 else cu_tokens_across_dp_cpu[ + self.dp_rank - 1] + end = cu_tokens_across_dp_cpu[self.dp_rank] + buffer[start:end, :].copy_(x) + for idx in range(get_dp_group().world_size): + start = 0 if idx == 0 else cu_tokens_across_dp_cpu[idx - 1] + end = cu_tokens_across_dp_cpu[idx] + get_dp_group().broadcast(buffer[start:end, :], idx) + + return buffer + + def forward(self, hidden_states: torch.Tensor, + router_logits: torch.Tensor): + if self.use_direct_call: + return self.forward_impl(hidden_states, router_logits) + else: + return torch.ops.vllm.moe_forward(hidden_states, router_logits, + self.layer_name) + + def forward_impl(self, hidden_states: torch.Tensor, + router_logits: torch.Tensor): + assert self.quant_method is not None + + # Matrix multiply. + final_hidden_states = self.quant_method.apply( + layer=self, + x=hidden_states, + router_logits=router_logits, + top_k=self.top_k, + renormalize=self.renormalize, + use_grouped_topk=self.use_grouped_topk, + global_num_experts=self.global_num_experts, + expert_map=self.expert_map if not is_rocm_aiter_moe_enabled() else self.expert_mask, + topk_group=self.topk_group, + num_expert_group=self.num_expert_group, + custom_routing_function=self.custom_routing_function, + scoring_func=self.scoring_func, + e_score_correction_bias=self.e_score_correction_bias, + activation=self.activation, + apply_router_weight_on_input=self.apply_router_weight_on_input, + ) + + if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): + # Default set to False. (May have to add shared expert outputs.) + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) + + return final_hidden_states + + @classmethod + def make_expert_params_mapping( + cls, ckpt_gate_proj_name: str, ckpt_down_proj_name: str, + ckpt_up_proj_name: str, + num_experts: int) -> List[Tuple[str, str, int, str]]: + + return [ + # (param_name, weight_name, expert_id, shard_id) + ("experts.w13_" if weight_name + in [ckpt_gate_proj_name, ckpt_up_proj_name] else "experts.w2_", + f"experts.{expert_id}.{weight_name}.", expert_id, shard_id) + for expert_id in range(num_experts) for shard_id, weight_name in [ + ("w1", ckpt_gate_proj_name), + ("w2", ckpt_down_proj_name), + ("w3", ckpt_up_proj_name), + ] + ] + + def extra_repr(self) -> str: + + s = ( + f"global_num_experts={self.global_num_experts}, " + f"local_num_experts={self.local_num_experts}, " + f"top_k={self.top_k}, " + f"intermediate_size_per_partition={self.intermediate_size_per_partition}, " # noqa: E501 + f"tp_size={self.tp_size},\n" + f"ep_size={self.ep_size}, " + f"reduce_results={self.reduce_results}, " + f"renormalize={self.renormalize}, " + f"use_grouped_topk={self.use_grouped_topk}") + + if self.use_grouped_topk: + s += f", num_expert_group={self.num_expert_group}, topk_group={self.topk_group}" # noqa: E501 + + s += f", scoring_func='{self.scoring_func}', activation='{self.activation}'" # noqa: E501 + + return s diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py new file mode 100644 index 0000000000..c3492a9fba --- /dev/null +++ b/atom/model_ops/topK.py @@ -0,0 +1,240 @@ +import torch +from functools import cache, lru_cache +from typing import Optional + + +aiter_topK_meta_data = None + +@lru_cache(maxsize=1) +def init_aiter_topK_meta_data( + n_routed_experts: int, + n_shared_experts: int, + top_k: int, + tp_rank: int, + tp_size: int, + shared_experts_score: float = 1.0, + max_num_tokens: int = 32768, + is_EP: bool = False +): + global aiter_topK_meta_data + fake_expertid = n_routed_experts + n_shared_experts + + # all layers reuse same buffer + total_topk_ids = torch.empty((max_num_tokens, top_k + n_shared_experts + is_EP), dtype=torch.int32, device='cuda') + ns_topk_ids, s_topk_ids = total_topk_ids.split([top_k, n_shared_experts + is_EP], dim=1) + shared_expert_ids = [n_routed_experts + i for i in range(n_shared_experts + is_EP)] + if is_EP: + s_topk_ids_list = [[fake_expertid]* (n_shared_experts + is_EP)] * max_num_tokens + for i in range(tp_rank, max_num_tokens, tp_size): + s_topk_ids_list[i] = shared_expert_ids + else: + s_topk_ids_list = [range(n_routed_experts, fake_expertid)] * max_num_tokens + s_topk_ids[:] = torch.tensor(s_topk_ids_list, dtype=torch.int32, device='cuda') + + total_topk_weights = torch.empty((max_num_tokens, top_k + n_shared_experts + is_EP), dtype=torch.float32, device='cuda') + ns_topk_weights, s_topk_weights = total_topk_weights.split([top_k, n_shared_experts + is_EP], dim=1) + s_topk_weights.fill_(shared_experts_score) + aiter_topK_meta_data = (total_topk_weights, total_topk_ids) + + +def rocm_aiter_topk_softmax_impl(topk_weights: torch.Tensor, + topk_indices: torch.Tensor, + token_expert_indices: torch.Tensor, + gating_output: torch.Tensor, + renormalize: bool) -> None: + from aiter import topk_softmax + topk_softmax(topk_weights, topk_indices, token_expert_indices, + gating_output, renormalize) + + +def rocm_aiter_topk_softmax_fake(topk_weights: torch.Tensor, + topk_indices: torch.Tensor, + token_expert_indices: torch.Tensor, + gating_output: torch.Tensor, + renormalize: bool) -> None: + pass + + +def rocm_aiter_biased_grouped_topk_impl( + gating_output: torch.Tensor, + correction_bias: torch.Tensor, + num_expert_group: int, + topk_group: int, + need_renorm: bool, + topk: int, + routed_scaling_factor: float = 1.0, # mul to topk_weights + num_fused_shared_experts: int = 0, +) -> tuple[torch.Tensor, torch.Tensor]: + + from aiter import biased_grouped_topk + token = gating_output.shape[0] + device = gating_output.device + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + assert aiter_topK_meta_data is not None, ( + "AITER topK meta data is not initialized. " + "Please ensure that init_aiter_topK_meta_data is called before this function." + ) + total_topk_weights, total_topk_ids = aiter_topK_meta_data + assert total_topk_weights.shape[0] >= token, ( + f"AITER topK meta data support {total_topk_weights.shape[0]} tokens which " + f"is determined by max_num_batched_tokens, but got {token} tokens now." + ) + total_topk_weights = total_topk_weights[:token] + total_topk_ids = total_topk_ids[:token] + topk_weights, _ = total_topk_weights.split( + [topk, total_topk_weights.shape[1] - topk], dim=1 + ) + topk_ids, _ = total_topk_ids.split( + [topk, total_topk_ids.shape[1] - topk], dim=1 + ) + else: + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + biased_grouped_topk(gating_output, correction_bias, topk_weights, topk_ids, + num_expert_group, topk_group, need_renorm, + routed_scaling_factor) + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + return total_topk_weights, total_topk_ids + return topk_weights, topk_ids + + +def rocm_aiter_biased_grouped_topk_fake( + gating_output: torch.Tensor, + correction_bias: torch.Tensor, + num_expert_group: int, + topk_group: int, + need_renorm: bool, + topk: int, + routed_scaling_factor: float = 1.0, # mul to topk_weights + num_fused_shared_experts: int = 0, +) -> tuple[torch.Tensor, torch.Tensor]: + token = gating_output.shape[0] + device = gating_output.device + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + assert aiter_topK_meta_data is not None, ( + "AITER topK meta data is not initialized. " + "Please ensure that init_aiter_topK_meta_data is called before this function." + ) + total_topk_weights, total_topk_ids = aiter_topK_meta_data + total_topk_ids = torch.empty( + total_topk_ids.shape, + dtype=torch.int32, + device=device, + ) + total_topk_weights = torch.empty( + total_topk_weights.shape, + dtype=torch.float32, + device=device, + ) + assert total_topk_weights.shape[0] >= token, ( + f"AITER topK meta data support {total_topk_weights.shape[0]} tokens which " + f"is determined by max_num_batched_tokens, but got {token} tokens now." + ) + total_topk_weights = total_topk_weights[:token] + total_topk_ids = total_topk_ids[:token] + topk_weights, _ = total_topk_weights.split( + [topk, total_topk_weights.shape[1] - topk], dim=1 + ) + topk_ids, _ = total_topk_ids.split( + [topk, total_topk_ids.shape[1] - topk], dim=1 + ) + else: + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + return total_topk_weights, total_topk_ids + return topk_weights, topk_ids + + +def rocm_aiter_grouped_topk_impl( + gating_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + num_expert_group: int, + topk_group: int, + need_renorm: bool, + topk: int, + scoring_func: str = "softmax", + routed_scaling_factor: float = 1.0, # mul to topk_weights + num_fused_shared_experts: int = 0, +) -> tuple[torch.Tensor, torch.Tensor]: + + from aiter import grouped_topk + token = gating_output.shape[0] + device = gating_output.device + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + assert aiter_topK_meta_data is not None, ( + "AITER topK meta data is not initialized. " + "Please ensure that init_aiter_topK_meta_data is called before this function." + ) + total_topk_weights, total_topk_ids = aiter_topK_meta_data + assert total_topk_weights.shape[0] >= token, ( + f"AITER topK meta data support {total_topk_weights.shape[0]} tokens which " + f"is determined by max_num_batched_tokens, but got {token} tokens now." + ) + total_topk_weights = total_topk_weights[:token] + total_topk_ids = total_topk_ids[:token] + topk_weights, _ = total_topk_weights.split( + [topk, total_topk_weights.shape[1] - topk], dim=1 + ) + topk_ids, _ = total_topk_ids.split( + [topk, total_topk_ids.shape[1] - topk], dim=1 + ) + else: + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + grouped_topk(gating_output, topk_weights, topk_ids, num_expert_group, + topk_group, need_renorm, scoring_func, routed_scaling_factor) + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + return total_topk_weights, total_topk_ids + return topk_weights, topk_ids + + +def rocm_aiter_grouped_topk_fake( + gating_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + num_expert_group: int, + topk_group: int, + need_renorm: bool, + topk: int, + scoring_func: str = "softmax", + routed_scaling_factor: float = 1.0, # mul to topk_weights + num_fused_shared_experts: int = 0, +) -> tuple[torch.Tensor, torch.Tensor]: + token = gating_output.shape[0] + device = gating_output.device + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + assert aiter_topK_meta_data is not None, ( + "AITER topK meta data is not initialized. " + "Please ensure that init_aiter_topK_meta_data is called before this function." + ) + total_topk_weights, total_topk_ids = aiter_topK_meta_data + total_topk_ids = torch.empty( + total_topk_ids.shape, + dtype=torch.int32, + device=device, + ) + total_topk_weights = torch.empty( + total_topk_weights.shape, + dtype=torch.float32, + device=device, + ) + assert total_topk_weights.shape[0] >= token, ( + f"AITER topK meta data support {total_topk_weights.shape[0]} tokens which " + f"is determined by max_num_batched_tokens, but got {token} tokens now." + ) + total_topk_weights = total_topk_weights[:token] + total_topk_ids = total_topk_ids[:token] + topk_weights, _ = total_topk_weights.split( + [topk, total_topk_weights.shape[1] - topk], dim=1 + ) + topk_ids, _ = total_topk_ids.split( + [topk, total_topk_ids.shape[1] - topk], dim=1 + ) + else: + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + return total_topk_weights, total_topk_ids + return topk_weights, topk_ids diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index dff1ff7648..71225dff2c 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -308,73 +308,6 @@ def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor: final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) return final_hidden_states - def forward_deepep( - self, hidden_states: torch.Tensor, forward_mode: ForwardMode - ) -> torch.Tensor: - shared_output = None - topk_idx = torch.full( - (0, self.top_k), -1, dtype=torch.int, device=hidden_states.device - ) - topk_weights = torch.empty( - (0, self.top_k), dtype=torch.float32, device=hidden_states.device - ) - if ( - forward_mode is not None - and not forward_mode.is_idle() - and hidden_states.shape[0] > 0 - ): - # router_logits: (num_tokens, n_experts) - router_logits = self.gate(hidden_states) - shared_output = self._forward_shared_experts(hidden_states) - topk_weights, topk_idx = select_experts( - hidden_states=hidden_states, - router_logits=router_logits, - top_k=self.top_k, - use_grouped_topk=True, - renormalize=self.renormalize, - topk_group=self.topk_group, - num_expert_group=self.num_expert_group, - correction_bias=self.correction_bias, - routed_scaling_factor=self.routed_scaling_factor, - ) - if self.ep_size > 1: - # TODO(ch-wan): allow users to set num_max_dispatch_tokens_per_rank value - ( - hidden_states, - topk_idx, - topk_weights, - reorder_topk_ids, - seg_indptr, - masked_m, - expected_m, - ) = self.deepep_dispatcher.dispatch( - hidden_states, - topk_idx, - topk_weights, - forward_mode=forward_mode, - ) - final_hidden_states = self.experts( - hidden_states=hidden_states, - reorder_topk_ids=reorder_topk_ids, - seg_indptr=seg_indptr, - masked_m=masked_m, - expected_m=expected_m, - forward_mode=forward_mode, - ) - if self.ep_size > 1: - final_hidden_states = self.deepep_dispatcher.combine( - final_hidden_states, - topk_idx, - topk_weights, - forward_mode, - ) - final_hidden_states *= self.routed_scaling_factor - - if shared_output is not None: - final_hidden_states = final_hidden_states + shared_output - - return final_hidden_states - def _forward_shared_experts(self, hidden_states): if self.n_share_experts_fusion == 0: return self.shared_experts(hidden_states) From 722a2b1e72a26a902a951a3518e0610f90457245 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 13 Aug 2025 11:26:52 +0800 Subject: [PATCH 003/119] update for running qwen3 --- atom/model_ops/layernorm.py | 10 ++++++---- atom/model_ops/linear.py | 27 ++++++++++++++++++++------- bench.py | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py index 9c6019d6d1..a2ce730c32 100644 --- a/atom/model_ops/layernorm.py +++ b/atom/model_ops/layernorm.py @@ -1,6 +1,6 @@ import torch from torch import nn -from aiter import rmsnorm2d_fwd, rmsnorm2d_fwd_with_add +from aiter import rmsnorm2d_fwd, rmsnorm2d_fwd_with_add, rms_norm class RMSNorm(nn.Module): @@ -46,10 +46,12 @@ def forward( x: torch.Tensor, residual: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + x = x.reshape(-1, x.shape[-1]) if residual is None: return rmsnorm2d_fwd(x, self.weight, self.eps) else: - out = torch.empty_like(out) - residual_out = torch.empty_like(residual) + # return self.add_rms_forward(x, residual) + residual_out = torch.empty_like(x) + out = torch.empty_like(x) rmsnorm2d_fwd_with_add(out, x, residual, residual_out, self.weight, self.eps) - return out, residual_out + return out, residual_out \ No newline at end of file diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 4e33e0bad0..3263af0028 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -20,13 +20,17 @@ def __init__( self, input_size: int, output_size: int, + tp_dim: int | None = None, bias: bool = False, params_dtype = dtypes.bf16, - quant_type = Optional.No, + quant_type = None, ): super().__init__() self.input_size = input_size self.output_size = output_size + self.tp_dim = tp_dim + self.tp_rank = dist.get_rank() + self.tp_size = dist.get_world_size() self.weight = nn.Parameter(torch.empty((self.output_size, self.input_size), dtype=params_dtype)) if bias: self.bias = nn.Parameter(torch.empty(self.output_size, dtype=params_dtype)) @@ -34,7 +38,7 @@ def __init__( else: self.register_parameter("bias", None) self.quant_type = quant_type - if quant_type != Optional.No: + if quant_type is not None: if quant_type == QuantType.per_Tensor: self.weight.weight_loader_process = functools_partial(self.weight_loader_process, post_process_func=shuffle_weight(16, 16)) self.weight_scale = nn.Parameter(torch.empty(1, dtype=dtypes.fp32)) @@ -57,7 +61,7 @@ def weight_loader_process(param: nn.Parameter, loaded_weight: torch.Tensor, post param.data.copy_(post_process_func(loaded_weight)) def forward(self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16) -> torch.Tensor: - if self.quant_type == Optional.No: + if self.quant_type is None: return tgemm.mm(x, self.weight, self.bias) else: assert x_scale is not None @@ -103,14 +107,23 @@ def __init__( output_size: int, bias: bool = False, params_dtype = dtypes.bf16, - quant_type = Optional.No, + quant_type = None, ): self.tp_dim = 0 self.tp_rank = dist.get_rank() self.tp_size = dist.get_world_size() self.input_size_per_partition = input_size self.output_size_per_partition = divide(output_size, self.tp_size) - super().__init__(input_size, self.output_size_per_partition, bias, params_dtype, quant_type) + super().__init__(input_size, output_size, self.tp_dim, bias, params_dtype, quant_type) + + self.weight = nn.Parameter(torch.empty(self.output_size_per_partition, self.input_size_per_partition)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size_per_partition)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data @@ -186,7 +199,7 @@ def __init__( bias: bool = False, ): super().__init__(input_size, output_size, 1) - self.input_size_per_partition = divide(input_size, self.tp_size) + self.input_size_per_partition = divide(input_size, 1) self.output_size_per_partition = output_size self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size_per_partition)) @@ -199,7 +212,7 @@ def __init__( def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data - shard_size = param_data.size(self.tp_dim) + shard_size = param_data.size(1) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) param_data.copy_(loaded_weight) diff --git a/bench.py b/bench.py index f67cec3566..b7e60dc8a8 100644 --- a/bench.py +++ b/bench.py @@ -12,7 +12,7 @@ def main(): max_input_len = 1024 max_ouput_len = 1024 - path = os.path.expanduser("/home/ljin1/model_zoo/Qwen3-0.6B/") + path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") llm = LLMEngine(path, enforce_eager=False, max_model_len=4096) prompt_token_ids = [ From 2923b6f0395b088c7bb9eec8bcd24c4aa94b2315 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 13 Aug 2025 17:34:33 +0800 Subject: [PATCH 004/119] enable attention for aiter --- atom/model_engine/model_runner.py | 36 +++++++++++++++++++++++++++++-- atom/model_ops/attention.py | 25 +++++++++++++++++---- atom/utils/context.py | 8 +++++-- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index e1d3fa4515..d5edffcf68 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -165,7 +165,18 @@ def prepare_decode(self, seqs: list[Sequence]): positions = [] slot_mapping = [] context_lens = [] + cu_seqlens_q = [0] + cu_seqlens_k = [0] + max_seqlen_q = 1 + max_seqlen_k = 0 + for seq in seqs: + current_seq_len = len(seq) + cu_seqlens_q.append(cu_seqlens_q[-1] + 1) + + cu_seqlens_k.append(cu_seqlens_k[-1] + current_seq_len) + max_seqlen_k = max(current_seq_len, max_seqlen_k) + input_ids.append(seq.last_token) positions.append(len(seq)) context_lens.append(len(seq)) @@ -175,7 +186,12 @@ def prepare_decode(self, seqs: list[Sequence]): slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) block_tables = self.prepare_block_tables(seqs) - set_context(False, slot_mapping=slot_mapping, context_lens=context_lens, block_tables=block_tables) + cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + + set_context(False, cu_seqlens_q=cu_seqlens_q, cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, slot_mapping=slot_mapping, context_lens=context_lens, block_tables=block_tables) + return input_ids, positions def prepare_sample(self, seqs: list[Sequence]): @@ -225,13 +241,29 @@ def capture_cudagraph(self): context_lens = torch.zeros(max_bs, dtype=torch.int32) block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32) outputs = torch.zeros(max_bs, hf_config.hidden_size) + + cu_seqlens_q = torch.arange(max_bs + 1, dtype=torch.int32) + cu_seqlens_k = torch.arange(max_bs + 1, dtype=torch.int32) + max_seqlen_q = 1 + max_seqlen_k = 1 + self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) self.graphs = {} self.graph_pool = None for bs in reversed(self.graph_bs): graph = torch.cuda.CUDAGraph() - set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) + + current_max_seqlen_k = context_lens[:bs].max().item() if bs > 0 else 1 + + set_context(False, + cu_seqlens_q=cu_seqlens_q[:bs+1], + cu_seqlens_k=cu_seqlens_k[:bs+1], + max_seqlen_q=max_seqlen_q, + max_seqlen_k=current_max_seqlen_k, + slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) + # set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup with torch.cuda.graph(graph, self.graph_pool): outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index c61125cb4d..e90094fdbf 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -5,6 +5,7 @@ import aiter from atom.utils.context import get_context +from flash_attn import flash_attn_with_kvcache @triton.jit @@ -70,7 +71,7 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): v = v.view(-1, self.num_kv_heads, self.head_dim) context = get_context() k_cache, v_cache = self.k_cache, self.v_cache - if k_cache.numel() and v_cache.numel(): + if k_cache.numel() and v_cache.numel() and context.slot_mapping is not None: store_kvcache(k, v, k_cache, v_cache, context.slot_mapping) if context.is_prefill: if context.block_tables is not None: # prefix cache @@ -79,15 +80,31 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): q, k, v, - max_seqlen_q=context.max_seqlen_q, cu_seqlens_q=context.cu_seqlens_q, - max_seqlen_k=context.max_seqlen_k, cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q = 0, + dropout_p=0.0, softmax_scale=self.scale, causal=True, ) else: # decode - o = torch.empty_like(q, dtype=q.dtype, device=q.device) + # o = torch.empty_like(q, dtype=q.dtype, device=q.device) + o = aiter.flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=context.cu_seqlens_q, + cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q = 0, + dropout_p=0.0, + softmax_scale=self.scale, + causal=True, + ) + # o = flash_attn_with_kvcache(q.unsqueeze(1), k_cache, v_cache, # cache_seqlens=context.context_lens, block_table=context.block_tables, # softmax_scale=self.scale, causal=True) diff --git a/atom/utils/context.py b/atom/utils/context.py index 2281888f80..660045f926 100644 --- a/atom/utils/context.py +++ b/atom/utils/context.py @@ -9,18 +9,22 @@ class Context: cu_seqlens_k: torch.Tensor | None = None max_seqlen_q: int = 0 max_seqlen_k: int = 0 + min_seqlen_q: int = 0 slot_mapping: torch.Tensor | None = None context_lens: torch.Tensor | None = None block_tables: torch.Tensor | None = None + dropout_p: float = 0.0 + _CONTEXT = Context() def get_context(): return _CONTEXT -def set_context(is_prefill, cu_seqlens_q=None, cu_seqlens_k=None, max_seqlen_q=0, max_seqlen_k=0, slot_mapping=None, context_lens=None, block_tables=None): +def set_context(is_prefill, cu_seqlens_q=None, cu_seqlens_k=None, max_seqlen_q=0, max_seqlen_k=0, min_seqlen_q=0, slot_mapping=None, context_lens=None, block_tables=None, + dropout_p=0.0): global _CONTEXT - _CONTEXT = Context(is_prefill, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, slot_mapping, context_lens, block_tables) + _CONTEXT = Context(is_prefill, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, min_seqlen_q, slot_mapping, context_lens, block_tables, dropout_p) def reset_context(): global _CONTEXT From 65154e44023132ed8e82d5ba4905793807032d71 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 13 Aug 2025 21:42:11 +0800 Subject: [PATCH 005/119] fix attention holding bug --- atom/model_engine/model_runner.py | 19 +++++++++++------ atom/model_ops/attention.py | 19 +++++++++++------ example.py | 35 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 example.py diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index d5edffcf68..bacb9d71bb 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -157,7 +157,9 @@ def prepare_prefill(self, seqs: list[Sequence]): cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) - set_context(True, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, slot_mapping, None, block_tables) + min_seqlen_q = 0 + dropout_p = 0.0 + set_context(True, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, min_seqlen_q, slot_mapping, None, block_tables, dropout_p) return input_ids, positions def prepare_decode(self, seqs: list[Sequence]): @@ -169,6 +171,8 @@ def prepare_decode(self, seqs: list[Sequence]): cu_seqlens_k = [0] max_seqlen_q = 1 max_seqlen_k = 0 + min_seqlen_q = 0 + dropout_p = 0.0 for seq in seqs: current_seq_len = len(seq) @@ -190,7 +194,8 @@ def prepare_decode(self, seqs: list[Sequence]): cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) set_context(False, cu_seqlens_q=cu_seqlens_q, cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, slot_mapping=slot_mapping, context_lens=context_lens, block_tables=block_tables) + max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, + slot_mapping=slot_mapping, context_lens=context_lens, block_tables=block_tables, dropout_p=dropout_p) return input_ids, positions @@ -244,8 +249,10 @@ def capture_cudagraph(self): cu_seqlens_q = torch.arange(max_bs + 1, dtype=torch.int32) cu_seqlens_k = torch.arange(max_bs + 1, dtype=torch.int32) - max_seqlen_q = 1 - max_seqlen_k = 1 + max_seqlen_q = 0 + max_seqlen_k = 0 + min_seqlen_q = 0 + dropout_p = 0.0 self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) self.graphs = {} @@ -255,13 +262,13 @@ def capture_cudagraph(self): graph = torch.cuda.CUDAGraph() current_max_seqlen_k = context_lens[:bs].max().item() if bs > 0 else 1 - set_context(False, cu_seqlens_q=cu_seqlens_q[:bs+1], cu_seqlens_k=cu_seqlens_k[:bs+1], max_seqlen_q=max_seqlen_q, max_seqlen_k=current_max_seqlen_k, - slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) + min_seqlen_q=min_seqlen_q, + slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs], dropout_p=dropout_p) # set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index e90094fdbf..7ec4b13ae9 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -84,13 +84,20 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): cu_seqlens_k=context.cu_seqlens_k, max_seqlen_q=context.max_seqlen_q, max_seqlen_k=context.max_seqlen_k, - min_seqlen_q = 0, - dropout_p=0.0, + min_seqlen_q = context.min_seqlen_q, + dropout_p=context.dropout_p, softmax_scale=self.scale, causal=True, ) else: # decode - # o = torch.empty_like(q, dtype=q.dtype, device=q.device) + # cu_seqlens_q=torch.tensor([0, q.shape[0]], + # dtype=torch.int32, + # device=q.device), + # cu_seqlens_k=torch.tensor([0, k], + # dtype=torch.int32, + # device=k.device), + # max_seqlen_q=q.shape[1] + # max_seqlen_k = k.shape[1] o = aiter.flash_attn_varlen_func( q=q, k=k, @@ -98,9 +105,9 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): cu_seqlens_q=context.cu_seqlens_q, cu_seqlens_k=context.cu_seqlens_k, max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q = 0, - dropout_p=0.0, + max_seqlen_k=0, + min_seqlen_q = context.min_seqlen_q, + dropout_p=context.dropout_p, softmax_scale=self.scale, causal=True, ) diff --git a/example.py b/example.py new file mode 100644 index 0000000000..76d2088e46 --- /dev/null +++ b/example.py @@ -0,0 +1,35 @@ +import os +from atom import LLMEngine, SamplingParams +from transformers import AutoTokenizer + + +def main(): + path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") + tokenizer = AutoTokenizer.from_pretrained(path) + llm = LLMEngine(path, enforce_eager=True, tensor_parallel_size=1) + + sampling_params = SamplingParams(temperature=0.6, max_tokens=256) + prompts = [ + "introduce yourself", + "list all prime numbers within 100", + ] + prompts = [ + tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=True + ) + for prompt in prompts + ] + print('This is prompts:', prompts) + outputs = llm.generate(prompts, sampling_params) + + for prompt, output in zip(prompts, outputs): + print("\n") + print(f"Prompt: {prompt!r}") + print(f"Completion: {output['text']!r}") + + +if __name__ == "__main__": + main() \ No newline at end of file From 855f67114b78e5b38f6698eaae63d6e89cc6720a Mon Sep 17 00:00:00 2001 From: chenjun Date: Fri, 15 Aug 2025 09:03:34 +0000 Subject: [PATCH 006/119] enable aiter pa --- atom/config.py | 8 +- atom/model_engine/llm_engine.py | 23 ++-- atom/model_engine/model_runner.py | 198 ++++++++++++++++++++++-------- atom/model_engine/sequence.py | 28 +++-- atom/model_ops/attention.py | 111 ++++++++--------- atom/models/qwen3.py | 22 ++-- 6 files changed, 252 insertions(+), 138 deletions(-) diff --git a/atom/config.py b/atom/config.py index 959ffb3576..8fffcfacf4 100644 --- a/atom/config.py +++ b/atom/config.py @@ -14,13 +14,15 @@ class Config: enforce_eager: bool = False hf_config: AutoConfig | None = None eos: int = -1 - kvcache_block_size: int = 256 + kvcache_block_size: int = 16 num_kvcache_blocks: int = -1 def __post_init__(self): assert os.path.isdir(self.model) - assert self.kvcache_block_size % 256 == 0 + assert self.kvcache_block_size % 16 == 0 assert 1 <= self.tensor_parallel_size <= 8 self.hf_config = AutoConfig.from_pretrained(self.model) - self.max_model_len = min(self.max_model_len, self.hf_config.max_position_embeddings) + self.max_model_len = min( + self.max_model_len, self.hf_config.max_position_embeddings + ) assert self.max_num_batched_tokens >= self.max_model_len diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index b34a0529c9..fb266d0e02 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -42,14 +42,16 @@ def exit(self): def add_request(self, prompt: str | list[int], sampling_params: SamplingParams): if isinstance(prompt, str): prompt = self.tokenizer.encode(prompt) - seq = Sequence(prompt, sampling_params) + seq = Sequence(prompt, self.model_runner.block_size, sampling_params) self.scheduler.add(seq) def step(self): seqs, is_prefill = self.scheduler.schedule() token_ids = self.model_runner.call("run", seqs, is_prefill) self.scheduler.postprocess(seqs, token_ids) - outputs = [(seq.seq_id, seq.completion_token_ids) for seq in seqs if seq.is_finished] + outputs = [ + (seq.seq_id, seq.completion_token_ids) for seq in seqs if seq.is_finished + ] num_tokens = sum(len(seq) for seq in seqs) if is_prefill else -len(seqs) return outputs, num_tokens @@ -69,7 +71,7 @@ def generate( for prompt, sp in zip(prompts, sampling_params): self.add_request(prompt, sp) outputs = {} - prefill_throughput = decode_throughput = 0. + prefill_throughput = decode_throughput = 0.0 while not self.is_finished(): t = perf_counter() output, num_tokens = self.step() @@ -78,16 +80,21 @@ def generate( prefill_throughput = num_tokens / (perf_counter() - t) else: decode_throughput = -num_tokens / (perf_counter() - t) - pbar.set_postfix({ - "Prefill": f"{int(prefill_throughput)}tok/s", - "Decode": f"{int(decode_throughput)}tok/s", - }) + pbar.set_postfix( + { + "Prefill": f"{int(prefill_throughput)}tok/s", + "Decode": f"{int(decode_throughput)}tok/s", + } + ) for seq_id, token_ids in output: outputs[seq_id] = token_ids if use_tqdm: pbar.update(1) outputs = [outputs[seq_id] for seq_id in sorted(outputs)] - outputs = [{"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} for token_ids in outputs] + outputs = [ + {"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} + for token_ids in outputs + ] if use_tqdm: pbar.close() return outputs diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index bacb9d71bb..f6a3e24a68 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -23,7 +23,9 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.rank = rank self.event = event - dist.init_process_group("nccl", "tcp://localhost:2333", world_size=self.world_size, rank=rank) + dist.init_process_group( + "nccl", "tcp://localhost:2333", world_size=self.world_size, rank=rank + ) torch.cuda.set_device(rank) default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) @@ -69,7 +71,7 @@ def read_shm(self): assert self.world_size > 1 and self.rank self.event.wait() n = int.from_bytes(self.shm.buf[0:4], "little") - method_name, *args = pickle.loads(self.shm.buf[4:n+4]) + method_name, *args = pickle.loads(self.shm.buf[4 : n + 4]) self.event.clear() return method_name, args @@ -78,7 +80,7 @@ def write_shm(self, method_name, *args): data = pickle.dumps([method_name, *args]) n = len(data) self.shm.buf[0:4] = n.to_bytes(4, "little") - self.shm.buf[4:n+4] = data + self.shm.buf[4 : n + 4] = data for event in self.event: event.set() @@ -91,9 +93,17 @@ def call(self, method_name, *args): def warmup_model(self): torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats() - max_num_batched_tokens, max_model_len = self.config.max_num_batched_tokens, self.config.max_model_len - num_seqs = min(max_num_batched_tokens // max_model_len, self.config.max_num_seqs) - seqs = [Sequence([0] * max_model_len) for _ in range(num_seqs)] + max_num_batched_tokens, max_model_len = ( + self.config.max_num_batched_tokens, + self.config.max_model_len, + ) + num_seqs = min( + max_num_batched_tokens // max_model_len, self.config.max_num_seqs + ) + seqs = [ + Sequence([0] * max_model_len, block_size=self.block_size) + for _ in range(num_seqs) + ] self.run(seqs, True) torch.cuda.empty_cache() @@ -105,21 +115,54 @@ def allocate_kv_cache(self): peak = torch.cuda.memory_stats()["allocated_bytes.all.peak"] current = torch.cuda.memory_stats()["allocated_bytes.all.current"] num_kv_heads = hf_config.num_key_value_heads // self.world_size - block_bytes = 2 * hf_config.num_hidden_layers * self.block_size * num_kv_heads * hf_config.head_dim * hf_config.torch_dtype.itemsize - config.num_kvcache_blocks = int(total * config.gpu_memory_utilization - used - peak + current) // block_bytes + block_bytes = ( + 2 + * hf_config.num_hidden_layers + * self.block_size + * num_kv_heads + * hf_config.head_dim + * hf_config.torch_dtype.itemsize + ) + config.num_kvcache_blocks = ( + int(total * config.gpu_memory_utilization - used - peak + current) + // block_bytes + ) assert config.num_kvcache_blocks > 0 - self.kv_cache = torch.zeros(2, hf_config.num_hidden_layers, config.num_kvcache_blocks, self.block_size, num_kv_heads, hf_config.head_dim) + self.kv_cache = torch.zeros( + 2, + hf_config.num_hidden_layers, + config.num_kvcache_blocks, + self.block_size, + num_kv_heads, + hf_config.head_dim, + ) layer_id = 0 + x = 16 // self.kv_cache.element_size() for module in self.model.modules(): if hasattr(module, "k_cache") and hasattr(module, "v_cache"): - module.k_cache = self.kv_cache[0, layer_id] - module.v_cache = self.kv_cache[1, layer_id] + module.k_cache = self.kv_cache[0, layer_id].view( + config.num_kvcache_blocks, + num_kv_heads, + hf_config.head_dim // x, + self.block_size, + x, + ) + module.v_cache = self.kv_cache[1, layer_id].view( + config.num_kvcache_blocks, + num_kv_heads, + hf_config.head_dim, + self.block_size, + ) layer_id += 1 def prepare_block_tables(self, seqs: list[Sequence]): max_len = max(len(seq.block_table) for seq in seqs) - block_tables = [seq.block_table + [-1] * (max_len - len(seq.block_table)) for seq in seqs] - block_tables = torch.tensor(block_tables, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + block_tables = [ + seq.block_table + [-1] * (max_len - len(seq.block_table)) for seq in seqs + ] + block_tables = torch.tensor( + block_tables, dtype=torch.int32, pin_memory=True + ).cuda(non_blocking=True) return block_tables def prepare_prefill(self, seqs: list[Sequence]): @@ -133,7 +176,7 @@ def prepare_prefill(self, seqs: list[Sequence]): block_tables = None for seq in seqs: seqlen = len(seq) - input_ids.extend(seq[seq.num_cached_tokens:]) + input_ids.extend(seq[seq.num_cached_tokens :]) positions.extend(list(range(seq.num_cached_tokens, seqlen))) seqlen_q = seqlen - seq.num_cached_tokens seqlen_k = seqlen @@ -148,18 +191,39 @@ def prepare_prefill(self, seqs: list[Sequence]): if i != seq.num_blocks - 1: end = start + self.block_size else: - end = start + seq.last_block_num_tokens + end = start + seq.last_block_num_tokens slot_mapping.extend(list(range(start, end))) - if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache + if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache block_tables = self.prepare_block_tables(seqs) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) - cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) - cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda( + non_blocking=True + ) + positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda( + non_blocking=True + ) + cu_seqlens_q = torch.tensor( + cu_seqlens_q, dtype=torch.int32, pin_memory=True + ).cuda(non_blocking=True) + cu_seqlens_k = torch.tensor( + cu_seqlens_k, dtype=torch.int32, pin_memory=True + ).cuda(non_blocking=True) + slot_mapping = torch.tensor( + slot_mapping, dtype=torch.int64, pin_memory=True + ).cuda(non_blocking=True) min_seqlen_q = 0 dropout_p = 0.0 - set_context(True, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, min_seqlen_q, slot_mapping, None, block_tables, dropout_p) + set_context( + True, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + slot_mapping, + None, + block_tables, + dropout_p, + ) return input_ids, positions def prepare_decode(self, seqs: list[Sequence]): @@ -180,22 +244,45 @@ def prepare_decode(self, seqs: list[Sequence]): cu_seqlens_k.append(cu_seqlens_k[-1] + current_seq_len) max_seqlen_k = max(current_seq_len, max_seqlen_k) - + input_ids.append(seq.last_token) positions.append(len(seq)) context_lens.append(len(seq)) - slot_mapping.append(seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda(non_blocking=True) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) - context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + slot_mapping.append( + seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 + ) + input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda( + non_blocking=True + ) + positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda( + non_blocking=True + ) + slot_mapping = torch.tensor( + slot_mapping, dtype=torch.int64, pin_memory=True + ).cuda(non_blocking=True) + context_lens = torch.tensor( + context_lens, dtype=torch.int32, pin_memory=True + ).cuda(non_blocking=True) block_tables = self.prepare_block_tables(seqs) - cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) - cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True).cuda(non_blocking=True) + cu_seqlens_q = torch.tensor( + cu_seqlens_q, dtype=torch.int32, pin_memory=True + ).cuda(non_blocking=True) + cu_seqlens_k = torch.tensor( + cu_seqlens_k, dtype=torch.int32, pin_memory=True + ).cuda(non_blocking=True) - set_context(False, cu_seqlens_q=cu_seqlens_q, cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping, context_lens=context_lens, block_tables=block_tables, dropout_p=dropout_p) + set_context( + False, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + min_seqlen_q=min_seqlen_q, + slot_mapping=slot_mapping, + context_lens=context_lens, + block_tables=block_tables, + dropout_p=dropout_p, + ) return input_ids, positions @@ -203,11 +290,15 @@ def prepare_sample(self, seqs: list[Sequence]): temperatures = [] for seq in seqs: temperatures.append(seq.temperature) - temperatures = torch.tensor(temperatures, dtype=torch.float32, pin_memory=True).cuda(non_blocking=True) + temperatures = torch.tensor( + temperatures, dtype=torch.float32, pin_memory=True + ).cuda(non_blocking=True) return temperatures @torch.inference_mode() - def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool): + def run_model( + self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool + ): if is_prefill or self.enforce_eager or input_ids.size(0) > 512: return self.model.compute_logits(self.model(input_ids, positions)) else: @@ -222,15 +313,21 @@ def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill graph_vars["positions"][:bs] = positions graph_vars["slot_mapping"][:bs] = context.slot_mapping graph_vars["context_lens"][:bs] = context.context_lens - graph_vars["block_tables"][:bs, :context.block_tables.size(1)] = context.block_tables + graph_vars["block_tables"][ + :bs, : context.block_tables.size(1) + ] = context.block_tables graph.replay() return self.model.compute_logits(graph_vars["outputs"][:bs]) def run(self, seqs: list[Sequence], is_prefill: bool) -> list[int]: - input_ids, positions = self.prepare_prefill(seqs) if is_prefill else self.prepare_decode(seqs) + input_ids, positions = ( + self.prepare_prefill(seqs) if is_prefill else self.prepare_decode(seqs) + ) temperatures = self.prepare_sample(seqs) if self.rank == 0 else None logits = self.run_model(input_ids, positions, is_prefill) - token_ids = self.sampler(logits, temperatures).tolist() if self.rank == 0 else None + token_ids = ( + self.sampler(logits, temperatures).tolist() if self.rank == 0 else None + ) reset_context() return token_ids @@ -242,7 +339,7 @@ def capture_cudagraph(self): max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size input_ids = torch.zeros(max_bs, dtype=torch.int64) positions = torch.zeros(max_bs, dtype=torch.int64) - slot_mapping = torch.zeros(max_bs, dtype=torch.int32) + slot_mapping = torch.zeros(max_bs, dtype=torch.int64) context_lens = torch.zeros(max_bs, dtype=torch.int32) block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32) outputs = torch.zeros(max_bs, hf_config.hidden_size) @@ -262,18 +359,23 @@ def capture_cudagraph(self): graph = torch.cuda.CUDAGraph() current_max_seqlen_k = context_lens[:bs].max().item() if bs > 0 else 1 - set_context(False, - cu_seqlens_q=cu_seqlens_q[:bs+1], - cu_seqlens_k=cu_seqlens_k[:bs+1], - max_seqlen_q=max_seqlen_q, - max_seqlen_k=current_max_seqlen_k, - min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs], dropout_p=dropout_p) + set_context( + False, + cu_seqlens_q=cu_seqlens_q[: bs + 1], + cu_seqlens_k=cu_seqlens_k[: bs + 1], + max_seqlen_q=max_seqlen_q, + max_seqlen_k=current_max_seqlen_k, + min_seqlen_q=min_seqlen_q, + slot_mapping=slot_mapping[:bs], + context_lens=context_lens[:bs], + block_tables=block_tables[:bs], + dropout_p=dropout_p, + ) # set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) - outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup with torch.cuda.graph(graph, self.graph_pool): - outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture if self.graph_pool is None: self.graph_pool = graph.pool() self.graphs[bs] = graph diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 5fb996a304..644fb6f2ec 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -12,10 +12,11 @@ class SequenceStatus(Enum): class Sequence: - block_size = 256 counter = count() - def __init__(self, token_ids: list[int], sampling_params = SamplingParams()): + def __init__( + self, token_ids: list[int], block_size: int, sampling_params=SamplingParams() + ): self.seq_id = next(Sequence.counter) self.status = SequenceStatus.WAITING self.token_ids = copy(token_ids) @@ -27,6 +28,7 @@ def __init__(self, token_ids: list[int], sampling_params = SamplingParams()): self.temperature = sampling_params.temperature self.max_tokens = sampling_params.max_tokens self.ignore_eos = sampling_params.ignore_eos + self.block_size = block_size def __len__(self): return self.num_tokens @@ -44,11 +46,11 @@ def num_completion_tokens(self): @property def prompt_token_ids(self): - return self.token_ids[:self.num_prompt_tokens] + return self.token_ids[: self.num_prompt_tokens] @property def completion_token_ids(self): - return self.token_ids[self.num_prompt_tokens:] + return self.token_ids[self.num_prompt_tokens :] @property def num_cached_blocks(self): @@ -64,7 +66,7 @@ def last_block_num_tokens(self): def block(self, i): assert 0 <= i < self.num_blocks - return self.token_ids[i*self.block_size: (i+1)*self.block_size] + return self.token_ids[i * self.block_size : (i + 1) * self.block_size] def append_token(self, token_id: int): self.token_ids.append(token_id) @@ -72,11 +74,21 @@ def append_token(self, token_id: int): self.num_tokens += 1 def __getstate__(self): - return (self.num_tokens, self.num_prompt_tokens, self.num_cached_tokens, self.block_table, - self.token_ids if self.num_completion_tokens == 0 else self.last_token) + return ( + self.num_tokens, + self.num_prompt_tokens, + self.num_cached_tokens, + self.block_table, + self.token_ids if self.num_completion_tokens == 0 else self.last_token, + ) def __setstate__(self, state): - self.num_tokens, self.num_prompt_tokens, self.num_cached_tokens, self.block_table = state[:-1] + ( + self.num_tokens, + self.num_prompt_tokens, + self.num_cached_tokens, + self.block_table, + ) = state[:-1] if self.num_completion_tokens == 0: self.token_ids = state[-1] else: diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 7ec4b13ae9..4f6e40589e 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -4,50 +4,11 @@ import triton.language as tl import aiter +from aiter.paged_attn import PagedAttention from atom.utils.context import get_context from flash_attn import flash_attn_with_kvcache -@triton.jit -def store_kvcache_kernel( - key_ptr, - key_stride, - value_ptr, - value_stride, - k_cache_ptr, - v_cache_ptr, - slot_mapping_ptr, - D: tl.constexpr, -): - idx = tl.program_id(0) - key_offsets = idx * key_stride + tl.arange(0, D) - value_offsets = idx * value_stride + tl.arange(0, D) - key = tl.load(key_ptr + key_offsets) - value = tl.load(value_ptr + value_offsets) - slot = tl.load(slot_mapping_ptr + idx) - cache_offsets = slot * D + tl.arange(0, D) - tl.store(k_cache_ptr + cache_offsets, key) - tl.store(v_cache_ptr + cache_offsets, value) - - -def store_kvcache( - key: torch.Tensor, - value: torch.Tensor, - k_cache: torch.Tensor, - v_cache: torch.Tensor, - slot_mapping: torch.Tensor, -): - N, num_heads, head_dim = key.shape - D = num_heads * head_dim - assert key.stride(-1) == 1 and value.stride(-1) == 1 - assert key.stride(1) == head_dim and value.stride(1) == head_dim - assert k_cache.stride(1) == D and v_cache.stride(1) == D - assert slot_mapping.numel() == N - store_kvcache_kernel[(N,)]( - key, key.stride(0), value, value.stride(0), k_cache, v_cache, slot_mapping, D - ) - - class Attention(nn.Module): def __init__( @@ -56,13 +17,17 @@ def __init__( head_dim, scale, num_kv_heads, + max_model_len=None, ): super().__init__() self.num_heads = num_heads self.head_dim = head_dim self.scale = scale self.num_kv_heads = num_kv_heads + self.max_model_len = max_model_len self.k_cache = self.v_cache = torch.tensor([]) + self.k_scale = self.v_scale = None + self.k_scale = self.v_scale = torch.tensor(1.0, dtype=torch.float32) def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): o: torch.Tensor @@ -72,7 +37,17 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): context = get_context() k_cache, v_cache = self.k_cache, self.v_cache if k_cache.numel() and v_cache.numel() and context.slot_mapping is not None: - store_kvcache(k, v, k_cache, v_cache, context.slot_mapping) + aiter.reshape_and_cache( + k, + v, + k_cache, + v_cache, + context.slot_mapping, + kv_cache_dtype="auto", + k_scale=None, + v_scale=None, + asm_layout=False, + ) if context.is_prefill: if context.block_tables is not None: # prefix cache k, v = k_cache, v_cache @@ -84,32 +59,44 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): cu_seqlens_k=context.cu_seqlens_k, max_seqlen_q=context.max_seqlen_q, max_seqlen_k=context.max_seqlen_k, - min_seqlen_q = context.min_seqlen_q, + min_seqlen_q=context.min_seqlen_q, dropout_p=context.dropout_p, softmax_scale=self.scale, causal=True, ) else: # decode - # cu_seqlens_q=torch.tensor([0, q.shape[0]], - # dtype=torch.int32, - # device=q.device), - # cu_seqlens_k=torch.tensor([0, k], - # dtype=torch.int32, - # device=k.device), - # max_seqlen_q=q.shape[1] - # max_seqlen_k = k.shape[1] - o = aiter.flash_attn_varlen_func( - q=q, - k=k, - v=v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=0, - min_seqlen_q = context.min_seqlen_q, - dropout_p=context.dropout_p, - softmax_scale=self.scale, - causal=True, + + # print(f"{context.block_tables=}") + # print(f"{context.context_lens=}") + # kvcache_block_size = self.k_cache.shape[-2] + # max_num_blocks = ( + # self.max_model_len + kvcache_block_size - 1 + # ) // kvcache_block_size + # o = aiter.pa_fwd_asm( + # q.contiguous(), + # k_cache, + # v_cache, + # context.block_tables, + # context.context_lens, + # max_num_blocks, + # K_QScale=self.k_scale, + # V_QScale=self.v_scale, + # ) + # self.k_scale = self.k_scale.to(q.device) + # self.v_scale = self.v_scale.to(q.device) + o = PagedAttention.forward_decode( + q, + k_cache, + v_cache, + context.block_tables, + context.context_lens, + max_seq_len=self.max_model_len, + kv_cache_dtype="auto", + num_kv_heads=self.num_kv_heads, + scale=self.scale, + alibi_slopes=None, + k_scale=self.k_scale, + v_scale=self.v_scale, ) # o = flash_attn_with_kvcache(q.unsqueeze(1), k_cache, v_cache, diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index a6536ca0db..bf969735e5 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -66,6 +66,7 @@ def __init__( self.head_dim, self.scaling, self.num_kv_heads, + max_model_len=max_position, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) @@ -131,8 +132,8 @@ def __init__( num_kv_heads=config.num_key_value_heads, max_position=config.max_position_embeddings, rms_norm_eps=config.rms_norm_eps, - qkv_bias=getattr(config, 'attention_bias', False), - head_dim=getattr(config, 'head_dim', None), + qkv_bias=getattr(config, "attention_bias", False), + head_dim=getattr(config, "head_dim", None), rope_theta=getattr(config, "rope_theta", 1000000), rope_scaling=getattr(config, "rope_scaling", None), ) @@ -142,7 +143,9 @@ def __init__( hidden_act=config.hidden_act, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) def forward( self, @@ -168,8 +171,12 @@ def __init__( config: Qwen3Config, ) -> None: super().__init__() - self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size) - self.layers = nn.ModuleList([Qwen3DecoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, config.hidden_size + ) + self.layers = nn.ModuleList( + [Qwen3DecoderLayer(config) for _ in range(config.num_hidden_layers)] + ) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) def forward( @@ -194,10 +201,7 @@ class Qwen3ForCausalLM(nn.Module): "up_proj": ("gate_up_proj", 1), } - def __init__( - self, - config: Qwen3Config - ) -> None: + def __init__(self, config: Qwen3Config) -> None: super().__init__() self.model = Qwen3Model(config) self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size) From af84ec72e37b6dd973aa2d82e071999e2707b3ef Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sun, 17 Aug 2025 11:49:34 +0800 Subject: [PATCH 007/119] revert self.to_dim --- atom/model_ops/linear.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 3263af0028..8f854692e2 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -212,7 +212,7 @@ def __init__( def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data - shard_size = param_data.size(1) + shard_size = param_data.size(self.tp_dim) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) param_data.copy_(loaded_weight) From ad27df69bbb5e755d33454569ce9fef7063e5db6 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Tue, 19 Aug 2025 10:47:51 +0800 Subject: [PATCH 008/119] fix pa_asm acc issur --- atom/model_engine/model_runner.py | 44 ++++++++---- atom/model_engine/scheduler.py | 4 ++ atom/model_ops/attention.py | 113 +++++++++++++++++++----------- atom/models/qwen3.py | 2 +- example.py | 3 +- 5 files changed, 108 insertions(+), 58 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index f6a3e24a68..74cb362629 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -3,6 +3,7 @@ import torch.distributed as dist from multiprocessing.synchronize import Event from multiprocessing.shared_memory import SharedMemory +from aiter import dtypes from atom.config import Config from atom.model_engine.sequence import Sequence @@ -135,7 +136,22 @@ def allocate_kv_cache(self): self.block_size, num_kv_heads, hf_config.head_dim, + dtype=dtypes.fp8, + device="cuda" ) + # kv_scale_shape = (config.num_kvcache_blocks, num_kv_heads, self.block_size) + # self.k_scale = torch.empty(kv_scale_shape) + # self.v_scale = torch.empty_like(self.k_scale) + self.kv_scale = torch.zeros( + 2, + hf_config.num_hidden_layers, + config.num_kvcache_blocks, + self.block_size, + num_kv_heads, + dtype=dtypes.fp32, + device="cuda" + ) + layer_id = 0 x = 16 // self.kv_cache.element_size() for module in self.model.modules(): @@ -153,8 +169,16 @@ def allocate_kv_cache(self): hf_config.head_dim, self.block_size, ) + # module.k_scale = self.k_scale + # module.v_scale = self.v_scale + module.k_scale = self.kv_scale[0, layer_id] + module.v_scale = self.kv_scale[1, layer_id] + module.max_model_len = self.config.max_model_len + # module.max_model_len = 256 + layer_id += 1 + def prepare_block_tables(self, seqs: list[Sequence]): max_len = max(len(seq.block_table) for seq in seqs) block_tables = [ @@ -270,7 +294,6 @@ def prepare_decode(self, seqs: list[Sequence]): cu_seqlens_k = torch.tensor( cu_seqlens_k, dtype=torch.int32, pin_memory=True ).cuda(non_blocking=True) - set_context( False, cu_seqlens_q=cu_seqlens_q, @@ -299,9 +322,13 @@ def prepare_sample(self, seqs: list[Sequence]): def run_model( self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool ): + torch.cuda.empty_cache() if is_prefill or self.enforce_eager or input_ids.size(0) > 512: + # print('input_ids', input_ids.size(0)) return self.model.compute_logits(self.model(input_ids, positions)) else: + # print('input_ids 2222', input_ids.shape) + bs = input_ids.size(0) context = get_context() graph = self.graphs[next(x for x in self.graph_bs if x >= bs)] @@ -357,21 +384,8 @@ def capture_cudagraph(self): for bs in reversed(self.graph_bs): graph = torch.cuda.CUDAGraph() + set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) - current_max_seqlen_k = context_lens[:bs].max().item() if bs > 0 else 1 - set_context( - False, - cu_seqlens_q=cu_seqlens_q[: bs + 1], - cu_seqlens_k=cu_seqlens_k[: bs + 1], - max_seqlen_q=max_seqlen_q, - max_seqlen_k=current_max_seqlen_k, - min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping[:bs], - context_lens=context_lens[:bs], - block_tables=block_tables[:bs], - dropout_p=dropout_p, - ) - # set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup with torch.cuda.graph(graph, self.graph_pool): diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 6bdc537a64..c2db93844b 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -26,6 +26,10 @@ def schedule(self) -> tuple[list[Sequence], bool]: scheduled_seqs = [] num_seqs = 0 num_batched_tokens = 0 + if not self.running and not self.waiting: + self.block_manager.reset() + + while self.waiting and num_seqs < self.max_num_seqs: seq = self.waiting[0] if num_batched_tokens + len(seq) > self.max_num_batched_tokens or not self.block_manager.can_allocate(seq): diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 4f6e40589e..b530a107bf 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -17,16 +17,17 @@ def __init__( head_dim, scale, num_kv_heads, - max_model_len=None, + # max_model_len=None, ): super().__init__() self.num_heads = num_heads self.head_dim = head_dim self.scale = scale self.num_kv_heads = num_kv_heads - self.max_model_len = max_model_len + # self.max_model_len = max_model_len self.k_cache = self.v_cache = torch.tensor([]) - self.k_scale = self.v_scale = None + # self.k_scale = self.v_scale = None + self.max_model_len = 0 self.k_scale = self.v_scale = torch.tensor(1.0, dtype=torch.float32) def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): @@ -37,20 +38,26 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): context = get_context() k_cache, v_cache = self.k_cache, self.v_cache if k_cache.numel() and v_cache.numel() and context.slot_mapping is not None: - aiter.reshape_and_cache( - k, - v, - k_cache, - v_cache, - context.slot_mapping, - kv_cache_dtype="auto", - k_scale=None, - v_scale=None, - asm_layout=False, - ) + # aiter.reshape_and_cache( + # k, + # v, + # k_cache, + # v_cache, + # context.slot_mapping, + # kv_cache_dtype="auto", + # k_scale=None, + # v_scale=None, + # asm_layout=False, + # ) + # k_scale = self.k_scale + # v_scale = self.v_scale + aiter.reshape_and_cache_with_pertoken_quant( + k, v, k_cache, v_cache, self.k_scale, self.v_scale, context.slot_mapping, asm_layout=True + ) + if context.is_prefill: - if context.block_tables is not None: # prefix cache - k, v = k_cache, v_cache + # if context.block_tables is not None: # prefix cache + # k, v = k_cache, v_cache o = aiter.flash_attn_varlen_func( q, k, @@ -68,36 +75,60 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): # print(f"{context.block_tables=}") # print(f"{context.context_lens=}") - # kvcache_block_size = self.k_cache.shape[-2] - # max_num_blocks = ( - # self.max_model_len + kvcache_block_size - 1 - # ) // kvcache_block_size - # o = aiter.pa_fwd_asm( - # q.contiguous(), - # k_cache, - # v_cache, - # context.block_tables, - # context.context_lens, - # max_num_blocks, - # K_QScale=self.k_scale, - # V_QScale=self.v_scale, - # ) - # self.k_scale = self.k_scale.to(q.device) - # self.v_scale = self.v_scale.to(q.device) - o = PagedAttention.forward_decode( - q, + kvcache_block_size = self.k_cache.shape[-2] + max_num_blocks = ( + self.max_model_len + kvcache_block_size - 1 + ) // kvcache_block_size + # print(q.shape) + # print(k_cache.shape) + # print(v_cache.shape) + # print(context.context_lens.shape) + # print(context.block_tables.shape) + # print(max_num_blocks) + # print(self.k_scale.shape) + # print(self.v_scale.shape) + + # print('=================================') + # def asm_V_shuffle(VC): + # # [num_blocks, num_kv_heads, head_size, block_size] + # x = 16 // VC.element_size() + # num_blocks, num_kv_heads, head_size, block_size = VC.shape + # VC = VC.view(num_blocks, num_kv_heads, head_size, block_size // x, x) + # # [num_blocks, num_kv_heads, block_size/X, head_size, X] + # VC = VC.permute(0, 1, 3, 2, 4).contiguous() + # return VC + + o = aiter.pa_fwd_asm( + q.contiguous(), k_cache, v_cache, context.block_tables, context.context_lens, - max_seq_len=self.max_model_len, - kv_cache_dtype="auto", - num_kv_heads=self.num_kv_heads, - scale=self.scale, - alibi_slopes=None, - k_scale=self.k_scale, - v_scale=self.v_scale, + max_num_blocks, + K_QScale=self.k_scale, + V_QScale=self.v_scale, + out_=None, + high_precision=0 ) + # self.k_scale = self.k_scale.to(q.device) + # self.v_scale = self.v_scale.to(q.device) + # print(f"{self.k_scale}") + # print(f"{self.k_scale}") + + # o = PagedAttention.forward_decode( + # q, + # k_cache, + # v_cache, + # context.block_tables, + # context.context_lens, + # max_seq_len=self.max_model_len, + # kv_cache_dtype="auto", + # num_kv_heads=self.num_kv_heads, + # scale=self.scale, + # alibi_slopes=None, + # k_scale=self.k_scale, + # v_scale=self.v_scale, + # ) # o = flash_attn_with_kvcache(q.unsqueeze(1), k_cache, v_cache, # cache_seqlens=context.context_lens, block_table=context.block_tables, diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index bf969735e5..a21be3dc7a 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -66,7 +66,7 @@ def __init__( self.head_dim, self.scaling, self.num_kv_heads, - max_model_len=max_position, + # max_model_len=max_position, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) diff --git a/example.py b/example.py index 76d2088e46..dc100a9ef5 100644 --- a/example.py +++ b/example.py @@ -6,12 +6,13 @@ def main(): path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") tokenizer = AutoTokenizer.from_pretrained(path) - llm = LLMEngine(path, enforce_eager=True, tensor_parallel_size=1) + llm = LLMEngine(path, enforce_eager=False, tensor_parallel_size=1) sampling_params = SamplingParams(temperature=0.6, max_tokens=256) prompts = [ "introduce yourself", "list all prime numbers within 100", + ] prompts = [ tokenizer.apply_chat_template( From 5a53f4857b6d0181989503921291d36d258d408c Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Tue, 19 Aug 2025 17:51:32 +0800 Subject: [PATCH 009/119] fix acc issue and support bf16 and fp8 quant --- atom/config.py | 1 + atom/model_engine/model_runner.py | 32 +++++--------- atom/model_ops/attention.py | 72 ++++++++++--------------------- atom/models/qwen3.py | 12 ++++-- example.py | 18 +++++++- 5 files changed, 58 insertions(+), 77 deletions(-) diff --git a/atom/config.py b/atom/config.py index 8fffcfacf4..b3c02e672d 100644 --- a/atom/config.py +++ b/atom/config.py @@ -16,6 +16,7 @@ class Config: eos: int = -1 kvcache_block_size: int = 16 num_kvcache_blocks: int = -1 + kv_cache_dtype: str = "bf16" def __post_init__(self): assert os.path.isdir(self.model) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 74cb362629..3a7156baaa 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -31,7 +31,7 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") - self.model = Qwen3ForCausalLM(hf_config) + self.model = Qwen3ForCausalLM(hf_config, config.kv_cache_dtype) load_model(self.model, config.model) self.sampler = Sampler() self.warmup_model() @@ -136,12 +136,10 @@ def allocate_kv_cache(self): self.block_size, num_kv_heads, hf_config.head_dim, - dtype=dtypes.fp8, + dtype=dtypes.fp8 if config.kv_cache_dtype == "fp8" else dtypes.bf16, device="cuda" ) - # kv_scale_shape = (config.num_kvcache_blocks, num_kv_heads, self.block_size) - # self.k_scale = torch.empty(kv_scale_shape) - # self.v_scale = torch.empty_like(self.k_scale) + self.kv_scale = torch.zeros( 2, hf_config.num_hidden_layers, @@ -169,12 +167,10 @@ def allocate_kv_cache(self): hf_config.head_dim, self.block_size, ) - # module.k_scale = self.k_scale - # module.v_scale = self.v_scale - module.k_scale = self.kv_scale[0, layer_id] - module.v_scale = self.kv_scale[1, layer_id] module.max_model_len = self.config.max_model_len - # module.max_model_len = 256 + if config.kv_cache_dtype == "fp8": + module.k_scale = self.kv_scale[0, layer_id] + module.v_scale = self.kv_scale[1, layer_id] layer_id += 1 @@ -324,18 +320,18 @@ def run_model( ): torch.cuda.empty_cache() if is_prefill or self.enforce_eager or input_ids.size(0) > 512: - # print('input_ids', input_ids.size(0)) return self.model.compute_logits(self.model(input_ids, positions)) else: - # print('input_ids 2222', input_ids.shape) - bs = input_ids.size(0) + graph_bs = next(x for x in self.graph_bs if x >= bs) + context = get_context() - graph = self.graphs[next(x for x in self.graph_bs if x >= bs)] + graph = self.graphs[graph_bs] graph_vars = self.graph_vars for k, v in graph_vars.items(): if k != "outputs": v.zero_() + graph_vars["slot_mapping"][bs:graph_bs] = -1 graph_vars["input_ids"][:bs] = input_ids graph_vars["positions"][:bs] = positions graph_vars["slot_mapping"][:bs] = context.slot_mapping @@ -371,13 +367,6 @@ def capture_cudagraph(self): block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32) outputs = torch.zeros(max_bs, hf_config.hidden_size) - cu_seqlens_q = torch.arange(max_bs + 1, dtype=torch.int32) - cu_seqlens_k = torch.arange(max_bs + 1, dtype=torch.int32) - max_seqlen_q = 0 - max_seqlen_k = 0 - min_seqlen_q = 0 - dropout_p = 0.0 - self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) self.graphs = {} self.graph_pool = None @@ -386,7 +375,6 @@ def capture_cudagraph(self): graph = torch.cuda.CUDAGraph() set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) - outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup with torch.cuda.graph(graph, self.graph_pool): outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index b530a107bf..74ca10d5e3 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -17,18 +17,17 @@ def __init__( head_dim, scale, num_kv_heads, - # max_model_len=None, + kv_cache_dtype="bf16", ): super().__init__() self.num_heads = num_heads self.head_dim = head_dim self.scale = scale self.num_kv_heads = num_kv_heads - # self.max_model_len = max_model_len self.k_cache = self.v_cache = torch.tensor([]) - # self.k_scale = self.v_scale = None + self.kv_cache_dtype = kv_cache_dtype self.max_model_len = 0 - self.k_scale = self.v_scale = torch.tensor(1.0, dtype=torch.float32) + self.k_scale = self.v_scale = None def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): o: torch.Tensor @@ -38,22 +37,23 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): context = get_context() k_cache, v_cache = self.k_cache, self.v_cache if k_cache.numel() and v_cache.numel() and context.slot_mapping is not None: - # aiter.reshape_and_cache( - # k, - # v, - # k_cache, - # v_cache, - # context.slot_mapping, - # kv_cache_dtype="auto", - # k_scale=None, - # v_scale=None, - # asm_layout=False, - # ) - # k_scale = self.k_scale - # v_scale = self.v_scale - aiter.reshape_and_cache_with_pertoken_quant( - k, v, k_cache, v_cache, self.k_scale, self.v_scale, context.slot_mapping, asm_layout=True - ) + if self.kv_cache_dtype == "fp8": + aiter.reshape_and_cache_with_pertoken_quant( + k, v, k_cache, v_cache, self.k_scale, self.v_scale, context.slot_mapping, asm_layout=True + ) + else: + aiter.reshape_and_cache( + k, + v, + k_cache, + v_cache, + context.slot_mapping, + kv_cache_dtype="auto", + k_scale=None, + v_scale=None, + asm_layout=True, + ) + if context.is_prefill: # if context.block_tables is not None: # prefix cache @@ -73,30 +73,6 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): ) else: # decode - # print(f"{context.block_tables=}") - # print(f"{context.context_lens=}") - kvcache_block_size = self.k_cache.shape[-2] - max_num_blocks = ( - self.max_model_len + kvcache_block_size - 1 - ) // kvcache_block_size - # print(q.shape) - # print(k_cache.shape) - # print(v_cache.shape) - # print(context.context_lens.shape) - # print(context.block_tables.shape) - # print(max_num_blocks) - # print(self.k_scale.shape) - # print(self.v_scale.shape) - - # print('=================================') - # def asm_V_shuffle(VC): - # # [num_blocks, num_kv_heads, head_size, block_size] - # x = 16 // VC.element_size() - # num_blocks, num_kv_heads, head_size, block_size = VC.shape - # VC = VC.view(num_blocks, num_kv_heads, head_size, block_size // x, x) - # # [num_blocks, num_kv_heads, block_size/X, head_size, X] - # VC = VC.permute(0, 1, 3, 2, 4).contiguous() - # return VC o = aiter.pa_fwd_asm( q.contiguous(), @@ -104,17 +80,13 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): v_cache, context.block_tables, context.context_lens, - max_num_blocks, + context.block_tables.stride(0), K_QScale=self.k_scale, V_QScale=self.v_scale, out_=None, high_precision=0 ) - # self.k_scale = self.k_scale.to(q.device) - # self.v_scale = self.v_scale.to(q.device) - # print(f"{self.k_scale}") - # print(f"{self.k_scale}") - + # o = PagedAttention.forward_decode( # q, # k_cache, diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index a21be3dc7a..6d7306f521 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -28,6 +28,7 @@ def __init__( qkv_bias: bool = False, rope_theta: float = 10000, rope_scaling: tuple | None = None, + kv_cache_dtype: str = "fp16" ) -> None: super().__init__() tp_size = dist.get_world_size() @@ -66,7 +67,7 @@ def __init__( self.head_dim, self.scaling, self.num_kv_heads, - # max_model_len=max_position, + kv_cache_dtype=kv_cache_dtype, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) @@ -124,6 +125,7 @@ class Qwen3DecoderLayer(nn.Module): def __init__( self, config: Qwen3Config, + kv_cache_dtype: str ) -> None: super().__init__() self.self_attn = Qwen3Attention( @@ -136,6 +138,7 @@ def __init__( head_dim=getattr(config, "head_dim", None), rope_theta=getattr(config, "rope_theta", 1000000), rope_scaling=getattr(config, "rope_scaling", None), + kv_cache_dtype=kv_cache_dtype, ) self.mlp = Qwen3MLP( hidden_size=config.hidden_size, @@ -169,13 +172,14 @@ class Qwen3Model(nn.Module): def __init__( self, config: Qwen3Config, + kv_cache_dtype: str ) -> None: super().__init__() self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size ) self.layers = nn.ModuleList( - [Qwen3DecoderLayer(config) for _ in range(config.num_hidden_layers)] + [Qwen3DecoderLayer(config, kv_cache_dtype) for _ in range(config.num_hidden_layers)] ) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -201,9 +205,9 @@ class Qwen3ForCausalLM(nn.Module): "up_proj": ("gate_up_proj", 1), } - def __init__(self, config: Qwen3Config) -> None: + def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: super().__init__() - self.model = Qwen3Model(config) + self.model = Qwen3Model(config, kv_cache_dtype) self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size) if config.tie_word_embeddings: self.lm_head.weight.data = self.model.embed_tokens.weight.data diff --git a/example.py b/example.py index dc100a9ef5..9817c31442 100644 --- a/example.py +++ b/example.py @@ -1,12 +1,28 @@ import os from atom import LLMEngine, SamplingParams from transformers import AutoTokenizer +import argparse + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="config of test", +) + +parser.add_argument( + "--kv_cache_dtype", + choices=["bf16", "fp8"], + type=str, + default="bf16", + help="""KV cache type. Default is 'bf16'. + e.g.: -kv_cache_dtype fp8""", +) def main(): + args = parser.parse_args() path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") tokenizer = AutoTokenizer.from_pretrained(path) - llm = LLMEngine(path, enforce_eager=False, tensor_parallel_size=1) + llm = LLMEngine(path, enforce_eager=False, tensor_parallel_size=1, kv_cache_dtype=args.kv_cache_dtype) sampling_params = SamplingParams(temperature=0.6, max_tokens=256) prompts = [ From f96200066ac6182fed54132a6ec1d8753b658bdd Mon Sep 17 00:00:00 2001 From: chenjun Date: Tue, 19 Aug 2025 11:57:54 +0000 Subject: [PATCH 010/119] add download_weights_from_hf --- atom/config.py | 2 +- atom/model_engine/model_runner.py | 6 +- atom/model_loader/loader.py | 13 +- atom/model_loader/weight_utils.py | 115 ++++++ atom/models/llama.py | 640 ++++++++++++++++++++++++++++++ example.py | 22 +- 6 files changed, 785 insertions(+), 13 deletions(-) create mode 100644 atom/model_loader/weight_utils.py create mode 100644 atom/models/llama.py diff --git a/atom/config.py b/atom/config.py index b3c02e672d..3adec52c57 100644 --- a/atom/config.py +++ b/atom/config.py @@ -19,7 +19,7 @@ class Config: kv_cache_dtype: str = "bf16" def __post_init__(self): - assert os.path.isdir(self.model) + # assert os.path.isdir(self.model) assert self.kvcache_block_size % 16 == 0 assert 1 <= self.tensor_parallel_size <= 8 self.hf_config = AutoConfig.from_pretrained(self.model) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 3a7156baaa..bfc3d2d3df 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -328,9 +328,9 @@ def run_model( context = get_context() graph = self.graphs[graph_bs] graph_vars = self.graph_vars - for k, v in graph_vars.items(): - if k != "outputs": - v.zero_() + # for k, v in graph_vars.items(): + # if k != "outputs": + # v.zero_() graph_vars["slot_mapping"][bs:graph_bs] = -1 graph_vars["input_ids"][:bs] = input_ids graph_vars["positions"][:bs] = positions diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index 74a6264381..5b66b2b470 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -3,15 +3,24 @@ import torch from torch import nn from safetensors import safe_open +from .weight_utils import download_weights_from_hf +from tqdm import tqdm def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor): param.data.copy_(loaded_weight) -def load_model(model: nn.Module, path: str): +def load_model(model: nn.Module, model_name_or_path: str): packed_modules_mapping = getattr(model, "packed_modules_mapping", {}) - for file in glob(os.path.join(path, "*.safetensors")): + is_path = os.path.isdir(model_name_or_path) + path = ( + model_name_or_path + if is_path + else download_weights_from_hf(model_name_or_path, None, ["*.safetensors"]) + ) + for file in tqdm(glob(os.path.join(path, "*.safetensors"))): + print(f"Loading weights from {file}") with safe_open(file, "pt", "cpu") as f: for weight_name in f.keys(): for k in packed_modules_mapping: diff --git a/atom/model_loader/weight_utils.py b/atom/model_loader/weight_utils.py new file mode 100644 index 0000000000..cdb0ce34be --- /dev/null +++ b/atom/model_loader/weight_utils.py @@ -0,0 +1,115 @@ +# This code is adapted from https://github.com/ROCm/vllm/blob/main/vllm/model_executor/model_loader/weight_utils.py + +from huggingface_hub import HfFileSystem, hf_hub_download, snapshot_download +import huggingface_hub.constants +import logging +from tqdm.auto import tqdm +import os +import time +import filelock +import tempfile +import hashlib +import fnmatch +from pathlib import Path +from typing import Any, Callable, Optional, Union + +logger = logging.getLogger(__name__) + +# use system-level temp directory for file locks, so that multiple users +# can share the same lock without error. +# lock files in the temp directory will be automatically deleted when the +# system reboots, so users will not complain about annoying lock files +temp_dir = tempfile.gettempdir() + + +def enable_hf_transfer(): + """automatically activates hf_transfer + """ + if "HF_HUB_ENABLE_HF_TRANSFER" not in os.environ: + try: + # enable hf hub transfer if available + import hf_transfer # type: ignore # noqa + huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER = True + except ImportError: + pass + + +enable_hf_transfer() + + +class DisabledTqdm(tqdm): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, disable=True) + +def get_lock(model_name_or_path: Union[str, Path], + cache_dir: Optional[str] = None): + lock_dir = cache_dir or temp_dir + model_name_or_path = str(model_name_or_path) + os.makedirs(os.path.dirname(lock_dir), exist_ok=True) + model_name = model_name_or_path.replace("/", "-") + hash_name = hashlib.sha256(model_name.encode()).hexdigest() + # add hash to avoid conflict with old users' lock files + lock_file_name = hash_name + model_name + ".lock" + # mode 0o666 is required for the filelock to be shared across users + lock = filelock.FileLock(os.path.join(lock_dir, lock_file_name), + mode=0o666) + return lock + + +def download_weights_from_hf( + model_name_or_path: str, + cache_dir: Optional[str], + allow_patterns: list[str], + revision: Optional[str] = None, + ignore_patterns: Optional[Union[str, list[str]]] = None, +) -> str: + """Download model weights from Hugging Face Hub. + + Args: + model_name_or_path (str): The model name or path. + cache_dir (Optional[str]): The cache directory to store the model + weights. If None, will use HF defaults. + allow_patterns (list[str]): The allowed patterns for the + weight files. Files matched by any of the patterns will be + downloaded. + revision (Optional[str]): The revision of the model. + ignore_patterns (Optional[Union[str, list[str]]]): The patterns to + filter out the weight files. Files matched by any of the patterns + will be ignored. + + Returns: + str: The path to the downloaded model weights. + """ + local_only = huggingface_hub.constants.HF_HUB_OFFLINE + if not local_only: + # Before we download we look at that is available: + fs = HfFileSystem() + file_list = fs.ls(model_name_or_path, detail=False, revision=revision) + + # depending on what is available we download different things + for pattern in allow_patterns: + matching = fnmatch.filter(file_list, pattern) + if len(matching) > 0: + allow_patterns = [pattern] + break + + logger.info("Using model weights format %s", allow_patterns) + # Use file lock to prevent multiple processes from + # downloading the same model weights at the same time. + with get_lock(model_name_or_path, cache_dir): + start_time = time.perf_counter() + hf_folder = snapshot_download( + model_name_or_path, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + cache_dir=cache_dir, + tqdm_class=DisabledTqdm, + revision=revision, + local_files_only=local_only, + ) + time_taken = time.perf_counter() - start_time + if time_taken > 0.5: + logger.info("Time spent downloading weights for %s: %.6f seconds", + model_name_or_path, time_taken) + return hf_folder \ No newline at end of file diff --git a/atom/models/llama.py b/atom/models/llama.py new file mode 100644 index 0000000000..65180ed767 --- /dev/null +++ b/atom/models/llama.py @@ -0,0 +1,640 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only LLaMA model compatible with HuggingFace weights.""" +from collections.abc import Iterable +from typing import Any, Optional, Union + +import torch +from torch import nn +from transformers import LlamaConfig + +from vllm.attention import Attention, AttentionType +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from .interfaces import SupportsEagle3, SupportsLoRA, SupportsPP +from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index, + is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers, + maybe_prefix) + + +class LlamaMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + reduce_results: bool = True, + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=hidden_size, + output_sizes=[intermediate_size] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + input_size=intermediate_size, + output_size=hidden_size, + bias=bias, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + x, _ = self.gate_up_proj(x) + x = self.act_fn(x) + x, _ = self.down_proj(x) + return x + + +class LlamaAttention(nn.Module): + + def __init__( + self, + config: LlamaConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[dict[str, Any]] = None, + max_position_embeddings: int = 8192, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + bias_o_proj: bool = False, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + attn_type: str = AttentionType.DECODER, + ) -> None: + super().__init__() + layer_idx = extract_layer_index(prefix) + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MistralConfig has an optional head_dim introduced by Mistral-Nemo + head_dim = getattr(config, "head_dim", None) + if head_dim is None: + head_dim = self.hidden_size // self.total_num_heads + self.head_dim = head_dim + # Phi models introduced a partial_rotary_factor parameter in the config + self.partial_rotary_factor = getattr(config, "partial_rotary_factor", + 1) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size=hidden_size, + head_size=self.head_dim, + total_num_heads=self.total_num_heads, + total_num_kv_heads=self.total_num_kv_heads, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.o_proj = RowParallelLinear( + input_size=self.total_num_heads * self.head_dim, + output_size=hidden_size, + bias=bias_o_proj, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self._init_rotary_emb(config, + rope_scaling=rope_scaling, + quant_config=quant_config) + + sliding_window = None + if layer_types := getattr(config, "layer_types", None): + is_sliding = layer_types[layer_idx] == "sliding_attention" + if is_sliding: + sliding_window = config.sliding_window + + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + per_layer_sliding_window=sliding_window, + attn_type=attn_type, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + def _init_rotary_emb(self, config: LlamaConfig, + rope_scaling: Optional[dict[str, Any]], + quant_config: Optional[QuantizationConfig]) -> None: + is_neox_style = True + is_gguf = quant_config and quant_config.get_name() == "gguf" + if is_gguf and config.model_type == "llama": + is_neox_style = False + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=self.max_position_embeddings, + base=self.rope_theta, + rope_scaling=rope_scaling, + is_neox_style=is_neox_style, + partial_rotary_factor=self.partial_rotary_factor, + ) + + +class LlamaDecoderLayer(nn.Module): + + def __init__( + self, + config: LlamaConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + if rope_scaling is not None and getattr( + config, "original_max_position_embeddings", None): + rope_scaling["original_max_position_embeddings"] = ( + config.original_max_position_embeddings) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # Support abacusai/Smaug-72B-v0.1 with attention_bias + # Support internlm/internlm-7b with bias + attention_bias = getattr(config, "attention_bias", False) or getattr( + config, "bias", False) + bias_o_proj = attention_bias + # support internlm/internlm3-8b with qkv_bias + if hasattr(config, 'qkv_bias'): + attention_bias = config.qkv_bias + + # By default, Llama uses causal attention as it is a decoder-only model. + # You can override the HF config with `is_causal=False` to enable + # bidirectional attention, which is used in some embedding models + # (e.g. parasail-ai/GritLM-7B-vllm) + if getattr(config, "is_causal", True): + attn_type = AttentionType.DECODER + else: + attn_type = AttentionType.ENCODER_ONLY + + self.self_attn = LlamaAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=getattr(config, "num_key_value_heads", + config.num_attention_heads), + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + quant_config=quant_config, + bias=attention_bias, + bias_o_proj=bias_o_proj, + cache_config=cache_config, + prefix=f"{prefix}.self_attn", + attn_type=attn_type, + ) + self.mlp = LlamaMLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + bias=getattr(config, "mlp_bias", False), + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn(positions=positions, + hidden_states=hidden_states) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +@support_torch_compile +class LlamaModel(nn.Module): + + def __init__(self, + *, + vllm_config: VllmConfig, + prefix: str = "", + layer_type: type[nn.Module] = LlamaDecoderLayer): + super().__init__() + + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + lora_config = vllm_config.lora_config + + self.config = config + self.quant_config = quant_config + lora_vocab = (lora_config.lora_extra_vocab_size * + (lora_config.max_loras or 1)) if lora_config else 0 + self.vocab_size = config.vocab_size + lora_vocab + self.org_vocab_size = config.vocab_size + if get_pp_group().is_first_rank or (config.tie_word_embeddings + and get_pp_group().is_last_rank): + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + quant_config=quant_config, + ) + else: + self.embed_tokens = PPMissingLayer() + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: layer_type(config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix), + prefix=f"{prefix}.layers", + ) + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + + self.aux_hidden_state_layers: tuple[int] = tuple() + + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: Optional[torch.Tensor], + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors, tuple[torch.Tensor, + list[torch.Tensor]]]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + aux_hidden_states = [] + for idx, layer in enumerate( + self.layers[self.start_layer:self.end_layer]): + if idx in self.aux_hidden_state_layers: + aux_hidden_states.append(hidden_states + residual) + hidden_states, residual = layer(positions, hidden_states, residual) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + + if len(aux_hidden_states) > 0: + return hidden_states, aux_hidden_states + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + if ("rotary_emb.cos_cached" in name + or "rotary_emb.sin_cached" in name): + # Models trained using ColossalAI may include these tensors in + # the checkpoint. Skip them. + continue + if (self.quant_config is not None and + (scale_name := self.quant_config.get_cache_scale(name))): + # Loading kv cache quantization scales + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else + loaded_weight[0]) + weight_loader(param, loaded_weight) + loaded_params.add(scale_name) + continue + if "scale" in name: + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"] + } + + # LoRA specific attributes + embedding_modules = { + "embed_tokens": "input_embeddings", + "lm_head": "output_embeddings" + } + embedding_padding_modules = ["lm_head"] + + # Mistral/Llama models can also be loaded with --load-format mistral + # from consolidated.safetensors checkpoints + mistral_mapping = { + "layers": "model.layers", + "attention": "self_attn", + "qscale_act": "input_scale", + "qscale_weight": "weight_scale", + "kv_fake_quantizer.qscale_act": "kv_scale", + "q_fake_quantizer.qscale_act": "attn.q_scale", + "k_fake_quantizer.qscale_act": "k_scale", + "v_fake_quantizer.qscale_act": "v_scale", + "wq": "q_proj", + "wk": "k_proj", + "wv": "v_proj", + "wo": "o_proj", + "attention_norm": "input_layernorm", + "feed_forward": "mlp", + "w1": "gate_proj", + "w2": "down_proj", + "w3": "up_proj", + "ffn_norm": "post_attention_layernorm", + "tok_embeddings": "model.embed_tokens", + "output": "lm_head", + "norm": "model.norm", + } + + def __init__(self, + *, + vllm_config: VllmConfig, + prefix: str = "", + layer_type: type[nn.Module] = LlamaDecoderLayer): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + lora_config = vllm_config.lora_config + self.config = config + self.lora_config = lora_config + + self.model = self._init_model(vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model"), + layer_type=layer_type) + + if get_pp_group().is_last_rank: + self.unpadded_vocab_size = config.vocab_size + if lora_config: + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + padding_size=( + DEFAULT_VOCAB_PADDING_SIZE + # We need bigger padding if using lora for kernel + # compatibility + if not lora_config else + lora_config.lora_vocab_padding_size), + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights( + self.model.embed_tokens) + + logit_scale = getattr(config, "logit_scale", 1.0) + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, + config.vocab_size, + logit_scale) + else: + self.lm_head = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def set_aux_hidden_state_layers(self, layers: tuple[int]) -> None: + self.model.aux_hidden_state_layers = layers + + def get_eagle3_aux_hidden_state_layers(self) -> tuple[int]: + num_layers = len(self.model.layers) + return (2, num_layers // 2, num_layers - 3) + + def _init_model(self, + vllm_config: VllmConfig, + prefix: str = "", + layer_type: type[nn.Module] = LlamaDecoderLayer): + return LlamaModel(vllm_config=vllm_config, + prefix=prefix, + layer_type=layer_type) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + model_output = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + return model_output + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + return loader.load_weights( + self.maybe_remap_mistral(name, loaded_weight) + for name, loaded_weight in weights) + + # This function is used to remap the mistral format as + # used by Mistral and Llama <=2 + def maybe_remap_mistral( + self, + name: str, + loaded_weight: torch.Tensor, + ) -> tuple[str, torch.Tensor]: + + def permute(w: torch.Tensor, n_heads: int): + attn_in = self.config.head_dim * n_heads + attn_out = self.config.hidden_size + + return w.view(n_heads, attn_in // n_heads // 2, 2, + attn_out).transpose(1, 2).reshape(attn_in, attn_out) + + mapping = self.mistral_mapping + modules = name.split(".") + + # rotary embeds should be sliced + if "wk" in modules and modules[-1] == "weight": + loaded_weight = permute(loaded_weight, + self.config.num_key_value_heads) + elif "wq" in modules and modules[-1] == "weight": + loaded_weight = permute(loaded_weight, + self.config.num_attention_heads) + + num_modules = len(modules) + for i in range(num_modules): + item = modules[i] + next_item = modules[i + 1] if i < num_modules - 1 else None + + combined_item = (f"{item}.{next_item}" + if next_item is not None else None) + + if combined_item in mapping: + name = name.replace(combined_item, mapping[combined_item]) + elif item in mapping and mapping[item] not in name: + name = name.replace(item, mapping[item]) + + return name, loaded_weight \ No newline at end of file diff --git a/example.py b/example.py index 9817c31442..e862134e01 100644 --- a/example.py +++ b/example.py @@ -17,29 +17,37 @@ e.g.: -kv_cache_dtype fp8""", ) +parser.add_argument( + "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." +) + def main(): args = parser.parse_args() - path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") - tokenizer = AutoTokenizer.from_pretrained(path) - llm = LLMEngine(path, enforce_eager=False, tensor_parallel_size=1, kv_cache_dtype=args.kv_cache_dtype) + model_name_or_path = args.model + tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + llm = LLMEngine( + model_name_or_path, + enforce_eager=False, + tensor_parallel_size=1, + kv_cache_dtype=args.kv_cache_dtype, + ) sampling_params = SamplingParams(temperature=0.6, max_tokens=256) prompts = [ "introduce yourself", "list all prime numbers within 100", - ] prompts = [ tokenizer.apply_chat_template( [{"role": "user", "content": prompt}], tokenize=False, add_generation_prompt=True, - enable_thinking=True + enable_thinking=True, ) for prompt in prompts ] - print('This is prompts:', prompts) + print("This is prompts:", prompts) outputs = llm.generate(prompts, sampling_params) for prompt, output in zip(prompts, outputs): @@ -49,4 +57,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 3b7c16da5c7d3da63062c528c226d6dbdd11c519 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 20 Aug 2025 16:11:23 +0800 Subject: [PATCH 011/119] support rotary aiter --- atom/model_ops/rotary_embedding.py | 100 ++++++++++++++++++++++++----- 1 file changed, 84 insertions(+), 16 deletions(-) diff --git a/atom/model_ops/rotary_embedding.py b/atom/model_ops/rotary_embedding.py index c473420f56..c52afbb419 100644 --- a/atom/model_ops/rotary_embedding.py +++ b/atom/model_ops/rotary_embedding.py @@ -1,7 +1,9 @@ from functools import lru_cache import torch from torch import nn - +import aiter +from aiter import dtypes +from typing import Union, Optional def apply_rotary_emb( x: torch.Tensor, @@ -24,34 +26,100 @@ def __init__( rotary_dim: int, max_position_embeddings: int, base: float, + is_neox_style: bool = True, + dtype: Optional[torch.dtype] = None, ) -> None: super().__init__() + if dtype is None: + dtype = torch.get_default_dtype() self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cos, sin = self._compute_cos_sin_cache() + cos = cos.to(dtype) + sin = sin.to(dtype) + self.cos_cache: torch.Tensor + self.sin_cache: torch.Tensor + self.register_buffer("cos_cache", cos, persistent=False) + self.register_buffer("sin_cache", sin, persistent=False) + assert rotary_dim == head_size - inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2, dtype=torch.float) / rotary_dim)) - t = torch.arange(max_position_embeddings, dtype=torch.float) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + """Compute the inverse frequency.""" + # NOTE(woosuk): To exactly match the HF implementation, we need to + # use CPU to compute the cache and then move it to GPU. However, we + # create the cache on GPU for faster initialization. This may cause + # a slight numerical difference between the HF implementation and ours. + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=dtypes.fp32) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=dtypes.fp32) + freqs = torch.einsum("i,j -> ij", t, inv_freq) - cos = freqs.cos() - sin = freqs.sin() - cache = torch.cat((cos, sin), dim=-1) - self.register_buffer("cos_sin_cache", cache, persistent=False) + cos = freqs.cos().unsqueeze(-2).unsqueeze(-2) + sin = freqs.sin().unsqueeze(-2).unsqueeze(-2) + return cos, sin - @torch.compile + + @torch.compile() def forward( self, positions: torch.Tensor, query: torch.Tensor, key: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - num_tokens = positions.size(0) - cos_sin = self.cos_sin_cache[positions] - cos, sin = cos_sin.chunk(2, dim=-1) + + is_nope_first = False + self.cos_cache = self.cos_cache.to(query.device, dtype=query.dtype) + self.sin_cache = self.sin_cache.to(query.device, dtype=query.dtype) + cos, sin = self.cos_cache, self.sin_cache + + rotate_style = 0 if self.is_neox_style else 1 + + num_tokens = positions.numel() + query_shape = query.shape - query = query.view(num_tokens, -1, self.head_size) - query = apply_rotary_emb(query, cos, sin).view(query_shape) - key_shape = key.shape - key = key.view(num_tokens, -1, self.head_size) - key = apply_rotary_emb(key, cos, sin).view(key_shape) + query = query.view(1, num_tokens, -1, self.head_size) + if key is not None: + key_shape = key.shape + key = key.view(1, num_tokens, -1, self.head_size) + + positions = positions.view(*query.shape[:2]) + + if not is_nope_first: + query_ = query[..., : self.rotary_dim] + key_ = key[..., : self.rotary_dim] if key is not None else None + else: + query_ = query[..., -self.rotary_dim :] + key_ = key[..., -self.rotary_dim :] if key is not None else None + + aiter.rope_cached_positions_2c_fwd_inplace( + query_, + key_, + cos, + sin, + positions, + rotate_style, + reuse_freqs_front_part=True, + nope_first=is_nope_first, + ) + query = query.view(query_shape) + + key = key.view(key_shape) + return query, key From 7d016840e2e38093130f006d2ba2cadee3097478 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 20 Aug 2025 21:25:52 +0800 Subject: [PATCH 012/119] use aiter get_rope --- atom/models/qwen3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 6d7306f521..dc83f5f180 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -11,7 +11,8 @@ MergedColumnParallelLinear, RowParallelLinear, ) -from atom.model_ops.rotary_embedding import get_rope +# from atom.model_ops.rotary_embedding import get_rope +from aiter.rotary_embedding import get_rope from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead From 225b83d36320f59f712918af1ac954f0e12388b8 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Thu, 21 Aug 2025 02:31:21 +0000 Subject: [PATCH 013/119] minor refine --- atom/model_engine/model_runner.py | 18 +++++++++++------- atom/model_loader/loader.py | 6 ++++-- atom/model_ops/activation.py | 1 - atom/model_ops/layernorm.py | 2 -- atom/model_ops/rotary_embedding.py | 2 -- example.py | 2 ++ 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index bfc3d2d3df..970e736e3d 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -4,7 +4,6 @@ from multiprocessing.synchronize import Event from multiprocessing.shared_memory import SharedMemory from aiter import dtypes - from atom.config import Config from atom.model_engine.sequence import Sequence from atom.model_ops.sampler import Sampler @@ -32,6 +31,7 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") self.model = Qwen3ForCausalLM(hf_config, config.kv_cache_dtype) + # self.model = torch.compile(self.model, fullgraph=True) load_model(self.model, config.model) self.sampler = Sampler() self.warmup_model() @@ -128,7 +128,7 @@ def allocate_kv_cache(self): int(total * config.gpu_memory_utilization - used - peak + current) // block_bytes ) - assert config.num_kvcache_blocks > 0 + assert config.num_kvcache_blocks > 0, f"need at least {block_bytes} KV cache" self.kv_cache = torch.zeros( 2, hf_config.num_hidden_layers, @@ -136,8 +136,8 @@ def allocate_kv_cache(self): self.block_size, num_kv_heads, hf_config.head_dim, - dtype=dtypes.fp8 if config.kv_cache_dtype == "fp8" else dtypes.bf16, - device="cuda" + dtype=dtypes.d_dtypes[config.kv_cache_dtype], + device="cuda", ) self.kv_scale = torch.zeros( @@ -147,7 +147,7 @@ def allocate_kv_cache(self): self.block_size, num_kv_heads, dtype=dtypes.fp32, - device="cuda" + device="cuda", ) layer_id = 0 @@ -174,7 +174,6 @@ def allocate_kv_cache(self): layer_id += 1 - def prepare_block_tables(self, seqs: list[Sequence]): max_len = max(len(seq.block_table) for seq in seqs) block_tables = [ @@ -373,7 +372,12 @@ def capture_cudagraph(self): for bs in reversed(self.graph_bs): graph = torch.cuda.CUDAGraph() - set_context(False, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs]) + set_context( + False, + slot_mapping=slot_mapping[:bs], + context_lens=context_lens[:bs], + block_tables=block_tables[:bs], + ) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup with torch.cuda.graph(graph, self.graph_pool): diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index 5b66b2b470..f492305d74 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -3,7 +3,7 @@ import torch from torch import nn from safetensors import safe_open -from .weight_utils import download_weights_from_hf +from atom.model_loader.weight_utils import download_weights_from_hf from tqdm import tqdm @@ -17,7 +17,9 @@ def load_model(model: nn.Module, model_name_or_path: str): path = ( model_name_or_path if is_path - else download_weights_from_hf(model_name_or_path, None, ["*.safetensors"]) + else download_weights_from_hf( + model_name_or_path, None, ["*.safetensors"], ignore_patterns=["original/*"] + ) ) for file in tqdm(glob(os.path.join(path, "*.safetensors"))): print(f"Loading weights from {file}") diff --git a/atom/model_ops/activation.py b/atom/model_ops/activation.py index d46f5299f6..ea48864e27 100644 --- a/atom/model_ops/activation.py +++ b/atom/model_ops/activation.py @@ -9,7 +9,6 @@ class SiluAndMul(nn.Module): def __init__(self): super().__init__() - @torch.compile def forward_native(self, x: torch.Tensor) -> torch.Tensor: x, y = x.chunk(2, -1) return F.silu(x) * y diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py index a2ce730c32..00c68275a1 100644 --- a/atom/model_ops/layernorm.py +++ b/atom/model_ops/layernorm.py @@ -15,7 +15,6 @@ def __init__( self.eps = eps self.weight = nn.Parameter(torch.ones(hidden_size)) - @torch.compile def rms_forward( self, x: torch.Tensor, @@ -27,7 +26,6 @@ def rms_forward( x = x.to(orig_dtype).mul_(self.weight) return x - @torch.compile def add_rms_forward( self, x: torch.Tensor, diff --git a/atom/model_ops/rotary_embedding.py b/atom/model_ops/rotary_embedding.py index c52afbb419..869c66f477 100644 --- a/atom/model_ops/rotary_embedding.py +++ b/atom/model_ops/rotary_embedding.py @@ -73,8 +73,6 @@ def _compute_cos_sin_cache(self) -> torch.Tensor: sin = freqs.sin().unsqueeze(-2).unsqueeze(-2) return cos, sin - - @torch.compile() def forward( self, positions: torch.Tensor, diff --git a/example.py b/example.py index e862134e01..606c13ad35 100644 --- a/example.py +++ b/example.py @@ -37,6 +37,8 @@ def main(): prompts = [ "introduce yourself", "list all prime numbers within 100", + "1+2+3=?", + # "2+3+4=?", ] prompts = [ tokenizer.apply_chat_template( From a17d4be75eb01075459e77b5bedbeda27579703c Mon Sep 17 00:00:00 2001 From: chenjun Date: Fri, 22 Aug 2025 03:41:39 +0000 Subject: [PATCH 014/119] enable TP --- atom/config.py | 1 + atom/model_engine/model_runner.py | 15 ++- atom/model_engine/sequence.py | 1 + atom/model_ops/embed_head.py | 17 ++- atom/model_ops/linear.py | 212 +++++++++++++++++++----------- atom/models/qwen3.py | 25 ++-- example.py | 13 +- 7 files changed, 181 insertions(+), 103 deletions(-) diff --git a/atom/config.py b/atom/config.py index 3adec52c57..6675ea9d3d 100644 --- a/atom/config.py +++ b/atom/config.py @@ -17,6 +17,7 @@ class Config: kvcache_block_size: int = 16 num_kvcache_blocks: int = -1 kv_cache_dtype: str = "bf16" + port: int = 8006 def __post_init__(self): # assert os.path.isdir(self.model) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 970e736e3d..7ac0b465a3 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -1,6 +1,9 @@ import pickle +import os import torch import torch.distributed as dist +from aiter import init_dist_env +from aiter.dist.parallel_state import destroy_distributed_environment from multiprocessing.synchronize import Event from multiprocessing.shared_memory import SharedMemory from aiter import dtypes @@ -23,10 +26,11 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.rank = rank self.event = event - dist.init_process_group( - "nccl", "tcp://localhost:2333", world_size=self.world_size, rank=rank - ) - torch.cuda.set_device(rank) + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = str(self.config.port) + init_dist_env(self.world_size, rankID=rank) default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") @@ -59,7 +63,8 @@ def exit(self): if not self.enforce_eager: del self.graphs, self.graph_pool torch.cuda.synchronize() - dist.destroy_process_group() + # dist.destroy_process_group() + destroy_distributed_environment() def loop(self): while True: diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 644fb6f2ec..dabf8e9103 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -13,6 +13,7 @@ class SequenceStatus(Enum): class Sequence: counter = count() + block_size = 16 def __init__( self, token_ids: list[int], block_size: int, sampling_params=SamplingParams() diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 24a6f3e4f7..a8600d86dd 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -2,6 +2,7 @@ from torch import nn import torch.nn.functional as F import torch.distributed as dist +from aiter.dist.parallel_state import get_tp_group from atom.utils.context import get_context @@ -14,14 +15,16 @@ def __init__( embedding_dim: int, ): super().__init__() - self.tp_rank = dist.get_rank() - self.tp_size = dist.get_world_size() + self.tp_rank = get_tp_group().rank + self.tp_size = get_tp_group().world_size assert num_embeddings % self.tp_size == 0 self.num_embeddings = num_embeddings self.num_embeddings_per_partition = self.num_embeddings // self.tp_size self.vocab_start_idx = self.num_embeddings_per_partition * self.tp_rank self.vocab_end_idx = self.vocab_start_idx + self.num_embeddings_per_partition - self.weight = nn.Parameter(torch.empty(self.num_embeddings_per_partition, embedding_dim)) + self.weight = nn.Parameter( + torch.empty(self.num_embeddings_per_partition, embedding_dim) + ) self.weight.weight_loader = self.weight_loader def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): @@ -39,7 +42,7 @@ def forward(self, x: torch.Tensor): y = F.embedding(x, self.weight) if self.tp_size > 1: y = mask.unsqueeze(1) * y - dist.all_reduce(y) + y = get_tp_group().all_reduce(y, open_fp8_quant=False) return y @@ -65,7 +68,11 @@ def forward(self, x: torch.Tensor): x = x[last_indices].contiguous() logits = F.linear(x, self.weight, self.bias) if self.tp_size > 1: - all_logits = [torch.empty_like(logits) for _ in range(self.tp_size)] if self.tp_rank == 0 else None + all_logits = ( + [torch.empty_like(logits) for _ in range(self.tp_size)] + if self.tp_rank == 0 + else None + ) dist.gather(logits, all_logits, 0) logits = torch.cat(all_logits, -1) if self.tp_rank == 0 else None return logits diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 8f854692e2..5d252e047b 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -1,8 +1,16 @@ import torch from torch import nn import torch.nn.functional as F -import torch.distributed as dist -from aiter import QuantType, dtypes, gemm_a8w8_bpreshuffle, gemm_a8w8_blockscale, gemm_a4w4 + +# import torch.distributed as dist +from aiter.dist.parallel_state import get_tp_group +from aiter import ( + QuantType, + dtypes, + gemm_a8w8_bpreshuffle, + gemm_a8w8_blockscale, + gemm_a4w4, +) from aiter.ops.shuffle import shuffle_weight from aiter.tuned_gemm import tgemm from typing import Optional, Callable @@ -22,57 +30,112 @@ def __init__( output_size: int, tp_dim: int | None = None, bias: bool = False, - params_dtype = dtypes.bf16, - quant_type = None, + params_dtype=dtypes.bf16, + quant_type=QuantType.No, ): super().__init__() self.input_size = input_size self.output_size = output_size self.tp_dim = tp_dim - self.tp_rank = dist.get_rank() - self.tp_size = dist.get_world_size() - self.weight = nn.Parameter(torch.empty((self.output_size, self.input_size), dtype=params_dtype)) + self.tp_rank = get_tp_group().rank + self.tp_size = get_tp_group().world_size + if tp_dim == 1: + self.input_size = divide(input_size, self.tp_size) + elif tp_dim == 0: + self.output_size = divide(output_size, self.tp_size) + self.weight = nn.Parameter( + torch.empty((self.output_size, self.input_size), dtype=params_dtype) + ) if bias: self.bias = nn.Parameter(torch.empty(self.output_size, dtype=params_dtype)) self.bias.weight_loader_process = self.weight_loader_process else: self.register_parameter("bias", None) self.quant_type = quant_type - if quant_type is not None: + if quant_type != QuantType.No: if quant_type == QuantType.per_Tensor: - self.weight.weight_loader_process = functools_partial(self.weight_loader_process, post_process_func=shuffle_weight(16, 16)) + self.weight.weight_loader_process = functools_partial( + self.weight_loader_process, post_process_func=shuffle_weight(16, 16) + ) self.weight_scale = nn.Parameter(torch.empty(1, dtype=dtypes.fp32)) elif quant_type == QuantType.per_Token: self.weight.weight_loader_process = self.weight_loader_process - self.weight_scale = nn.Parameter(torch.empty(self.output_size, dtype=dtypes.fp32)) + self.weight_scale = nn.Parameter( + torch.empty(self.output_size, dtype=dtypes.fp32) + ) elif quant_type == QuantType.per_128x128: self.weight.weight_loader_process = self.weight_loader_process - self.weight_scale = nn.Parameter(torch.empty(divide(self.output_size, 128), (self.input_size + 127) // 128, dtype=dtypes.fp32)) + self.weight_scale = nn.Parameter( + torch.empty( + divide(self.output_size, 128), + (self.input_size + 127) // 128, + dtype=dtypes.fp32, + ) + ) elif quant_type == QuantType.per_1x32: - self.weight.weight_loader_process = functools_partial(self.weight_loader_process, post_process_func=shuffle_weight(16, 16)) - self.weight_scale = nn.Parameter(torch.empty(self.output_size, (self.input_size + 31) // 32, dtype=dtypes.fp8_e8m0)) + self.weight.weight_loader_process = functools_partial( + self.weight_loader_process, post_process_func=shuffle_weight(16, 16) + ) + self.weight_scale = nn.Parameter( + torch.empty( + self.output_size, + (self.input_size + 31) // 32, + dtype=dtypes.fp8_e8m0, + ) + ) self.weight_scale.weight_loader_process = self.weight_loader_process else: self.weight.weight_loader_process = self.weight_loader_process self.register_parameter("weight_scale", None) + self.weight.weight_loader = self.weight_loader + if self.bias is not None: + self.bias.weight_loader = self.weight_loader + if self.weight_scale is not None: + self.weight_scale.weight_loader = self.weight_loader @staticmethod - def weight_loader_process(param: nn.Parameter, loaded_weight: torch.Tensor, post_process_func: Callable=lambda a : a): + def weight_loader_process( + param: nn.Parameter, + loaded_weight: torch.Tensor, + post_process_func: Callable = lambda a: a, + ): param.data.copy_(post_process_func(loaded_weight)) - def forward(self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16) -> torch.Tensor: - if self.quant_type is None: - return tgemm.mm(x, self.weight, self.bias) + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + param.weight_loader_process(param_data, loaded_weight) + + def forward( + self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 + ) -> torch.Tensor: + if self.quant_type == QuantType.No: + y = tgemm.mm(x, self.weight, self.bias) else: assert x_scale is not None if self.quant_type == QuantType.per_Tensor: - return tgemm.mm(x, self.weight, self.bias, otype=otype, scale_a=x_scale, scale_b=self.weight_scale) + y = tgemm.mm( + x, + self.weight, + self.bias, + otype=otype, + scale_a=x_scale, + scale_b=self.weight_scale, + ) elif self.quant_type == QuantType.per_Token: - return gemm_a8w8_bpreshuffle(x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype) + y = gemm_a8w8_bpreshuffle( + x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype + ) elif self.quant_type == QuantType.per_128x128: - return gemm_a8w8_blockscale(x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype) + y = gemm_a8w8_blockscale( + x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype + ) elif self.quant_type == QuantType.per_1x32: - return gemm_a4w4(x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype) + y = gemm_a4w4( + x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype + ) + if self.tp_dim == 1 and self.tp_size > 1: + y = get_tp_group().all_reduce(y, open_fp8_quant=False) + return y class ReplicatedLinear(LinearBase): @@ -82,21 +145,21 @@ def __init__( input_size: int, output_size: int, bias: bool = False, + params_dtype=dtypes.bf16, + quant_type=QuantType.No, ): - super().__init__(input_size, output_size) - self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) + super().__init__( + input_size, + output_size, + tp_dim=None, + bias=bias, + params_dtype=params_dtype, + quant_type=quant_type, + ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): - param.data.copy_(loaded_weight) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return F.linear(x, self.weight, self.bias) + param_data = param.data + param.weight_loader_process(param_data, loaded_weight) class ColumnParallelLinear(LinearBase): @@ -106,31 +169,20 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - params_dtype = dtypes.bf16, - quant_type = None, + params_dtype=dtypes.bf16, + quant_type=QuantType.No, ): self.tp_dim = 0 - self.tp_rank = dist.get_rank() - self.tp_size = dist.get_world_size() - self.input_size_per_partition = input_size - self.output_size_per_partition = divide(output_size, self.tp_size) - super().__init__(input_size, output_size, self.tp_dim, bias, params_dtype, quant_type) - - self.weight = nn.Parameter(torch.empty(self.output_size_per_partition, self.input_size_per_partition)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size_per_partition)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) - + super().__init__( + input_size, output_size, self.tp_dim, bias, params_dtype, quant_type + ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param_data.weight_loader_process(param_data, loaded_weight) + param.weight_loader_process(param_data, loaded_weight) class MergedColumnParallelLinear(ColumnParallelLinear): @@ -140,17 +192,21 @@ def __init__( input_size: int, output_sizes: list[int], bias: bool = False, + params_dtype=dtypes.bf16, + quant_type=QuantType.No, ): self.output_sizes = output_sizes - super().__init__(input_size, sum(output_sizes), bias=bias) + super().__init__(input_size, sum(output_sizes), bias, params_dtype, quant_type) - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int): + def weight_loader( + self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int + ): param_data = param.data shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size shard_size = self.output_sizes[loaded_shard_id] // self.tp_size param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] - param_data.copy_(loaded_weight) + param.weight_loader_process(param_data, loaded_weight) class QKVParallelLinear(ColumnParallelLinear): @@ -162,18 +218,24 @@ def __init__( total_num_heads: int, total_num_kv_heads: int | None = None, bias: bool = False, + params_dtype=dtypes.bf16, + quant_type=QuantType.No, ): self.head_size = head_size self.total_num_heads = total_num_heads self.total_num_kv_heads = total_num_kv_heads or total_num_heads - tp_size = dist.get_world_size() + tp_size = get_tp_group().world_size self.num_heads = divide(self.total_num_heads, tp_size) self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) input_size = hidden_size - output_size = (self.total_num_heads + 2 * self.total_num_kv_heads) * self.head_size - super().__init__(input_size, output_size, bias) + output_size = ( + self.total_num_heads + 2 * self.total_num_kv_heads + ) * self.head_size + super().__init__(input_size, output_size, bias, params_dtype, quant_type) - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str): + def weight_loader( + self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str + ): param_data = param.data assert loaded_shard_id in ["q", "k", "v"] if loaded_shard_id == "q": @@ -184,10 +246,12 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded shard_offset = self.num_heads * self.head_size else: shard_size = self.num_kv_heads * self.head_size - shard_offset = self.num_heads * self.head_size + self.num_kv_heads * self.head_size + shard_offset = ( + self.num_heads * self.head_size + self.num_kv_heads * self.head_size + ) param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] - param_data.copy_(loaded_weight) + param.weight_loader_process(param_data, loaded_weight) class RowParallelLinear(LinearBase): @@ -197,28 +261,22 @@ def __init__( input_size: int, output_size: int, bias: bool = False, + params_dtype=dtypes.bf16, + quant_type=QuantType.No, ): - super().__init__(input_size, output_size, 1) - self.input_size_per_partition = divide(input_size, 1) - self.output_size_per_partition = output_size - - self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size_per_partition)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) + self.tp_rank = get_tp_group().rank + super().__init__( + input_size, + output_size, + tp_dim=1, + bias=bias if self.tp_rank == 0 else False, + params_dtype=params_dtype, + quant_type=quant_type, + ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param_data.copy_(loaded_weight) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None) - if self.tp_size > 1: - dist.all_reduce(y) - return y + param.weight_loader_process(param_data, loaded_weight) diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index dc83f5f180..a74f3efbff 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -1,6 +1,8 @@ import torch from torch import nn -import torch.distributed as dist + +# import torch.distributed as dist +from aiter.dist.parallel_state import get_tp_group from transformers import Qwen3Config from atom.model_ops.activation import SiluAndMul @@ -29,10 +31,10 @@ def __init__( qkv_bias: bool = False, rope_theta: float = 10000, rope_scaling: tuple | None = None, - kv_cache_dtype: str = "fp16" + kv_cache_dtype: str = "fp16", ) -> None: super().__init__() - tp_size = dist.get_world_size() + tp_size = get_tp_group().world_size self.total_num_heads = num_heads assert self.total_num_heads % tp_size == 0 self.num_heads = self.total_num_heads // tp_size @@ -123,11 +125,7 @@ def forward(self, x): class Qwen3DecoderLayer(nn.Module): - def __init__( - self, - config: Qwen3Config, - kv_cache_dtype: str - ) -> None: + def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: super().__init__() self.self_attn = Qwen3Attention( hidden_size=config.hidden_size, @@ -170,17 +168,16 @@ def forward( class Qwen3Model(nn.Module): - def __init__( - self, - config: Qwen3Config, - kv_cache_dtype: str - ) -> None: + def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: super().__init__() self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size ) self.layers = nn.ModuleList( - [Qwen3DecoderLayer(config, kv_cache_dtype) for _ in range(config.num_hidden_layers)] + [ + Qwen3DecoderLayer(config, kv_cache_dtype) + for _ in range(config.num_hidden_layers) + ] ) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/example.py b/example.py index 606c13ad35..85915c5f89 100644 --- a/example.py +++ b/example.py @@ -21,6 +21,14 @@ "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." ) +parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=2) + +parser.add_argument( + "--enforce-eager", action="store_true", help="Enforce eager mode execution." +) + +parser.add_argument("--port", type=int, default=8006, help="API server port") + def main(): args = parser.parse_args() @@ -28,9 +36,10 @@ def main(): tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) llm = LLMEngine( model_name_or_path, - enforce_eager=False, - tensor_parallel_size=1, + enforce_eager=args.enforce_eager, + tensor_parallel_size=args.tensor_parallel_size, kv_cache_dtype=args.kv_cache_dtype, + port=args.port, ) sampling_params = SamplingParams(temperature=0.6, max_tokens=256) From 5a4824a6e739ef84e149216fab42fa82c1c065df Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Fri, 22 Aug 2025 06:32:47 +0000 Subject: [PATCH 015/119] update --- atom/model_engine/model_runner.py | 2 +- atom/model_ops/attention.py | 18 +++++++++++------- atom/model_ops/linear.py | 10 +++++----- atom/models/llama.py | 4 ++-- atom/models/qwen3.py | 2 +- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 7ac0b465a3..3e7fba0747 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -35,7 +35,7 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") self.model = Qwen3ForCausalLM(hf_config, config.kv_cache_dtype) - # self.model = torch.compile(self.model, fullgraph=True) + # self.model = torch.compile(self.model, fullgraph=True, backend="eager") load_model(self.model, config.model) self.sampler = Sampler() self.warmup_model() diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 74ca10d5e3..10928ad07b 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -39,8 +39,15 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): if k_cache.numel() and v_cache.numel() and context.slot_mapping is not None: if self.kv_cache_dtype == "fp8": aiter.reshape_and_cache_with_pertoken_quant( - k, v, k_cache, v_cache, self.k_scale, self.v_scale, context.slot_mapping, asm_layout=True - ) + k, + v, + k_cache, + v_cache, + self.k_scale, + self.v_scale, + context.slot_mapping, + asm_layout=True, + ) else: aiter.reshape_and_cache( k, @@ -54,7 +61,6 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): asm_layout=True, ) - if context.is_prefill: # if context.block_tables is not None: # prefix cache # k, v = k_cache, v_cache @@ -72,10 +78,8 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): causal=True, ) else: # decode - - o = aiter.pa_fwd_asm( - q.contiguous(), + q, k_cache, v_cache, context.block_tables, @@ -84,7 +88,7 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): K_QScale=self.k_scale, V_QScale=self.v_scale, out_=None, - high_precision=0 + high_precision=0, ) # o = PagedAttention.forward_decode( diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 5d252e047b..edbd8dfe89 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -108,11 +108,11 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): def forward( self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 ) -> torch.Tensor: - if self.quant_type == QuantType.No: + if self.quant_type.value == QuantType.No.value: y = tgemm.mm(x, self.weight, self.bias) else: assert x_scale is not None - if self.quant_type == QuantType.per_Tensor: + if self.quant_type.value == QuantType.per_Tensor.value: y = tgemm.mm( x, self.weight, @@ -121,15 +121,15 @@ def forward( scale_a=x_scale, scale_b=self.weight_scale, ) - elif self.quant_type == QuantType.per_Token: + elif self.quant_type.value == QuantType.per_Token.value: y = gemm_a8w8_bpreshuffle( x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype ) - elif self.quant_type == QuantType.per_128x128: + elif self.quant_type.value == QuantType.per_128x128.value: y = gemm_a8w8_blockscale( x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype ) - elif self.quant_type == QuantType.per_1x32: + elif self.quant_type.value == QuantType.per_1x32.value: y = gemm_a4w4( x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype ) diff --git a/atom/models/llama.py b/atom/models/llama.py index 65180ed767..5187d5c014 100644 --- a/atom/models/llama.py +++ b/atom/models/llama.py @@ -191,7 +191,7 @@ def forward( hidden_states: torch.Tensor, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.o_proj(attn_output) @@ -637,4 +637,4 @@ def permute(w: torch.Tensor, n_heads: int): elif item in mapping and mapping[item] not in name: name = name.replace(item, mapping[item]) - return name, loaded_weight \ No newline at end of file + return name, loaded_weight diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index a74f3efbff..60700350a2 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -81,7 +81,7 @@ def forward( hidden_states: torch.Tensor, ) -> torch.Tensor: qkv = self.qkv_proj(hidden_states) - q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) q_by_head = q.view(-1, self.num_heads, self.head_dim) q_by_head = self.q_norm(q_by_head) q = q_by_head.view(q.shape) From b025dcea886898e4415246d494d30f1ae249140e Mon Sep 17 00:00:00 2001 From: chenjun Date: Fri, 22 Aug 2025 06:35:49 +0000 Subject: [PATCH 016/119] enable torch profiler, but only main process --- atom/config.py | 4 +++ atom/model_engine/llm_engine.py | 60 +++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/atom/config.py b/atom/config.py index 6675ea9d3d..761a22ab29 100644 --- a/atom/config.py +++ b/atom/config.py @@ -18,6 +18,7 @@ class Config: num_kvcache_blocks: int = -1 kv_cache_dtype: str = "bf16" port: int = 8006 + torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) def __post_init__(self): # assert os.path.isdir(self.model) @@ -28,3 +29,6 @@ def __post_init__(self): self.max_model_len, self.hf_config.max_position_embeddings ) assert self.max_num_batched_tokens >= self.max_model_len + assert self.torch_profiler_dir is None or os.path.isdir( + self.torch_profiler_dir + ), f"torch_profiler_dir {self.torch_profiler_dir} is not a valid directory" diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index fb266d0e02..50bdb27d71 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -4,6 +4,7 @@ from tqdm.auto import tqdm from transformers import AutoTokenizer import torch.multiprocessing as mp +import torch.profiler as torch_profiler from atom.config import Config from atom.sampling_params import SamplingParams @@ -12,6 +13,14 @@ from atom.model_engine.model_runner import ModelRunner +class EmptyContext: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + class LLMEngine: def __init__(self, model, **kwargs): @@ -31,6 +40,22 @@ def __init__(self, model, **kwargs): self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True) config.eos = self.tokenizer.eos_token_id self.scheduler = Scheduler(config) + self.profiler = ( + EmptyContext() + if config.torch_profiler_dir is None + else torch_profiler.profile( + activities=[ + torch_profiler.ProfilerActivity.CPU, + torch_profiler.ProfilerActivity.CUDA, + ], + record_shapes=True, + with_stack=True, + profile_memory=True, + on_trace_ready=torch_profiler.tensorboard_trace_handler( + str(config.torch_profiler_dir) + ), + ) + ) atexit.register(self.exit) def exit(self): @@ -72,24 +97,25 @@ def generate( self.add_request(prompt, sp) outputs = {} prefill_throughput = decode_throughput = 0.0 - while not self.is_finished(): - t = perf_counter() - output, num_tokens = self.step() - if use_tqdm: - if num_tokens > 0: - prefill_throughput = num_tokens / (perf_counter() - t) - else: - decode_throughput = -num_tokens / (perf_counter() - t) - pbar.set_postfix( - { - "Prefill": f"{int(prefill_throughput)}tok/s", - "Decode": f"{int(decode_throughput)}tok/s", - } - ) - for seq_id, token_ids in output: - outputs[seq_id] = token_ids + with self.profiler: + while not self.is_finished(): + t = perf_counter() + output, num_tokens = self.step() if use_tqdm: - pbar.update(1) + if num_tokens > 0: + prefill_throughput = num_tokens / (perf_counter() - t) + else: + decode_throughput = -num_tokens / (perf_counter() - t) + pbar.set_postfix( + { + "Prefill": f"{int(prefill_throughput)}tok/s", + "Decode": f"{int(decode_throughput)}tok/s", + } + ) + for seq_id, token_ids in output: + outputs[seq_id] = token_ids + if use_tqdm: + pbar.update(1) outputs = [outputs[seq_id] for seq_id in sorted(outputs)] outputs = [ {"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} From 535966adee65dca5ef164b54bbcc768f71b04584 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Fri, 22 Aug 2025 06:44:57 +0000 Subject: [PATCH 017/119] refine --- atom/models/qwen3.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 60700350a2..06005d71cb 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -13,6 +13,7 @@ MergedColumnParallelLinear, RowParallelLinear, ) + # from atom.model_ops.rotary_embedding import get_rope from aiter.rotary_embedding import get_rope from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead @@ -82,12 +83,13 @@ def forward( ) -> torch.Tensor: qkv = self.qkv_proj(hidden_states) q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) + q_by_head = q.view(-1, self.num_heads, self.head_dim) - q_by_head = self.q_norm(q_by_head) - q = q_by_head.view(q.shape) + q = self.q_norm(q_by_head).view(q.shape) + k_by_head = k.view(-1, self.num_kv_heads, self.head_dim) - k_by_head = self.k_norm(k_by_head) - k = k_by_head.view(k.shape) + k = self.k_norm(k_by_head).view(k.shape) + q, k = self.rotary_emb(positions, q, k) o = self.attn(q, k, v) output = self.o_proj(o) From 8c5d96691d78f08f9435613c936e07ed3c68e2ee Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Fri, 22 Aug 2025 06:49:42 +0000 Subject: [PATCH 018/119] fix ressource release --- atom/model_engine/model_runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 3e7fba0747..6d24937ec7 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -2,8 +2,7 @@ import os import torch import torch.distributed as dist -from aiter import init_dist_env -from aiter.dist.parallel_state import destroy_distributed_environment +from aiter import init_dist_env, destroy_dist_env from multiprocessing.synchronize import Event from multiprocessing.shared_memory import SharedMemory from aiter import dtypes @@ -63,8 +62,7 @@ def exit(self): if not self.enforce_eager: del self.graphs, self.graph_pool torch.cuda.synchronize() - # dist.destroy_process_group() - destroy_distributed_environment() + destroy_dist_env() def loop(self): while True: From 571c81d5dd80ef4d8f524f9fc436afc3f1f3cc68 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Fri, 22 Aug 2025 08:55:38 +0000 Subject: [PATCH 019/119] refine norm --- atom/model_engine/llm_engine.py | 2 +- atom/model_ops/layernorm.py | 61 +++++++++++++++++---------------- atom/models/qwen3.py | 7 ++-- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 50bdb27d71..a7b1faf39a 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -52,7 +52,7 @@ def __init__(self, model, **kwargs): with_stack=True, profile_memory=True, on_trace_ready=torch_profiler.tensorboard_trace_handler( - str(config.torch_profiler_dir) + str(config.torch_profiler_dir), use_gzip=True ), ) ) diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py index 00c68275a1..4dd1a22878 100644 --- a/atom/model_ops/layernorm.py +++ b/atom/model_ops/layernorm.py @@ -7,49 +7,52 @@ class RMSNorm(nn.Module): def __init__( self, - hidden_size: int, + dim: int, eps: float = 1e-6, ) -> None: super().__init__() - self.hidden_size = hidden_size + self.dim = dim self.eps = eps - self.weight = nn.Parameter(torch.ones(hidden_size)) + self.weight = nn.Parameter(torch.ones(dim)) - def rms_forward( - self, - x: torch.Tensor, - ) -> torch.Tensor: - orig_dtype = x.dtype - x = x.to(torch.float32) - var = x.pow(2).mean(dim=-1, keepdim=True) - x.mul_(torch.rsqrt(var + self.eps)) - x = x.to(orig_dtype).mul_(self.weight) - return x + # def rms_forward( + # self, + # x: torch.Tensor, + # ) -> torch.Tensor: + # orig_dtype = x.dtype + # x = x.to(torch.float32) + # var = x.pow(2).mean(dim=-1, keepdim=True) + # x.mul_(torch.rsqrt(var + self.eps)) + # x = x.to(orig_dtype).mul_(self.weight) + # return x - def add_rms_forward( - self, - x: torch.Tensor, - residual: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - orig_dtype = x.dtype - x = x.to(torch.float32).add_(residual.to(torch.float32)) - residual = x.to(orig_dtype) - var = x.pow(2).mean(dim=-1, keepdim=True) - x.mul_(torch.rsqrt(var + self.eps)) - x = x.to(orig_dtype).mul_(self.weight) - return x, residual + # def add_rms_forward( + # self, + # x: torch.Tensor, + # residual: torch.Tensor, + # ) -> tuple[torch.Tensor, torch.Tensor]: + # orig_dtype = x.dtype + # x = x.to(torch.float32).add_(residual.to(torch.float32)) + # residual = x.to(orig_dtype) + # var = x.pow(2).mean(dim=-1, keepdim=True) + # x.mul_(torch.rsqrt(var + self.eps)) + # x = x.to(orig_dtype).mul_(self.weight) + # return x, residual def forward( self, x: torch.Tensor, residual: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - x = x.reshape(-1, x.shape[-1]) + ori_shape = x.shape + x = x.reshape(-1, self.dim) if residual is None: - return rmsnorm2d_fwd(x, self.weight, self.eps) + return rmsnorm2d_fwd(x, self.weight, self.eps).view(ori_shape) else: # return self.add_rms_forward(x, residual) residual_out = torch.empty_like(x) out = torch.empty_like(x) - rmsnorm2d_fwd_with_add(out, x, residual, residual_out, self.weight, self.eps) - return out, residual_out \ No newline at end of file + rmsnorm2d_fwd_with_add( + out, x, residual, residual_out, self.weight, self.eps + ) + return out.view(ori_shape), residual_out.view(ori_shape) diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 06005d71cb..801727ba23 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -84,11 +84,8 @@ def forward( qkv = self.qkv_proj(hidden_states) q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) - q_by_head = q.view(-1, self.num_heads, self.head_dim) - q = self.q_norm(q_by_head).view(q.shape) - - k_by_head = k.view(-1, self.num_kv_heads, self.head_dim) - k = self.k_norm(k_by_head).view(k.shape) + q = self.q_norm(q) + k = self.k_norm(k) q, k = self.rotary_emb(positions, q, k) o = self.attn(q, k, v) From 183e9e07fa40660a58bb4531bf6561915f8659b2 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Mon, 25 Aug 2025 22:16:52 +0800 Subject: [PATCH 020/119] fix recompilation errors in torch.compile and fullgraph=true --- atom/config.py | 27 +++++++++++++++++++++++++++ atom/model_engine/model_runner.py | 7 +++++-- example.py | 20 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/atom/config.py b/atom/config.py index 761a22ab29..cc7af04a6a 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,6 +1,31 @@ import os from dataclasses import dataclass from transformers import AutoConfig +from typing import List +from dataclasses import field + + +@dataclass +class CompilationConfig: + level: int = 3 + """The level of compilation: + + - 0: no compilation. + - 1: dynamo as is. + - 2: dynamo once. + - 3: piecewise compilation.""" + use_cudagraph: bool = field(default_factory=lambda: 0) + local_cache_dir: str = field(default=None, init=False) # type: ignore + # cudagraph_capture_sizes: Optional[list[int]] = [1,2,4,8] + cudagraph_capture_sizes: List[int] = field( + default_factory=lambda: [1, 2, 4, 8] + ) + + def __post_init__(self): + if not isinstance(self.cudagraph_capture_sizes, list): + raise ValueError("cudagraph_capture_sizes must be list") + if self.level not in {0, 1, 2, 3}: + raise ValueError("level must in 0-3") @dataclass @@ -19,6 +44,8 @@ class Config: kv_cache_dtype: str = "bf16" port: int = 8006 torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) + compilation_config: CompilationConfig = field( + default_factory=CompilationConfig) def __post_init__(self): # assert os.path.isdir(self.model) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 6d24937ec7..425c278793 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -34,7 +34,8 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") self.model = Qwen3ForCausalLM(hf_config, config.kv_cache_dtype) - # self.model = torch.compile(self.model, fullgraph=True, backend="eager") + if self.config.compilation_config.level == 3: + self.model = torch.compile(self.model, fullgraph=True, backend="eager") load_model(self.model, config.model) self.sampler = Sampler() self.warmup_model() @@ -369,7 +370,9 @@ def capture_cudagraph(self): block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32) outputs = torch.zeros(max_bs, hf_config.hidden_size) - self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) + # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) + self.graph_bs = self.config.compilation_config.cudagraph_capture_sizes + self.graphs = {} self.graph_pool = None diff --git a/example.py b/example.py index 85915c5f89..c97c518d16 100644 --- a/example.py +++ b/example.py @@ -2,6 +2,8 @@ from atom import LLMEngine, SamplingParams from transformers import AutoTokenizer import argparse +from atom.config import CompilationConfig +from typing import List parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, @@ -30,16 +32,34 @@ parser.add_argument("--port", type=int, default=8006, help="API server port") +parser.add_argument( + "--cudagraph-capture-sizes", type=str, default="[1,2,4,8,16]", help="Sizes to capture cudagraph." +) + +parser.add_argument("--level", type=int, default=0, help="The level of compilation") + +def parse_size_list(size_str: str) -> List[int]: + import ast + try: + return ast.literal_eval(size_str) + except (ValueError, SyntaxError) as e: + raise ValueError(f"Error list size: {size_str}") from e + def main(): args = parser.parse_args() model_name_or_path = args.model tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) + # If you want to use torch.compile, please --level=3 llm = LLMEngine( model_name_or_path, enforce_eager=args.enforce_eager, tensor_parallel_size=args.tensor_parallel_size, kv_cache_dtype=args.kv_cache_dtype, port=args.port, + compilation_config=CompilationConfig( + level = args.level, + cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes) + ) ) sampling_params = SamplingParams(temperature=0.6, max_tokens=256) From 71dad514b465b3aa725d264236fc91f8da8bf444 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Tue, 26 Aug 2025 11:08:59 +0800 Subject: [PATCH 021/119] impl cuda_graph capture size --- atom/config.py | 21 ++++++++++++++------- atom/model_engine/model_runner.py | 13 ++++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/atom/config.py b/atom/config.py index cc7af04a6a..6061ea9fff 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,13 +1,13 @@ import os from dataclasses import dataclass from transformers import AutoConfig -from typing import List +from typing import Optional from dataclasses import field @dataclass class CompilationConfig: - level: int = 3 + level: int = 0 """The level of compilation: - 0: no compilation. @@ -17,15 +17,22 @@ class CompilationConfig: use_cudagraph: bool = field(default_factory=lambda: 0) local_cache_dir: str = field(default=None, init=False) # type: ignore # cudagraph_capture_sizes: Optional[list[int]] = [1,2,4,8] - cudagraph_capture_sizes: List[int] = field( - default_factory=lambda: [1, 2, 4, 8] - ) + cudagraph_capture_sizes: Optional[list[int]] = None + + cuda_graph_sizes: list[int] = field(default_factory=list) + """Cuda graph capture sizes + 1. if none provided, then default set to [min(max_num_seqs * 2, 512)] + 2. if one value is provided, then the capture list would follow the + pattern: [1, 2, 4] + [i for i in range(8, cuda_graph_sizes + 1, 8)] + 3. more than one value (e.g. 1 2 128) is provided, then the capture list + will follow the provided list.""" + def __post_init__(self): - if not isinstance(self.cudagraph_capture_sizes, list): - raise ValueError("cudagraph_capture_sizes must be list") if self.level not in {0, 1, 2, 3}: raise ValueError("level must in 0-3") + if not self.cuda_graph_sizes: + self.cuda_graph_sizes = [128] @dataclass diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 425c278793..f898468b0c 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -371,7 +371,18 @@ def capture_cudagraph(self): outputs = torch.zeros(max_bs, hf_config.hidden_size) # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) - self.graph_bs = self.config.compilation_config.cudagraph_capture_sizes + self.graph_bs = [] + if self.config.compilation_config.cudagraph_capture_sizes: + self.graph_bs = self.config.compilation_config.cudagraph_capture_sizes + else: + cuda_graph_sizes = self.config.compilation_config.cuda_graph_sizes + if len(cuda_graph_sizes) == 1: + self.graph_bs = [1, 2, 4, 8] + [ + i for i in range(64, cuda_graph_sizes[0] + 1, 64) + ] + elif len(cuda_graph_sizes) > 1: + self.graph_bs = sorted(cuda_graph_sizes) + self.graphs = {} self.graph_pool = None From b73657e5e5915461ea38f381b4dedd3034d0113e Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Tue, 26 Aug 2025 03:18:36 +0000 Subject: [PATCH 022/119] fix_allreduce_with_graph --- atom/model_engine/model_runner.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index f898468b0c..bfbac1f858 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -2,10 +2,11 @@ import os import torch import torch.distributed as dist +from aiter import dtypes from aiter import init_dist_env, destroy_dist_env +from aiter.dist.parallel_state import graph_capture from multiprocessing.synchronize import Event from multiprocessing.shared_memory import SharedMemory -from aiter import dtypes from atom.config import Config from atom.model_engine.sequence import Sequence from atom.model_ops.sampler import Sampler @@ -327,6 +328,7 @@ def run_model( else: bs = input_ids.size(0) graph_bs = next(x for x in self.graph_bs if x >= bs) + assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" context = get_context() graph = self.graphs[graph_bs] @@ -397,8 +399,10 @@ def capture_cudagraph(self): ) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup - with torch.cuda.graph(graph, self.graph_pool): - outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture + + with graph_capture() as gc: + with torch.cuda.graph(graph, self.graph_pool, stream=gc.stream): + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture if self.graph_pool is None: self.graph_pool = graph.pool() self.graphs[bs] = graph From 783edbee2cf19dd76c0da3fd09866eb8adb03343 Mon Sep 17 00:00:00 2001 From: chenjun Date: Tue, 26 Aug 2025 03:21:20 +0000 Subject: [PATCH 023/119] Enable llama and Simple support for quantification --- atom/config.py | 74 ++++- atom/model_engine/block_manager.py | 18 +- atom/model_engine/model_runner.py | 10 +- atom/model_engine/scheduler.py | 11 +- atom/model_ops/attention.py | 1 + atom/model_ops/embed_head.py | 1 + atom/model_ops/linear.py | 136 ++++++--- atom/models/llama.py | 444 +++++++++-------------------- atom/models/utils.py | 261 +++++++++++++++++ example.py | 6 +- 10 files changed, 605 insertions(+), 357 deletions(-) create mode 100644 atom/models/utils.py diff --git a/atom/config.py b/atom/config.py index 6061ea9fff..b5f929589a 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,8 +1,13 @@ import os +import torch +import re from dataclasses import dataclass from transformers import AutoConfig from typing import Optional from dataclasses import field +from aiter import QuantType +from transformers import PretrainedConfig +from aiter.utility.dtypes import d_dtypes @dataclass @@ -35,6 +40,71 @@ def __post_init__(self): self.cuda_graph_sizes = [128] +class QuantizationConfig(dict): + def __init__( + self, quant_type=QuantType.No, quant_dtype=torch.bfloat16, quant_name="" + ): + super().__init__() + self["quant_type"] = quant_type if quant_type is not None else QuantType.No + self["quant_dtype"] = quant_dtype if quant_dtype is not None else torch.bfloat16 + self["quant_name"] = quant_name + + def get_name(self): + return self["quant_name"] + + +def get_quant_config(config: PretrainedConfig) -> QuantizationConfig: + torch_dtype = getattr(config, "torch_dtype", "bf16") + orig_quant_config = getattr(config, "quantization_config", None) + if orig_quant_config is None: + return QuantizationConfig( + quant_type=QuantType.No, + quant_dtype=torch_dtype, + ) + + quant_method = orig_quant_config.get("quant_method", None) + RE_QUANT_BLOCKSIZE = r"\'(?:group_size|weight_block_size)\'\:\s*(?:\[\n*)\s*(\d+)," + orig_quant_config_str = str(orig_quant_config) + if quant_method == "compressed-tensors" or 'channel",' in orig_quant_config_str: + quant_type = QuantType.per_Token + elif group_size := re.search(RE_QUANT_BLOCKSIZE, orig_quant_config_str): + group_size = int(group_size.group(1)) + assert group_size in (32, 128), f"Unsupported group size {group_size}" + if group_size == 128: + quant_type = QuantType.per_1x128 + elif group_size == 32: + quant_type = QuantType.per_1x32 + else: + quant_type = QuantType.per_Tensor + + RE_QUANT_DTYPE = r"\'(?:d?type|weight_dtype|quant_method)\'\:\s*\'(\w+)\'" + quant_dtype = None + m = re.search(RE_QUANT_DTYPE, orig_quant_config_str) + if m and m.group(1).lower() in ["fp8", "fp4", "int8", "int4"]: + dtype = m.group(1).lower() + if dtype.endswith("4"): + dtype += "x2" + quant_dtype = d_dtypes[dtype] + else: + bit_match = re.search(r"\'(?:num_)?bits\'\:\s*(\d+)", orig_quant_config_str) + if bit_match: + bit = int(bit_match.group(1)) + dtype_match = re.search(RE_QUANT_DTYPE, orig_quant_config_str) + if dtype_match: + dtype = dtype_match.group(1).lower() + dtype_prefix = "i" if dtype.startswith("int") else "fp" + else: + dtype_prefix = "i" + quant_dtype_str = ( + f"{dtype_prefix}{bit}" if bit != 4 else f"{dtype_prefix}{bit}x2" + ) + quant_dtype = d_dtypes.get(quant_dtype_str, None) + assert ( + quant_dtype is not None + ), f"Cannot parse quant dtype from {orig_quant_config_str}" + return QuantizationConfig(quant_type, quant_dtype) + + @dataclass class Config: model: str @@ -49,10 +119,10 @@ class Config: kvcache_block_size: int = 16 num_kvcache_blocks: int = -1 kv_cache_dtype: str = "bf16" + enable_prefix_caching: bool = False port: int = 8006 torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) - compilation_config: CompilationConfig = field( - default_factory=CompilationConfig) + compilation_config: CompilationConfig = field(default_factory=CompilationConfig) def __post_init__(self): # assert os.path.isdir(self.model) diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index 43373a0c98..669ede6a66 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -3,6 +3,7 @@ import numpy as np from atom.model_engine.sequence import Sequence +from atom.config import Config class Block: @@ -25,13 +26,16 @@ def reset(self): class BlockManager: - def __init__(self, num_blocks: int, block_size: int): + def __init__(self, config: Config): + block_size = config.kvcache_block_size + num_blocks = config.num_kvcache_blocks assert num_blocks > 0 self.block_size = block_size self.blocks: list[Block] = [Block(i) for i in range(num_blocks)] self.hash_to_block_id: dict[int, int] = dict() self.free_block_ids: deque[int] = deque(range(num_blocks)) self.used_block_ids: set[int] = set() + self.enable_prefix_caching = config.enable_prefix_caching @classmethod def compute_hash(cls, token_ids: list[int], prefix: int = -1): @@ -63,8 +67,14 @@ def allocate(self, seq: Sequence): cache_miss = False for i in range(seq.num_blocks): token_ids = seq.block(i) - h = self.compute_hash(token_ids, h) if len(token_ids) == self.block_size else -1 - block_id = self.hash_to_block_id.get(h, -1) + h = ( + self.compute_hash(token_ids, h) + if len(token_ids) == self.block_size + else -1 + ) + block_id = ( + self.hash_to_block_id.get(h, -1) if self.enable_prefix_caching else -1 + ) if block_id == -1 or self.blocks[block_id].token_ids != token_ids: cache_miss = True if cache_miss: @@ -104,7 +114,7 @@ def may_append(self, seq: Sequence): block_table.append(block_id) elif len(seq) % self.block_size == 0: assert last_block.hash == -1 - token_ids = seq.block(seq.num_blocks-1) + token_ids = seq.block(seq.num_blocks - 1) prefix = self.blocks[block_table[-2]].hash if len(block_table) > 1 else -1 h = self.compute_hash(token_ids, prefix) last_block.update(h, token_ids) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index bfbac1f858..7b9c12e311 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -11,9 +11,15 @@ from atom.model_engine.sequence import Sequence from atom.model_ops.sampler import Sampler from atom.models.qwen3 import Qwen3ForCausalLM +from atom.models.llama import LlamaForCausalLM from atom.utils.context import set_context, get_context, reset_context from atom.model_loader.loader import load_model +suppot_model_arch_dict = { + "Qwen3ForCausalLM": Qwen3ForCausalLM, + "LlamaForCausalLM": LlamaForCausalLM, +} + class ModelRunner: @@ -34,7 +40,9 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") - self.model = Qwen3ForCausalLM(hf_config, config.kv_cache_dtype) + self.model = suppot_model_arch_dict[hf_config.architectures[0]]( + hf_config, config.kv_cache_dtype + ) if self.config.compilation_config.level == 3: self.model = torch.compile(self.model, fullgraph=True, backend="eager") load_model(self.model, config.model) diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index c2db93844b..edb6962e9a 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -11,7 +11,7 @@ def __init__(self, config: Config): self.max_num_seqs = config.max_num_seqs self.max_num_batched_tokens = config.max_num_batched_tokens self.eos = config.eos - self.block_manager = BlockManager(config.num_kvcache_blocks, config.kvcache_block_size) + self.block_manager = BlockManager(config) self.waiting: deque[Sequence] = deque() self.running: deque[Sequence] = deque() @@ -28,11 +28,12 @@ def schedule(self) -> tuple[list[Sequence], bool]: num_batched_tokens = 0 if not self.running and not self.waiting: self.block_manager.reset() - while self.waiting and num_seqs < self.max_num_seqs: seq = self.waiting[0] - if num_batched_tokens + len(seq) > self.max_num_batched_tokens or not self.block_manager.can_allocate(seq): + if num_batched_tokens + len( + seq + ) > self.max_num_batched_tokens or not self.block_manager.can_allocate(seq): break num_seqs += 1 self.block_manager.allocate(seq) @@ -69,7 +70,9 @@ def preempt(self, seq: Sequence): def postprocess(self, seqs: list[Sequence], token_ids: list[int]) -> list[bool]: for seq, token_id in zip(seqs, token_ids): seq.append_token(token_id) - if (not seq.ignore_eos and token_id == self.eos) or seq.num_completion_tokens == seq.max_tokens: + if ( + not seq.ignore_eos and token_id == self.eos + ) or seq.num_completion_tokens == seq.max_tokens: seq.status = SequenceStatus.FINISHED self.block_manager.deallocate(seq) self.running.remove(seq) diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 10928ad07b..0fb1a4d8db 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -18,6 +18,7 @@ def __init__( scale, num_kv_heads, kv_cache_dtype="bf16", + **kwargs, ): super().__init__() self.num_heads = num_heads diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index a8600d86dd..100a031789 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -53,6 +53,7 @@ def __init__( num_embeddings: int, embedding_dim: int, bias: bool = False, + **kwargs, ): super().__init__(num_embeddings, embedding_dim) if bias: diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index edbd8dfe89..0a25233486 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -7,14 +7,17 @@ from aiter import ( QuantType, dtypes, + gemm_a8w8, gemm_a8w8_bpreshuffle, gemm_a8w8_blockscale, gemm_a4w4, ) from aiter.ops.shuffle import shuffle_weight from aiter.tuned_gemm import tgemm +from aiter import get_hip_quant from typing import Optional, Callable from functools import partial as functools_partial +from atom.config import QuantizationConfig def divide(numerator, denominator): @@ -30,9 +33,12 @@ def __init__( output_size: int, tp_dim: int | None = None, bias: bool = False, - params_dtype=dtypes.bf16, - quant_type=QuantType.No, + quant_config: Optional[QuantizationConfig] = None, ): + if quant_config is None: + quant_config = QuantizationConfig() + quant_type = quant_config["quant_type"] + params_dtype = quant_config["quant_dtype"] super().__init__() self.input_size = input_size self.output_size = output_size @@ -44,44 +50,61 @@ def __init__( elif tp_dim == 0: self.output_size = divide(output_size, self.tp_size) self.weight = nn.Parameter( - torch.empty((self.output_size, self.input_size), dtype=params_dtype) + torch.empty((self.output_size, self.input_size), dtype=params_dtype), + requires_grad=False, ) if bias: - self.bias = nn.Parameter(torch.empty(self.output_size, dtype=params_dtype)) + self.bias = nn.Parameter( + torch.empty(self.output_size, dtype=params_dtype), requires_grad=False + ) self.bias.weight_loader_process = self.weight_loader_process else: self.register_parameter("bias", None) self.quant_type = quant_type + self.params_dtype = params_dtype if quant_type != QuantType.No: if quant_type == QuantType.per_Tensor: self.weight.weight_loader_process = functools_partial( - self.weight_loader_process, post_process_func=shuffle_weight(16, 16) + self.weight_loader_process, + post_process_func=lambda x: shuffle_weight(x, (16, 16)), + ) + self.weight_scale = nn.Parameter( + torch.empty(1, dtype=dtypes.fp32), requires_grad=False ) - self.weight_scale = nn.Parameter(torch.empty(1, dtype=dtypes.fp32)) elif quant_type == QuantType.per_Token: - self.weight.weight_loader_process = self.weight_loader_process + if params_dtype == dtypes.i8: + self.weight.weight_loader_process = self.weight_loader_process + else: + self.weight.weight_loader_process = functools_partial( + self.weight_loader_process, + post_process_func=lambda x: shuffle_weight(x, (16, 16)), + ) self.weight_scale = nn.Parameter( - torch.empty(self.output_size, dtype=dtypes.fp32) + torch.empty(self.output_size, 1, dtype=dtypes.fp32), + requires_grad=False, ) - elif quant_type == QuantType.per_128x128: + elif quant_type == QuantType.per_1x128: self.weight.weight_loader_process = self.weight_loader_process self.weight_scale = nn.Parameter( torch.empty( divide(self.output_size, 128), (self.input_size + 127) // 128, dtype=dtypes.fp32, - ) + ), + requires_grad=False, ) elif quant_type == QuantType.per_1x32: self.weight.weight_loader_process = functools_partial( - self.weight_loader_process, post_process_func=shuffle_weight(16, 16) + self.weight_loader_process, + post_process_func=lambda x: shuffle_weight(x, (16, 16)), ) self.weight_scale = nn.Parameter( torch.empty( self.output_size, (self.input_size + 31) // 32, dtype=dtypes.fp8_e8m0, - ) + ), + requires_grad=False, ) self.weight_scale.weight_loader_process = self.weight_loader_process else: @@ -111,7 +134,9 @@ def forward( if self.quant_type.value == QuantType.No.value: y = tgemm.mm(x, self.weight, self.bias) else: - assert x_scale is not None + if x_scale is None: + quant_func = get_hip_quant(self.quant_type) + x, x_scale = quant_func(x, quant_dtype=self.params_dtype) if self.quant_type.value == QuantType.per_Tensor.value: y = tgemm.mm( x, @@ -122,16 +147,43 @@ def forward( scale_b=self.weight_scale, ) elif self.quant_type.value == QuantType.per_Token.value: - y = gemm_a8w8_bpreshuffle( - x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype - ) - elif self.quant_type.value == QuantType.per_128x128.value: + if self.params_dtype == dtypes.i8: + y = gemm_a8w8( + x, + self.weight, + x_scale, + self.weight_scale, + self.bias, + dtype=otype, + ) + else: + y = gemm_a8w8_bpreshuffle( + x, + self.weight, + x_scale, + self.weight_scale, + self.bias, + dtype=otype, + ) + elif self.quant_type.value == QuantType.per_1x128.value: y = gemm_a8w8_blockscale( x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype ) elif self.quant_type.value == QuantType.per_1x32.value: + m = x.view(-1, x.size(-1)).shape[0] + y = torch.empty( + ((m + 31) // 32 * 32, self.output_size), + dtype=otype, + device=x.device, + ) y = gemm_a4w4( - x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype + x, + self.weight, + x_scale, + self.weight_scale, + y, + self.bias, + dtype=otype, ) if self.tp_dim == 1 and self.tp_size > 1: y = get_tp_group().all_reduce(y, open_fp8_quant=False) @@ -145,16 +197,15 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - params_dtype=dtypes.bf16, - quant_type=QuantType.No, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): super().__init__( input_size, output_size, tp_dim=None, bias=bias, - params_dtype=params_dtype, - quant_type=quant_type, + quant_config=quant_config, ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): @@ -169,12 +220,16 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - params_dtype=dtypes.bf16, - quant_type=QuantType.No, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): self.tp_dim = 0 super().__init__( - input_size, output_size, self.tp_dim, bias, params_dtype, quant_type + input_size, + output_size, + self.tp_dim, + bias, + quant_config=quant_config, ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): @@ -192,11 +247,16 @@ def __init__( input_size: int, output_sizes: list[int], bias: bool = False, - params_dtype=dtypes.bf16, - quant_type=QuantType.No, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): self.output_sizes = output_sizes - super().__init__(input_size, sum(output_sizes), bias, params_dtype, quant_type) + super().__init__( + input_size, + sum(output_sizes), + bias, + quant_config=quant_config, + ) def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int @@ -218,8 +278,8 @@ def __init__( total_num_heads: int, total_num_kv_heads: int | None = None, bias: bool = False, - params_dtype=dtypes.bf16, - quant_type=QuantType.No, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): self.head_size = head_size self.total_num_heads = total_num_heads @@ -231,7 +291,12 @@ def __init__( output_size = ( self.total_num_heads + 2 * self.total_num_kv_heads ) * self.head_size - super().__init__(input_size, output_size, bias, params_dtype, quant_type) + super().__init__( + input_size, + output_size, + bias, + quant_config=quant_config, + ) def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str @@ -261,8 +326,8 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - params_dtype=dtypes.bf16, - quant_type=QuantType.No, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): self.tp_rank = get_tp_group().rank super().__init__( @@ -270,13 +335,14 @@ def __init__( output_size, tp_dim=1, bias=bias if self.tp_rank == 0 else False, - params_dtype=params_dtype, - quant_type=quant_type, + quant_config=quant_config, ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) + if loaded_weight.size(self.tp_dim) == 1 and self.tp_size > 1: + loaded_weight = loaded_weight.repeat(1, self.tp_size) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) param.weight_loader_process(param_data, loaded_weight) diff --git a/atom/models/llama.py b/atom/models/llama.py index 5187d5c014..10fb47ed55 100644 --- a/atom/models/llama.py +++ b/atom/models/llama.py @@ -30,30 +30,32 @@ from torch import nn from transformers import LlamaConfig -from vllm.attention import Attention, AttentionType -from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, VllmConfig -from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, - QKVParallelLinear, - RowParallelLinear) -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ( - DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, maybe_remap_kv_scale_name) -from vllm.model_executor.sampling_metadata import SamplingMetadata -from vllm.sequence import IntermediateTensors - -from .interfaces import SupportsEagle3, SupportsLoRA, SupportsPP -from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index, - is_pp_missing_parameter, - make_empty_intermediate_tensors_factory, make_layers, - maybe_prefix) +from aiter import QuantType +from atom.model_ops.attention import Attention +from aiter.dist.parallel_state import ( + get_pp_group, + get_tp_group, + get_tensor_model_parallel_world_size, +) +from atom.model_ops.activation import SiluAndMul +from atom.model_ops.layernorm import RMSNorm +from atom.model_ops.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from aiter.rotary_embedding import get_rope +from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead +from atom.config import QuantizationConfig, get_quant_config + +from atom.models.utils import ( + PPMissingLayer, + extract_layer_index, + IntermediateTensors, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) class LlamaMLP(nn.Module): @@ -85,14 +87,16 @@ def __init__( prefix=f"{prefix}.down_proj", ) if hidden_act != "silu": - raise ValueError(f"Unsupported activation: {hidden_act}. " - "Only silu is supported for now.") + raise ValueError( + f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now." + ) self.act_fn = SiluAndMul() def forward(self, x): - x, _ = self.gate_up_proj(x) + x = self.gate_up_proj(x) x = self.act_fn(x) - x, _ = self.down_proj(x) + x = self.down_proj(x) return x @@ -110,9 +114,8 @@ def __init__( quant_config: Optional[QuantizationConfig] = None, bias: bool = False, bias_o_proj: bool = False, - cache_config: Optional[CacheConfig] = None, + cache_config: str = "bf16", prefix: str = "", - attn_type: str = AttentionType.DECODER, ) -> None: super().__init__() layer_idx = extract_layer_index(prefix) @@ -137,8 +140,7 @@ def __init__( head_dim = self.hidden_size // self.total_num_heads self.head_dim = head_dim # Phi models introduced a partial_rotary_factor parameter in the config - self.partial_rotary_factor = getattr(config, "partial_rotary_factor", - 1) + self.partial_rotary_factor = getattr(config, "partial_rotary_factor", 1) self.q_size = self.num_heads * self.head_dim self.kv_size = self.num_kv_heads * self.head_dim self.scaling = self.head_dim**-0.5 @@ -163,9 +165,9 @@ def __init__( prefix=f"{prefix}.o_proj", ) - self._init_rotary_emb(config, - rope_scaling=rope_scaling, - quant_config=quant_config) + self._init_rotary_emb( + config, rope_scaling=rope_scaling, quant_config=quant_config + ) sliding_window = None if layer_types := getattr(config, "layer_types", None): @@ -178,10 +180,8 @@ def __init__( self.head_dim, self.scaling, num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, + kv_cache_dtype=cache_config, per_layer_sliding_window=sliding_window, - attn_type=attn_type, prefix=f"{prefix}.attn", ) @@ -190,16 +190,19 @@ def forward( positions: torch.Tensor, hidden_states: torch.Tensor, ) -> torch.Tensor: - qkv, _ = self.qkv_proj(hidden_states) + qkv = self.qkv_proj(hidden_states) q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) - output, _ = self.o_proj(attn_output) + output = self.o_proj(attn_output) return output - def _init_rotary_emb(self, config: LlamaConfig, - rope_scaling: Optional[dict[str, Any]], - quant_config: Optional[QuantizationConfig]) -> None: + def _init_rotary_emb( + self, + config: LlamaConfig, + rope_scaling: Optional[dict[str, Any]], + quant_config: Optional[QuantizationConfig], + ) -> None: is_neox_style = True is_gguf = quant_config and quant_config.get_name() == "gguf" if is_gguf and config.model_type == "llama": @@ -221,7 +224,7 @@ class LlamaDecoderLayer(nn.Module): def __init__( self, config: LlamaConfig, - cache_config: Optional[CacheConfig] = None, + cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", ) -> None: @@ -230,35 +233,29 @@ def __init__( rope_theta = getattr(config, "rope_theta", 10000) rope_scaling = getattr(config, "rope_scaling", None) if rope_scaling is not None and getattr( - config, "original_max_position_embeddings", None): + config, "original_max_position_embeddings", None + ): rope_scaling["original_max_position_embeddings"] = ( - config.original_max_position_embeddings) - max_position_embeddings = getattr(config, "max_position_embeddings", - 8192) + config.original_max_position_embeddings + ) + max_position_embeddings = getattr(config, "max_position_embeddings", 8192) # Support abacusai/Smaug-72B-v0.1 with attention_bias # Support internlm/internlm-7b with bias attention_bias = getattr(config, "attention_bias", False) or getattr( - config, "bias", False) + config, "bias", False + ) bias_o_proj = attention_bias # support internlm/internlm3-8b with qkv_bias - if hasattr(config, 'qkv_bias'): + if hasattr(config, "qkv_bias"): attention_bias = config.qkv_bias - # By default, Llama uses causal attention as it is a decoder-only model. - # You can override the HF config with `is_causal=False` to enable - # bidirectional attention, which is used in some embedding models - # (e.g. parasail-ai/GritLM-7B-vllm) - if getattr(config, "is_causal", True): - attn_type = AttentionType.DECODER - else: - attn_type = AttentionType.ENCODER_ONLY - self.self_attn = LlamaAttention( config=config, hidden_size=self.hidden_size, num_heads=config.num_attention_heads, - num_kv_heads=getattr(config, "num_key_value_heads", - config.num_attention_heads), + num_kv_heads=getattr( + config, "num_key_value_heads", config.num_attention_heads + ), rope_theta=rope_theta, rope_scaling=rope_scaling, max_position_embeddings=max_position_embeddings, @@ -267,7 +264,6 @@ def __init__( bias_o_proj=bias_o_proj, cache_config=cache_config, prefix=f"{prefix}.self_attn", - attn_type=attn_type, ) self.mlp = LlamaMLP( hidden_size=self.hidden_size, @@ -277,10 +273,10 @@ def __init__( bias=getattr(config, "mlp_bias", False), prefix=f"{prefix}.mlp", ) - self.input_layernorm = RMSNorm(config.hidden_size, - eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm(config.hidden_size, - eps=config.rms_norm_eps) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) def forward( self, @@ -293,55 +289,47 @@ def forward( residual = hidden_states hidden_states = self.input_layernorm(hidden_states) else: - hidden_states, residual = self.input_layernorm( - hidden_states, residual) - hidden_states = self.self_attn(positions=positions, - hidden_states=hidden_states) + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states) # Fully Connected - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual) + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) hidden_states = self.mlp(hidden_states) return hidden_states, residual -@support_torch_compile +# @support_torch_compile class LlamaModel(nn.Module): - - def __init__(self, - *, - vllm_config: VllmConfig, - prefix: str = "", - layer_type: type[nn.Module] = LlamaDecoderLayer): + def __init__( + self, + config: LlamaConfig, + cache_config: str = "bf16", + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + layer_type: type[nn.Module] = LlamaDecoderLayer, + ): super().__init__() - - config = vllm_config.model_config.hf_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - lora_config = vllm_config.lora_config - self.config = config self.quant_config = quant_config - lora_vocab = (lora_config.lora_extra_vocab_size * - (lora_config.max_loras or 1)) if lora_config else 0 - self.vocab_size = config.vocab_size + lora_vocab + self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size - if get_pp_group().is_first_rank or (config.tie_word_embeddings - and get_pp_group().is_last_rank): + if get_pp_group().is_first_rank or ( + config.tie_word_embeddings and get_pp_group().is_last_rank + ): self.embed_tokens = VocabParallelEmbedding( self.vocab_size, config.hidden_size, - org_num_embeddings=config.vocab_size, - quant_config=quant_config, ) else: self.embed_tokens = PPMissingLayer() self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, - lambda prefix: layer_type(config=config, - cache_config=cache_config, - quant_config=quant_config, - prefix=prefix), + lambda prefix: layer_type( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ), prefix=f"{prefix}.layers", ) if get_pp_group().is_last_rank: @@ -351,9 +339,9 @@ def __init__(self, self.aux_hidden_state_layers: tuple[int] = tuple() - self.make_empty_intermediate_tensors = ( - make_empty_intermediate_tensors_factory( - ["hidden_states", "residual"], config.hidden_size)) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -364,8 +352,9 @@ def forward( positions: torch.Tensor, intermediate_tensors: Optional[IntermediateTensors], inputs_embeds: Optional[torch.Tensor] = None, - ) -> Union[torch.Tensor, IntermediateTensors, tuple[torch.Tensor, - list[torch.Tensor]]]: + ) -> Union[ + torch.Tensor, IntermediateTensors, tuple[torch.Tensor, list[torch.Tensor]] + ]: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds @@ -378,17 +367,15 @@ def forward( residual = intermediate_tensors["residual"] aux_hidden_states = [] - for idx, layer in enumerate( - self.layers[self.start_layer:self.end_layer]): + for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]): if idx in self.aux_hidden_state_layers: aux_hidden_states.append(hidden_states + residual) hidden_states, residual = layer(positions, hidden_states, residual) if not get_pp_group().is_last_rank: - return IntermediateTensors({ - "hidden_states": hidden_states, - "residual": residual - }) + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) hidden_states, _ = self.norm(hidden_states, residual) @@ -396,173 +383,63 @@ def forward( return hidden_states, aux_hidden_states return hidden_states - def load_weights(self, weights: Iterable[tuple[str, - torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if ("rotary_emb.cos_cached" in name - or "rotary_emb.sin_cached" in name): - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - if (self.quant_config is not None and - (scale_name := self.quant_config.get_cache_scale(name))): - # Loading kv cache quantization scales - param = params_dict[scale_name] - weight_loader = getattr(param, "weight_loader", - default_weight_loader) - loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else - loaded_weight[0]) - weight_loader(param, loaded_weight) - loaded_params.add(scale_name) - continue - if "scale" in name: - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", - default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - -class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3): +class LlamaForCausalLM(nn.Module): packed_modules_mapping = { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - "gate_up_proj": ["gate_proj", "up_proj"] + "q_proj": ("qkv_proj", "q"), + "k_proj": ("qkv_proj", "k"), + "v_proj": ("qkv_proj", "v"), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), } - # LoRA specific attributes - embedding_modules = { - "embed_tokens": "input_embeddings", - "lm_head": "output_embeddings" - } - embedding_padding_modules = ["lm_head"] - - # Mistral/Llama models can also be loaded with --load-format mistral - # from consolidated.safetensors checkpoints - mistral_mapping = { - "layers": "model.layers", - "attention": "self_attn", - "qscale_act": "input_scale", - "qscale_weight": "weight_scale", - "kv_fake_quantizer.qscale_act": "kv_scale", - "q_fake_quantizer.qscale_act": "attn.q_scale", - "k_fake_quantizer.qscale_act": "k_scale", - "v_fake_quantizer.qscale_act": "v_scale", - "wq": "q_proj", - "wk": "k_proj", - "wv": "v_proj", - "wo": "o_proj", - "attention_norm": "input_layernorm", - "feed_forward": "mlp", - "w1": "gate_proj", - "w2": "down_proj", - "w3": "up_proj", - "ffn_norm": "post_attention_layernorm", - "tok_embeddings": "model.embed_tokens", - "output": "lm_head", - "norm": "model.norm", - } - - def __init__(self, - *, - vllm_config: VllmConfig, - prefix: str = "", - layer_type: type[nn.Module] = LlamaDecoderLayer): + def __init__( + self, + config: LlamaConfig, + kv_cache_dtype: str = "bf16", + prefix: str = "", + layer_type: type[nn.Module] = LlamaDecoderLayer, + ): super().__init__() - config = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - lora_config = vllm_config.lora_config self.config = config - self.lora_config = lora_config - - self.model = self._init_model(vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "model"), - layer_type=layer_type) + self.model = self._init_model( + config=config, + cache_config=kv_cache_dtype, + prefix=maybe_prefix(prefix, "model"), + layer_type=layer_type, + ) if get_pp_group().is_last_rank: self.unpadded_vocab_size = config.vocab_size - if lora_config: - self.unpadded_vocab_size += lora_config.lora_extra_vocab_size self.lm_head = ParallelLMHead( self.unpadded_vocab_size, config.hidden_size, org_num_embeddings=config.vocab_size, - padding_size=( - DEFAULT_VOCAB_PADDING_SIZE - # We need bigger padding if using lora for kernel - # compatibility - if not lora_config else - lora_config.lora_vocab_padding_size), - quant_config=quant_config, prefix=maybe_prefix(prefix, "lm_head"), ) if config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights( - self.model.embed_tokens) - - logit_scale = getattr(config, "logit_scale", 1.0) - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size, - config.vocab_size, - logit_scale) + self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) else: self.lm_head = PPMissingLayer() self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors) - - def set_aux_hidden_state_layers(self, layers: tuple[int]) -> None: - self.model.aux_hidden_state_layers = layers - - def get_eagle3_aux_hidden_state_layers(self) -> tuple[int]: - num_layers = len(self.model.layers) - return (2, num_layers // 2, num_layers - 3) + self.model.make_empty_intermediate_tensors + ) - def _init_model(self, - vllm_config: VllmConfig, - prefix: str = "", - layer_type: type[nn.Module] = LlamaDecoderLayer): - return LlamaModel(vllm_config=vllm_config, - prefix=prefix, - layer_type=layer_type) + def _init_model( + self, + config: LlamaConfig, + cache_config: str = "bf16", + prefix: str = "", + layer_type: type[nn.Module] = LlamaDecoderLayer, + ): + return LlamaModel( + config=config, + cache_config=cache_config, + quant_config=get_quant_config(config), + prefix=prefix, + layer_type=layer_type, + ) def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.get_input_embeddings(input_ids) @@ -574,67 +451,14 @@ def forward( intermediate_tensors: Optional[IntermediateTensors] = None, inputs_embeds: Optional[torch.Tensor] = None, ) -> Union[torch.Tensor, IntermediateTensors]: - model_output = self.model(input_ids, positions, intermediate_tensors, - inputs_embeds) + model_output = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds + ) return model_output def compute_logits( self, hidden_states: torch.Tensor, - sampling_metadata: SamplingMetadata, ) -> Optional[torch.Tensor]: - logits = self.logits_processor(self.lm_head, hidden_states, - sampling_metadata) + logits = self.lm_head(hidden_states) return logits - - def load_weights(self, weights: Iterable[tuple[str, - torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader( - self, - skip_prefixes=(["lm_head."] - if self.config.tie_word_embeddings else None), - ) - return loader.load_weights( - self.maybe_remap_mistral(name, loaded_weight) - for name, loaded_weight in weights) - - # This function is used to remap the mistral format as - # used by Mistral and Llama <=2 - def maybe_remap_mistral( - self, - name: str, - loaded_weight: torch.Tensor, - ) -> tuple[str, torch.Tensor]: - - def permute(w: torch.Tensor, n_heads: int): - attn_in = self.config.head_dim * n_heads - attn_out = self.config.hidden_size - - return w.view(n_heads, attn_in // n_heads // 2, 2, - attn_out).transpose(1, 2).reshape(attn_in, attn_out) - - mapping = self.mistral_mapping - modules = name.split(".") - - # rotary embeds should be sliced - if "wk" in modules and modules[-1] == "weight": - loaded_weight = permute(loaded_weight, - self.config.num_key_value_heads) - elif "wq" in modules and modules[-1] == "weight": - loaded_weight = permute(loaded_weight, - self.config.num_attention_heads) - - num_modules = len(modules) - for i in range(num_modules): - item = modules[i] - next_item = modules[i + 1] if i < num_modules - 1 else None - - combined_item = (f"{item}.{next_item}" - if next_item is not None else None) - - if combined_item in mapping: - name = name.replace(combined_item, mapping[combined_item]) - elif item in mapping and mapping[item] not in name: - name = name.replace(item, mapping[item]) - - return name, loaded_weight diff --git a/atom/models/utils.py b/atom/models/utils.py new file mode 100644 index 0000000000..6f07df0d20 --- /dev/null +++ b/atom/models/utils.py @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 + +import itertools +from dataclasses import dataclass, field +from typing import ( + Callable, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Protocol, + Set, + Tuple, + Union, + overload, +) + +import torch +import os + +import logging + +logger = logging.getLogger(__name__) + + +class LayerFn(Protocol): + + def __call__(self, prefix: str) -> torch.nn.Module: ... + + +class PPMissingLayer(torch.nn.Identity): + """ + A placeholder layer for missing layers in a pipeline parallel model. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + self.return_tuple = kwargs.get("return_tuple", False) + + def forward(self, *args, **kwargs): + """ + Return the first arg from args or the first value from kwargs. + + Wraps the input in a tuple if `self.return_tuple` is True. + """ + input = args[0] if args else next(iter(kwargs.values())) + return (input,) if self.return_tuple else input + + +def get_pp_indices( + num_hidden_layers: int, pp_rank: int, pp_size: int +) -> Tuple[int, int]: + """Try to evenly distribute layers across partitions. + + If the number of layers is not divisible by the number of partitions, + the remaining layers are evenly distributed across all but the last + partition. The last partition is excluded because it often contains an + additional norm layer and we are attempting to balance compute. + + If `pp_size > 2` and the number of remaining layers is + `0 < x <= pp_size - 2` then the remaining layers are evenly distributed + across the middle partitions. The first and last partitions are excluded + because they contain the input and output embeddings respectively and we + are attempting to reduce maximum memory consumption across partitions. + """ + partition_list_str = os.getenv("VLLM_PP_LAYER_PARTITION", None) + if partition_list_str is not None: + try: + partitions = [int(layer) for layer in partition_list_str.split(",")] + except ValueError as err: + raise ValueError( + "Invalid partition string: {}".format(partition_list_str) + ) from err + if len(partitions) != pp_size: + raise ValueError(f"{len(partitions)=} does not match {pp_size=}.") + if sum(partitions) != num_hidden_layers: + raise ValueError(f"{sum(partitions)=} does not match {num_hidden_layers=}.") + else: + layers_per_partition = num_hidden_layers // pp_size + partitions = [layers_per_partition for _ in range(pp_size)] + + if remaining_layers := num_hidden_layers % pp_size: + for i in range(2, remaining_layers + 2): + partitions[-i] += 1 + logger.info( + "Hidden layers were unevenly partitioned: [%s]. " + "This can be manually overridden using the " + "VLLM_PP_LAYER_PARTITION environment variable", + ",".join(str(p) for p in partitions), + ) + + start_layer = sum(partitions[:pp_rank]) + end_layer = start_layer + partitions[pp_rank] + + return (start_layer, end_layer) + + +def make_layers( + num_hidden_layers: int, + layer_fn: LayerFn, + prefix: str, +) -> Tuple[int, int, torch.nn.ModuleList]: + """Make a list of layers with the given layer function, taking + pipeline parallelism into account. + """ + from aiter.dist.parallel_state import get_pp_group + + start_layer, end_layer = get_pp_indices( + num_hidden_layers, get_pp_group().rank_in_group, get_pp_group().world_size + ) + modules = torch.nn.ModuleList( + [PPMissingLayer() for _ in range(start_layer)] + + [layer_fn(prefix=f"{prefix}.{idx}") for idx in range(start_layer, end_layer)] + + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)] + ) + return start_layer, end_layer, modules + + +# NOTE: don't use lru_cache here because it can prevent garbage collection +_model_to_pp_missing_layer_names: Dict[int, List[str]] = {} + + +def get_pp_missing_layer_names(model: torch.nn.Module) -> List[str]: + """Get the names of the missing layers in a pipeline parallel model.""" + model_id = id(model) + if model_id in _model_to_pp_missing_layer_names: + return _model_to_pp_missing_layer_names[model_id] + + missing_layer_names = [] + for name, module in model.named_modules(): + if isinstance(module, PPMissingLayer): + # NOTE: the trailing dot is used to match the prefix of the layer. + # without the dot, we could match a layer that is not missing, + # e.g., 'encoder.layer.1' would match 'encoder.layer.11' + missing_layer_names.append(name + ".") + _model_to_pp_missing_layer_names[model_id] = missing_layer_names + + return missing_layer_names + + +def is_pp_missing_parameter(name: str, model: torch.nn.Module) -> bool: + """Check if a parameter is missing in a pipeline parallel model.""" + if isinstance(model, PPMissingLayer): + return True + + return any( + name.startswith(missing_layer_name) + for missing_layer_name in get_pp_missing_layer_names(model) + ) + + +# cannot use msgspec.Struct here because Dynamo does not support it +@dataclass +class IntermediateTensors: + """For all pipeline stages except the last, we need to return the hidden + states and residuals to be sent to the next stage. This data structure + contains the hidden states and residuals for a request. + """ + + tensors: dict[str, torch.Tensor] + + def __init__(self, tensors): + # manually define this function, so that + # Dynamo knows `IntermediateTensors()` comes from this file. + # Otherwise, dataclass will generate this function by evaluating + # a string, and we will lose the information about the source file. + self.tensors = tensors + + def __getitem__(self, key: Union[str, slice]): + if isinstance(key, str): + return self.tensors[key] + elif isinstance(key, slice): + return self.__class__({k: v[key] for k, v in self.tensors.items()}) + + def __setitem__(self, key: str, value: torch.Tensor): + self.tensors[key] = value + + def items(self): + return self.tensors.items() + + def __len__(self): + return len(self.tensors) + + def __eq__(self, other: object): + return isinstance(other, self.__class__) and self + + def __repr__(self) -> str: + return f"IntermediateTensors(tensors={self.tensors})" + + +def make_empty_intermediate_tensors_factory(keys: List[str], hidden_size: int): + + def make_empty_intermediate_tensors( + batch_size: int, + dtype: torch.dtype, + device: torch.device, + ) -> IntermediateTensors: + return IntermediateTensors( + { + key: torch.zeros((batch_size, hidden_size), dtype=dtype, device=device) + for key in keys + } + ) + + return make_empty_intermediate_tensors + + +def maybe_prefix(prefix: str, name: str) -> str: + """Add a prefix to a name if the prefix is non-empty. + + Args: + prefix: The prefix to add. If empty, no prefix will be added. + name: The name to potentially prefix. + + Returns: + The string "prefix.name" if prefix was non-empty, otherwise just "name". + """ + return name if not prefix else f"{prefix}.{name}" + + +def extract_layer_index(layer_name: str) -> int: + """ + Extract the layer index from the module name. + Examples: + - "encoder.layers.0" -> 0 + - "encoder.layers.1.self_attn" -> 1 + - "2.self_attn" -> 2 + - "model.encoder.layers.0.sub.1" -> ValueError + """ + subnames = layer_name.split(".") + int_vals: List[int] = [] + for subname in subnames: + try: + int_vals.append(int(subname)) + except ValueError: + continue + assert len(int_vals) == 1, ( + f"layer name {layer_name} should" " only contain one integer" + ) + return int_vals[0] + + +def cast_overflow_tensors( + tensors: torch.Tensor, + offset: float = 1000, +) -> torch.Tensor: + if tensors.isinf().any() or tensors.isnan().any(): + clamp_value = torch.finfo(tensors.dtype).max - offset + tensors = torch.clamp(tensors, min=-clamp_value, max=clamp_value) + return tensors + + +def fast_topk(values, topk, dim): + if topk == 1: + # Use max along the specified dimension to get both value and index + return torch.max(values, dim=dim, keepdim=True) + else: + # Use topk for efficiency with larger k values + return torch.topk(values, topk, dim=dim) diff --git a/example.py b/example.py index c97c518d16..1c17b5bf5b 100644 --- a/example.py +++ b/example.py @@ -23,12 +23,16 @@ "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." ) -parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=2) +parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1) parser.add_argument( "--enforce-eager", action="store_true", help="Enforce eager mode execution." ) +parser.add_argument( + "--enable_prefix_caching", action="store_true", help="Enable prefix caching." +) + parser.add_argument("--port", type=int, default=8006, help="API server port") From d76687cebd62833d901cfd57e941dbb4ab56449b Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Wed, 27 Aug 2025 02:01:48 +0000 Subject: [PATCH 024/119] update graph cap info --- atom/model_engine/model_runner.py | 41 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 7b9c12e311..c877ebf522 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -2,9 +2,10 @@ import os import torch import torch.distributed as dist +import tqdm from aiter import dtypes from aiter import init_dist_env, destroy_dist_env -from aiter.dist.parallel_state import graph_capture +from aiter.dist.parallel_state import graph_capture, get_tensor_model_parallel_rank from multiprocessing.synchronize import Event from multiprocessing.shared_memory import SharedMemory from atom.config import Config @@ -391,31 +392,37 @@ def capture_cudagraph(self): i for i in range(64, cuda_graph_sizes[0] + 1, 64) ] elif len(cuda_graph_sizes) > 1: - self.graph_bs = sorted(cuda_graph_sizes) - + self.graph_bs = sorted(cuda_graph_sizes, reverse=True) self.graphs = {} self.graph_pool = None - for bs in reversed(self.graph_bs): - graph = torch.cuda.CUDAGraph() - set_context( - False, - slot_mapping=slot_mapping[:bs], - context_lens=context_lens[:bs], - block_tables=block_tables[:bs], + with graph_capture() as gc: + capture_range = ( + tqdm.tqdm(self.graph_bs) + if get_tensor_model_parallel_rank() == 0 + else self.graph_bs ) + for bs in capture_range: + if get_tensor_model_parallel_rank() == 0: + capture_range.set_description(f"Capturing {bs=}") + graph = torch.cuda.CUDAGraph() + set_context( + False, + slot_mapping=slot_mapping[:bs], + context_lens=context_lens[:bs], + block_tables=block_tables[:bs], + ) - outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup + outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup - with graph_capture() as gc: with torch.cuda.graph(graph, self.graph_pool, stream=gc.stream): outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # capture - if self.graph_pool is None: - self.graph_pool = graph.pool() - self.graphs[bs] = graph - torch.cuda.synchronize() - reset_context() + if self.graph_pool is None: + self.graph_pool = graph.pool() + self.graphs[bs] = graph + torch.cuda.synchronize() + reset_context() self.graph_vars = dict( input_ids=input_ids, From a3c5ddd5b0a6012d9f422b043952a81243353207 Mon Sep 17 00:00:00 2001 From: chenjun Date: Mon, 1 Sep 2025 14:59:30 +0000 Subject: [PATCH 025/119] enable fp8 quant linear (ptpc, per tensor, per block ) --- atom/config.py | 19 +- atom/model_engine/model_runner.py | 6 +- atom/model_loader/loader.py | 16 +- atom/model_ops/attention.py | 2 +- atom/model_ops/linear.py | 134 +++++++-- atom/model_ops/moe.py | 478 +++++++++++++++++------------- atom/model_ops/topK.py | 195 ++++++++++-- atom/model_ops/utils.py | 74 +++++ atom/models/llama.py | 27 +- atom/models/mixtral.py | 390 ++++++++++++++++++++++++ atom/models/qwen3.py | 33 ++- 11 files changed, 1075 insertions(+), 299 deletions(-) create mode 100644 atom/model_ops/utils.py create mode 100644 atom/models/mixtral.py diff --git a/atom/config.py b/atom/config.py index b5f929589a..fb1ca7deef 100644 --- a/atom/config.py +++ b/atom/config.py @@ -42,12 +42,13 @@ def __post_init__(self): class QuantizationConfig(dict): def __init__( - self, quant_type=QuantType.No, quant_dtype=torch.bfloat16, quant_name="" + self, quant_type=QuantType.No, quant_dtype=torch.bfloat16, is_dynamic=True, quant_name="" ): super().__init__() self["quant_type"] = quant_type if quant_type is not None else QuantType.No self["quant_dtype"] = quant_dtype if quant_dtype is not None else torch.bfloat16 self["quant_name"] = quant_name + self["is_dynamic"] = is_dynamic def get_name(self): return self["quant_name"] @@ -65,7 +66,7 @@ def get_quant_config(config: PretrainedConfig) -> QuantizationConfig: quant_method = orig_quant_config.get("quant_method", None) RE_QUANT_BLOCKSIZE = r"\'(?:group_size|weight_block_size)\'\:\s*(?:\[\n*)\s*(\d+)," orig_quant_config_str = str(orig_quant_config) - if quant_method == "compressed-tensors" or 'channel",' in orig_quant_config_str: + if quant_method == "compressed-tensors" or "channel'," in orig_quant_config_str: quant_type = QuantType.per_Token elif group_size := re.search(RE_QUANT_BLOCKSIZE, orig_quant_config_str): group_size = int(group_size.group(1)) @@ -80,8 +81,8 @@ def get_quant_config(config: PretrainedConfig) -> QuantizationConfig: RE_QUANT_DTYPE = r"\'(?:d?type|weight_dtype|quant_method)\'\:\s*\'(\w+)\'" quant_dtype = None m = re.search(RE_QUANT_DTYPE, orig_quant_config_str) - if m and m.group(1).lower() in ["fp8", "fp4", "int8", "int4"]: - dtype = m.group(1).lower() + if m and m.group(1).lower() in ["fp8", "fp4", "int8", "int4", "fp8_e4m3"]: + dtype = m.group(1).lower().split("_")[0] if dtype.endswith("4"): dtype += "x2" quant_dtype = d_dtypes[dtype] @@ -102,7 +103,13 @@ def get_quant_config(config: PretrainedConfig) -> QuantizationConfig: assert ( quant_dtype is not None ), f"Cannot parse quant dtype from {orig_quant_config_str}" - return QuantizationConfig(quant_type, quant_dtype) + + RE_STATIC_QUANT = r"\'(?:activation_scheme)\'\:\s*\'(static)\'" + if re.search(RE_STATIC_QUANT, orig_quant_config_str): + is_dynamic = False + else: + is_dynamic = True + return QuantizationConfig(quant_type, quant_dtype, is_dynamic) @dataclass @@ -123,12 +130,14 @@ class Config: port: int = 8006 torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) compilation_config: CompilationConfig = field(default_factory=CompilationConfig) + quant_config: QuantizationConfig = field(default_factory=lambda: QuantizationConfig()) def __post_init__(self): # assert os.path.isdir(self.model) assert self.kvcache_block_size % 16 == 0 assert 1 <= self.tensor_parallel_size <= 8 self.hf_config = AutoConfig.from_pretrained(self.model) + self.quant_config = get_quant_config(self.hf_config) self.max_model_len = min( self.max_model_len, self.hf_config.max_position_embeddings ) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index c877ebf522..995dc4d4f9 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -13,12 +13,14 @@ from atom.model_ops.sampler import Sampler from atom.models.qwen3 import Qwen3ForCausalLM from atom.models.llama import LlamaForCausalLM +# from atom.models.mixtral import MixtralForCausalLM from atom.utils.context import set_context, get_context, reset_context from atom.model_loader.loader import load_model suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, "LlamaForCausalLM": LlamaForCausalLM, + # "MixtralForCausalLM": MixtralForCausalLM, } @@ -41,9 +43,7 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") - self.model = suppot_model_arch_dict[hf_config.architectures[0]]( - hf_config, config.kv_cache_dtype - ) + self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) if self.config.compilation_config.level == 3: self.model = torch.compile(self.model, fullgraph=True, backend="eager") load_model(self.model, config.model) diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index f492305d74..be84b51d11 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -25,17 +25,29 @@ def load_model(model: nn.Module, model_name_or_path: str): print(f"Loading weights from {file}") with safe_open(file, "pt", "cpu") as f: for weight_name in f.keys(): + if weight_name.endswith( + "kv_scale" + ): + continue + weight_tensor = f.get_tensor(weight_name) + if "weight_scale_inv" in weight_name: + weight_name = weight_name.replace( + "weight_scale_inv", "weight_scale" + ) for k in packed_modules_mapping: if k in weight_name: v, shard_id = packed_modules_mapping[k] param_name = weight_name.replace(k, v) param = model.get_parameter(param_name) weight_loader = getattr(param, "weight_loader") - weight_loader(param, f.get_tensor(weight_name), shard_id) + weight_loader(param, weight_tensor, shard_id) break else: param = model.get_parameter(weight_name) weight_loader = getattr( param, "weight_loader", default_weight_loader ) - weight_loader(param, f.get_tensor(weight_name)) + weight_loader(param, weight_tensor) + for _, module in model.named_modules(): + if hasattr(module, "process_weights_after_loading"): + module.process_weights_after_loading() diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 0fb1a4d8db..a78c0a823a 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -6,7 +6,7 @@ import aiter from aiter.paged_attn import PagedAttention from atom.utils.context import get_context -from flash_attn import flash_attn_with_kvcache +# from flash_attn import flash_attn_with_kvcache class Attention(nn.Module): diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 0a25233486..1ca72e8213 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -14,10 +14,11 @@ ) from aiter.ops.shuffle import shuffle_weight from aiter.tuned_gemm import tgemm -from aiter import get_hip_quant +from aiter import get_hip_quant, get_triton_quant from typing import Optional, Callable from functools import partial as functools_partial from atom.config import QuantizationConfig +from atom.model_ops.utils import normalize_e4m3fn_to_e4m3fnuz, requantize_with_max_scale def divide(numerator, denominator): @@ -30,7 +31,7 @@ class LinearBase(nn.Module): def __init__( self, input_size: int, - output_size: int, + output_size: int | list[int], tp_dim: int | None = None, bias: bool = False, quant_config: Optional[QuantizationConfig] = None, @@ -41,14 +42,22 @@ def __init__( params_dtype = quant_config["quant_dtype"] super().__init__() self.input_size = input_size - self.output_size = output_size + self.output_size = ( + output_size if isinstance(output_size, int) else sum(output_size) + ) self.tp_dim = tp_dim self.tp_rank = get_tp_group().rank self.tp_size = get_tp_group().world_size + self.output_partition_sizes = ( + output_size if isinstance(output_size, list) else [output_size] + ) if tp_dim == 1: self.input_size = divide(input_size, self.tp_size) elif tp_dim == 0: - self.output_size = divide(output_size, self.tp_size) + self.output_size = divide(self.output_size, self.tp_size) + self.output_partition_sizes = [ + divide(s, self.tp_size) for s in self.output_partition_sizes + ] self.weight = nn.Parameter( torch.empty((self.output_size, self.input_size), dtype=params_dtype), requires_grad=False, @@ -64,27 +73,25 @@ def __init__( self.params_dtype = params_dtype if quant_type != QuantType.No: if quant_type == QuantType.per_Tensor: - self.weight.weight_loader_process = functools_partial( - self.weight_loader_process, - post_process_func=lambda x: shuffle_weight(x, (16, 16)), - ) self.weight_scale = nn.Parameter( - torch.empty(1, dtype=dtypes.fp32), requires_grad=False + torch.empty(len(self.output_partition_sizes), 1, dtype=dtypes.fp32), + requires_grad=False, ) - elif quant_type == QuantType.per_Token: - if params_dtype == dtypes.i8: - self.weight.weight_loader_process = self.weight_loader_process - else: - self.weight.weight_loader_process = functools_partial( - self.weight_loader_process, - post_process_func=lambda x: shuffle_weight(x, (16, 16)), + if not quant_config["is_dynamic"]: + self.input_scale = nn.Parameter( + torch.empty( + len(self.output_partition_sizes), 1, dtype=dtypes.fp32 + ), + requires_grad=False, ) + self.input_scale.weight_loader_process = self.weight_loader_process + self.input_scale.weight_loader = self.weight_loader + elif quant_type == QuantType.per_Token: self.weight_scale = nn.Parameter( torch.empty(self.output_size, 1, dtype=dtypes.fp32), requires_grad=False, ) elif quant_type == QuantType.per_1x128: - self.weight.weight_loader_process = self.weight_loader_process self.weight_scale = nn.Parameter( torch.empty( divide(self.output_size, 128), @@ -94,10 +101,6 @@ def __init__( requires_grad=False, ) elif quant_type == QuantType.per_1x32: - self.weight.weight_loader_process = functools_partial( - self.weight_loader_process, - post_process_func=lambda x: shuffle_weight(x, (16, 16)), - ) self.weight_scale = nn.Parameter( torch.empty( self.output_size, @@ -106,6 +109,7 @@ def __init__( ), requires_grad=False, ) + self.weight.weight_loader_process = self.weight_loader_process self.weight_scale.weight_loader_process = self.weight_loader_process else: self.weight.weight_loader_process = self.weight_loader_process @@ -115,6 +119,7 @@ def __init__( self.bias.weight_loader = self.weight_loader if self.weight_scale is not None: self.weight_scale.weight_loader = self.weight_loader + self.need_normalize_e4m3fn_to_e4m3fnuz = params_dtype == torch.float8_e4m3fnuz @staticmethod def weight_loader_process( @@ -122,12 +127,45 @@ def weight_loader_process( loaded_weight: torch.Tensor, post_process_func: Callable = lambda a: a, ): + if ( + param.data.dtype != loaded_weight.dtype + and param.data.element_size() == loaded_weight.element_size() + ): + param.data = param.data.view(loaded_weight.dtype) param.data.copy_(post_process_func(loaded_weight)) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data param.weight_loader_process(param_data, loaded_weight) + def process_weights_after_loading(self): + if ( + self.quant_type == QuantType.per_Tensor + and len(self.output_partition_sizes) > 1 + ): + weight_scale, weight = requantize_with_max_scale( + weight=self.weight.data, + weight_scale=self.weight_scale.data, + logical_widths=self.output_partition_sizes, + normalize_e4m3fn_to_e4m3fnuz=self.need_normalize_e4m3fn_to_e4m3fnuz, + ) + self.weight.data = weight + self.weight_scale.data = weight_scale.view(-1) + if hasattr(self, "input_scale"): + self.input_scale.data = ( + self.input_scale.data.max() * 2.0 + if self.need_normalize_e4m3fn_to_e4m3fnuz + else self.input_scale.data.max() + ) + elif self.need_normalize_e4m3fn_to_e4m3fnuz: + self.weight.data, self.weight_scale.data, _ = normalize_e4m3fn_to_e4m3fnuz( + self.weight.data, self.weight_scale.data + ) + if ( + self.quant_type == QuantType.per_Token and self.params_dtype == dtypes.fp8 + ) or (self.quant_type == QuantType.per_1x32): + self.weight.data = shuffle_weight(self.weight.data, (16, 16)) + def forward( self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 ) -> torch.Tensor: @@ -136,7 +174,11 @@ def forward( else: if x_scale is None: quant_func = get_hip_quant(self.quant_type) - x, x_scale = quant_func(x, quant_dtype=self.params_dtype) + x, x_scale = quant_func( + x, + quant_dtype=self.params_dtype, + scale=getattr(self, "input_scale", None), + ) if self.quant_type.value == QuantType.per_Tensor.value: y = tgemm.mm( x, @@ -167,8 +209,10 @@ def forward( ) elif self.quant_type.value == QuantType.per_1x128.value: y = gemm_a8w8_blockscale( - x, self.weight, x_scale, self.weight_scale, self.bias, dtype=otype + x, self.weight, x_scale, self.weight_scale, dtype=otype ) + if self.bias is not None: + y += self.bias elif self.quant_type.value == QuantType.per_1x32.value: m = x.view(-1, x.size(-1)).shape[0] y = torch.empty( @@ -240,7 +284,7 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param.weight_loader_process(param_data, loaded_weight) -class MergedColumnParallelLinear(ColumnParallelLinear): +class MergedColumnParallelLinear(LinearBase): def __init__( self, @@ -253,8 +297,9 @@ def __init__( self.output_sizes = output_sizes super().__init__( input_size, - sum(output_sizes), - bias, + output_sizes, + tp_dim=0, + bias=bias, quant_config=quant_config, ) @@ -264,6 +309,17 @@ def weight_loader( param_data = param.data shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size shard_size = self.output_sizes[loaded_shard_id] // self.tp_size + if param is getattr(self, "weight_scale", None) or param is getattr( + self, "input_scale", None + ): + if self.quant_type == QuantType.per_1x128: + shard_offset //= 128 + shard_size //= 128 + elif self.quant_type == QuantType.per_Tensor: + loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) + shard_offset = loaded_shard_id + shard_size = 1 + param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] param.weight_loader_process(param_data, loaded_weight) @@ -288,13 +344,16 @@ def __init__( self.num_heads = divide(self.total_num_heads, tp_size) self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) input_size = hidden_size - output_size = ( - self.total_num_heads + 2 * self.total_num_kv_heads - ) * self.head_size + output_sizes = [ + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ] + super().__init__( input_size, - output_size, - bias, + output_sizes, + bias=bias, quant_config=quant_config, ) @@ -314,6 +373,17 @@ def weight_loader( shard_offset = ( self.num_heads * self.head_size + self.num_kv_heads * self.head_size ) + if param is getattr(self, "weight_scale", None) or param is getattr( + self, "input_scale", None + ): + if self.quant_type == QuantType.per_1x128: + shard_offset //= 128 + shard_size //= 128 + elif self.quant_type == QuantType.per_Tensor: + loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) + shard_offset = ["q", "k", "v"].index(loaded_shard_id) + shard_size = 1 + param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] param.weight_loader_process(param_data, loaded_weight) @@ -341,6 +411,8 @@ def __init__( def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.view(1, 1) if loaded_weight.size(self.tp_dim) == 1 and self.tp_size > 1: loaded_weight = loaded_weight.repeat(1, self.tp_size) start_idx = self.tp_rank * shard_size diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 7121420dcd..55e73fff30 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -2,34 +2,40 @@ import torch from torch import nn from typing import Callable, List, Optional, Tuple -from aiter.dist.parallel_state import get_tensor_model_parallel_world_size, get_tensor_model_parallel_rank -from aiter.dist.communication_op import tensor_model_parallel_all_reduce +from aiter.dist.parallel_state import get_tp_group from transformers import PretrainedConfig from aiter import QuantType, dtypes +from atom.config import QuantizationConfig +from atom.model_ops.topK import rocm_aiter_grouped_topk as grouped_topk +from atom.model_ops.topK import rocm_aiter_topk_softmax as fused_topk +from atom.model_ops.topK import ( + init_aiter_topK_meta_data, + is_rocm_aiter_fusion_shared_expert_enabled, +) def determine_expert_map( - ep_size: int, ep_rank: int, - global_num_experts: int) -> Tuple[int, Optional[torch.Tensor]]: + ep_size: int, ep_rank: int, global_num_experts: int +) -> Tuple[int, Optional[torch.Tensor]]: + """ + Calculates how many experts should be assigned to each rank for EP and + creates a mapping from global to local expert index. Experts are + distributed evenly across ranks. Any remaining are assigned to the + last rank. + + Args: + ep_size (int): The size of the expert parallel group + global_num_experts (int): The total number of experts in the model. + + Returns: + Tuple[int, Optional[torch.Tensor]]: A tuple containing: + - local_num_experts (int): The number of experts assigned + to the current rank. + - expert_map (Optional[torch.Tensor]): A tensor of shape + (global_num_experts,) mapping from global to local index. + Contains -1 for experts not assigned to the current rank. + Returns None if ep_size is 1. """ - Calculates how many experts should be assigned to each rank for EP and - creates a mapping from global to local expert index. Experts are - distributed evenly across ranks. Any remaining are assigned to the - last rank. - - Args: - ep_size (int): The size of the expert parallel group - global_num_experts (int): The total number of experts in the model. - - Returns: - Tuple[int, Optional[torch.Tensor]]: A tuple containing: - - local_num_experts (int): The number of experts assigned - to the current rank. - - expert_map (Optional[torch.Tensor]): A tensor of shape - (global_num_experts,) mapping from global to local index. - Contains -1 for experts not assigned to the current rank. - Returns None if ep_size is 1. - """ assert ep_size > 0 if ep_size == 1: return (global_num_experts, None) @@ -37,19 +43,20 @@ def determine_expert_map( local_num_experts = global_num_experts // ep_size # Create a tensor of size num_experts filled with -1 - expert_map = torch.full((global_num_experts, ), -1, dtype=torch.int32) + expert_map = torch.full((global_num_experts,), -1, dtype=torch.int32) # Create a expert map for the local experts if ep_rank < (ep_size - 1): # Each non-last rank gets local_num_experts experts. - expert_map[ep_rank * local_num_experts: - (ep_rank + 1) * local_num_experts] = \ + expert_map[ep_rank * local_num_experts : (ep_rank + 1) * local_num_experts] = ( torch.arange(0, local_num_experts, dtype=torch.int32) + ) else: # All remaining experts are assigned to the last rank. - local_num_experts = (global_num_experts - ep_rank * local_num_experts) + local_num_experts = global_num_experts - ep_rank * local_num_experts - expert_map[-local_num_experts:] = \ - torch.arange(0, local_num_experts, dtype=torch.int32) + expert_map[-local_num_experts:] = torch.arange( + 0, local_num_experts, dtype=torch.int32 + ) return (local_num_experts, expert_map) @@ -80,13 +87,12 @@ def __init__( top_k: int, hidden_size: int, intermediate_size: int, - params_dtype: Optional[torch.dtype] = None, reduce_results: bool = False, renormalize: bool = True, use_grouped_topk: bool = False, num_expert_group: Optional[int] = None, topk_group: Optional[int] = None, - quant_type = QuantType.No, + quant_config: Optional[QuantizationConfig] = None, tp_size: Optional[int] = None, ep_size: Optional[int] = None, dp_size: Optional[int] = None, @@ -100,21 +106,18 @@ def __init__( ): super().__init__() - if params_dtype is None: - params_dtype = torch.get_default_dtype() - self.params_dtype = params_dtype + self.params_dtype = ( + quant_config["quant_dtype"] if quant_config else torch.get_default_dtype() + ) # Note: here we guard against accessing the TP and DP groups when # uninitialized (this happens when testing) - self.tp_size = (tp_size if tp_size is not None else - get_tensor_model_parallel_world_size()) - tp_rank = 0 if self.tp_size == 1 else get_tensor_model_parallel_rank() + self.tp_size = tp_size if tp_size is not None else get_tp_group().world_size + tp_rank = 0 if self.tp_size == 1 else get_tp_group().rank_in_group self.global_num_experts = num_experts # Use expert parallelism instead of tensor parallelism? - vllm_config = get_current_vllm_config() - use_ep = (vllm_config.parallel_config.enable_expert_parallel - and self.tp_size > 1) + use_ep = self.tp_size > 1 if use_ep: # Set TP size to 1 to adjust for EP and adjust EP size and rank @@ -127,7 +130,8 @@ def __init__( self.local_num_experts, self.expert_map = determine_expert_map( ep_size=self.ep_size, ep_rank=self.ep_rank, - global_num_experts=self.global_num_experts) + global_num_experts=self.global_num_experts, + ) else: # Adjust TP size for DP attention self.tp_rank = tp_rank + self.tp_size * self.dp_rank @@ -139,24 +143,50 @@ def __init__( self.top_k = top_k self.global_num_experts = num_experts - self.num_fused_shared_experts = config.n_shared_experts if config is not None and is_rocm_aiter_fusion_shared_expert_enabled() else 0 - self.routed_scaling_factor = config.routed_scaling_factor if config is not None and config.torch_dtype != torch.float16 else 1.0 + self.num_fused_shared_experts = ( + config.n_shared_experts + if config is not None and is_rocm_aiter_fusion_shared_expert_enabled() + else 0 + ) + self.routed_scaling_factor = ( + config.routed_scaling_factor + if config is not None and config.torch_dtype != torch.float16 + else 1.0 + ) self.expert_mask = None - if use_ep and is_rocm_aiter_moe_enabled(): - expert_mask = torch.ones((self.global_num_experts + self.num_fused_shared_experts + 1,), dtype=torch.int32) + if use_ep: + expert_mask = torch.ones( + (self.global_num_experts + self.num_fused_shared_experts + 1,), + dtype=torch.int32, + ) expert_mask[-1] = 0 - expert_mask[:self.global_num_experts] = self.expert_map > -1 + expert_mask[: self.global_num_experts] = self.expert_map > -1 self.expert_mask = expert_mask - self.expert_map = torch.cat((self.expert_map, torch.tensor([self.local_num_experts + i for i in range(self.num_fused_shared_experts)],dtype=torch.int32)), dim=0) - if is_rocm_aiter_fusion_shared_expert_enabled() and self.num_fused_shared_experts > 0: + self.expert_map = torch.cat( + ( + self.expert_map, + torch.tensor( + [ + self.local_num_experts + i + for i in range(self.num_fused_shared_experts) + ], + dtype=torch.int32, + ), + ), + dim=0, + ) + if ( + is_rocm_aiter_fusion_shared_expert_enabled() + and self.num_fused_shared_experts > 0 + ): init_aiter_topK_meta_data( n_routed_experts=self.global_num_experts, - n_shared_experts= self.num_fused_shared_experts, + n_shared_experts=self.num_fused_shared_experts, top_k=self.top_k, tp_rank=self.ep_rank if use_ep else tp_rank, tp_size=self.ep_size if use_ep else tp_size, shared_experts_score=1.0, - max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens, + max_num_tokens=c, is_EP=use_ep, ) if is_rocm_aiter_fusion_shared_expert_enabled(): @@ -178,24 +208,27 @@ def __init__( self.activation = activation if self.scoring_func != "softmax" and not self.use_grouped_topk: - raise ValueError("Only softmax scoring function is supported for " - "non-grouped topk.") + raise ValueError( + "Only softmax scoring function is supported for " "non-grouped topk." + ) self.apply_router_weight_on_input = apply_router_weight_on_input moe_quant_params = { "num_experts": self.local_num_experts, "hidden_size": hidden_size, - "intermediate_size_per_partition": - self.intermediate_size_per_partition, + "intermediate_size_per_partition": self.intermediate_size_per_partition, "params_dtype": params_dtype, "weight_loader": self.weight_loader, } self.quant_method.create_weights(layer=self, **moe_quant_params) - def _load_per_tensor_weight_scale(self, shard_id: str, - param: torch.nn.Parameter, - loaded_weight: torch.Tensor, - expert_id: int): + def _load_per_tensor_weight_scale( + self, + shard_id: str, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + expert_id: int, + ): param_data = param.data # for per tensor weight quantization if shard_id in ("w1", "w3"): @@ -207,13 +240,15 @@ def _load_per_tensor_weight_scale(self, shard_id: str, elif shard_id == "w2": param_data[expert_id] = loaded_weight - def _load_model_weight_or_group_weight_scale(self, - shard_dim: int, - expert_data: torch.Tensor, - shard_id: str, - loaded_weight: torch.Tensor, - tp_rank: int, - load_full_w2: bool = False): + def _load_model_weight_or_group_weight_scale( + self, + shard_dim: int, + expert_data: torch.Tensor, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + load_full_w2: bool = False, + ): """ Load grouped weight scales for group quantization or model weights :param shard_dim: dimension to shard @@ -226,40 +261,57 @@ def _load_model_weight_or_group_weight_scale(self, if shard_id == "w2": # In the case where we have actorder/g_idx, we do not partition the # w2 scales, as indicated by `load_full` argument, for all tp cases - self._load_w2(shard_dim=shard_dim, - loaded_weight=loaded_weight, - expert_data=expert_data, - tp_rank=tp_rank, - load_full=load_full_w2) + self._load_w2( + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + load_full=load_full_w2, + ) elif shard_id in ("w1", "w3"): - self._load_w13(shard_id=shard_id, - shard_dim=shard_dim, - loaded_weight=loaded_weight, - expert_data=expert_data, - tp_rank=tp_rank) - - def _load_per_channel_weight_scale(self, expert_data: torch.Tensor, - shard_dim: int, shard_id: str, - loaded_weight: torch.Tensor, - tp_rank: int): + self._load_w13( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + + def _load_per_channel_weight_scale( + self, + expert_data: torch.Tensor, + shard_dim: int, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + ): # for per channel weight quantization if shard_id == "w2": expert_data.copy_(loaded_weight) elif shard_id in ("w1", "w3"): - self._load_w13(shard_id=shard_id, - shard_dim=shard_dim, - loaded_weight=loaded_weight, - expert_data=expert_data, - tp_rank=tp_rank) + self._load_w13( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) - def _load_w13(self, expert_data: torch.Tensor, shard_dim: int, - shard_id: str, loaded_weight: torch.Tensor, tp_rank: int): + def _load_w13( + self, + expert_data: torch.Tensor, + shard_dim: int, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + ): # Index the loaded weight for tp sharding. # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim shard_size = expert_data.shape[shard_dim] // 2 - loaded_weight = loaded_weight.narrow(shard_dim, shard_size * tp_rank, - shard_size) + loaded_weight = loaded_weight.narrow( + shard_dim, shard_size * tp_rank, shard_size + ) # Narrow parameter and load. # w1, gate_proj: Load into first logical weight of w13. if shard_id == "w1": @@ -270,39 +322,50 @@ def _load_w13(self, expert_data: torch.Tensor, shard_dim: int, expert_data = expert_data.narrow(shard_dim, shard_size, shard_size) expert_data.copy_(loaded_weight) - def _load_w2(self, - expert_data: torch.Tensor, - shard_dim: int, - loaded_weight: torch.Tensor, - tp_rank: int, - load_full: bool = False): + def _load_w2( + self, + expert_data: torch.Tensor, + shard_dim: int, + loaded_weight: torch.Tensor, + tp_rank: int, + load_full: bool = False, + ): # Index the loaded weight for tp sharding. # down_proj: "RowParallel" so tp sharding on input_dim # Narrow parameter and load. shard_size = expert_data.shape[shard_dim] if not load_full: - loaded_weight = loaded_weight.narrow(shard_dim, - shard_size * tp_rank, - shard_size) + loaded_weight = loaded_weight.narrow( + shard_dim, shard_size * tp_rank, shard_size + ) # w2, down_proj: Load into only logical weight of w2. expert_data.copy_(loaded_weight) - def _load_single_value(self, param: torch.nn.Parameter, - loaded_weight: torch.Tensor, expert_id: int): + def _load_single_value( + self, param: torch.nn.Parameter, loaded_weight: torch.Tensor, expert_id: int + ): param_data = param.data # Input scales can be loaded directly and should be equal. param_data[expert_id] = loaded_weight - def _load_g_idx(self, shard_id: str, expert_data: torch.Tensor, - shard_dim: int, loaded_weight: torch.Tensor, tp_rank: int): + def _load_g_idx( + self, + shard_id: str, + expert_data: torch.Tensor, + shard_dim: int, + loaded_weight: torch.Tensor, + tp_rank: int, + ): if shard_id == "w2": - self._load_w2(shard_dim=shard_dim, - loaded_weight=loaded_weight, - expert_data=expert_data, - tp_rank=tp_rank) + self._load_w2( + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) else: assert shard_id in ("w1", "w3") expert_data.copy_(loaded_weight) @@ -312,9 +375,14 @@ def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int: return expert_id return self.expert_map[expert_id].item() - def weight_loader(self, param: torch.nn.Parameter, - loaded_weight: torch.Tensor, weight_name: str, - shard_id: str, expert_id: int) -> None: + def weight_loader( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + ) -> None: expert_id = self._map_global_expert_id_to_local_expert_id(expert_id) if expert_id == -1: @@ -324,17 +392,16 @@ def weight_loader(self, param: torch.nn.Parameter, # TODO (mgoin): check self.quant_method.quant_config.quant_format # against known CompressionFormat enum values that have this quality if self.quant_method.__class__.__name__ in ( - "CompressedTensorsWNA16MarlinMoEMethod", - "CompressedTensorsWNA16MoEMethod"): + "CompressedTensorsWNA16MarlinMoEMethod", + "CompressedTensorsWNA16MoEMethod", + ): loaded_weight = loaded_weight.t().contiguous() if shard_id not in ("w1", "w2", "w3"): - raise ValueError(f"shard_id must be ['w1','w2','w3'] but " - f"got {shard_id}.") + raise ValueError( + f"shard_id must be ['w1','w2','w3'] but " f"got {shard_id}." + ) - WEIGHT_SCALE_SUPPORTED = [ - e.value for e in FusedMoeWeightScaleSupported - ] # Fetch the dim to shard the parameter/loaded weight # based on the shard id. This will be whatever # dimension intermediate_size_per_partition is used. @@ -359,60 +426,57 @@ def weight_loader(self, param: torch.nn.Parameter, if full_load: shard_dim += 1 - # Materialize GGUF UninitializedParameter - if is_gguf_weight and isinstance(param, UninitializedParameter): - final_shape = list(loaded_weight.shape) - if shard_id in ["w1", "w3"]: - final_shape[1] *= 2 - final_shape[shard_dim] = final_shape[shard_dim] // self.tp_size - param.materialize(final_shape, dtype=loaded_weight.dtype) - expert_data = param.data if full_load else param.data[expert_id] # Case input scale: input_scale loading is only supported for fp8 if "input_scale" in weight_name: # this is needed for compressed-tensors only loaded_weight = loaded_weight.to(param.data.device) - if param.data[expert_id] != 1 and (param.data[expert_id] - - loaded_weight).abs() > 1e-5: + if ( + param.data[expert_id] != 1 + and (param.data[expert_id] - loaded_weight).abs() > 1e-5 + ): raise ValueError( "input_scales of w1 and w3 of a layer " f"must be equal. But got {param.data[expert_id]} " - f"vs. {loaded_weight}") + f"vs. {loaded_weight}" + ) - self._load_single_value(param=param, - loaded_weight=loaded_weight, - expert_id=expert_id) + self._load_single_value( + param=param, loaded_weight=loaded_weight, expert_id=expert_id + ) return # Case g_idx if "g_idx" in weight_name: - self._load_g_idx(shard_dim=0, - shard_id=shard_id, - loaded_weight=loaded_weight, - expert_data=expert_data, - tp_rank=self.tp_rank) + self._load_g_idx( + shard_dim=0, + shard_id=shard_id, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=self.tp_rank, + ) return # Case weight scales, zero_points and offset - if ("scale" in weight_name or "zero" in weight_name - or "offset" in weight_name): + if "scale" in weight_name or "zero" in weight_name or "offset" in weight_name: # load the weight scales and zp based on the quantization scheme # supported weight scales/zp can be found in # FusedMoeWeightScaleSupported # TODO @dsikka: once hardened, refactor to use vLLM Parameters # specific to each case - quant_method = getattr(param, "quant_method", None) - if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value: + quant_method = self.quant_config["quant_type"] + if quant_method == QuantType.per_Token: self._load_per_channel_weight_scale( shard_id=shard_id, shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=self.tp_rank) + tp_rank=self.tp_rank, + ) elif quant_method in [ - FusedMoeWeightScaleSupported.GROUP.value, - FusedMoeWeightScaleSupported.BLOCK.value, + QuantType.per_1x128, + QuantType.per_1x32, ]: self._load_model_weight_or_group_weight_scale( shard_id=shard_id, @@ -420,23 +484,23 @@ def weight_loader(self, param: torch.nn.Parameter, loaded_weight=loaded_weight, expert_data=expert_data, tp_rank=self.tp_rank, - load_full_w2=getattr(param, "load_full_w2", False)) - elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: - self._load_per_tensor_weight_scale(shard_id=shard_id, - param=param, - loaded_weight=loaded_weight, - expert_id=expert_id) - else: - raise ValueError( - f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}") + load_full_w2=getattr(param, "load_full_w2", False), + ) + elif quant_method == QuantType.per_Tensor: + self._load_per_tensor_weight_scale( + shard_id=shard_id, + param=param, + loaded_weight=loaded_weight, + expert_id=expert_id, + ) return # Case weight_shape if "weight_shape" in weight_name: # only required by compressed-tensors - self._load_single_value(param=param, - loaded_weight=loaded_weight, - expert_id=expert_id) + self._load_single_value( + param=param, loaded_weight=loaded_weight, expert_id=expert_id + ) return # Case model weights @@ -446,23 +510,25 @@ def weight_loader(self, param: torch.nn.Parameter, shard_dim=shard_dim, loaded_weight=loaded_weight, expert_data=expert_data, - tp_rank=self.tp_rank) + tp_rank=self.tp_rank, + ) return @staticmethod - def select_experts(hidden_states: torch.Tensor, - router_logits: torch.Tensor, - top_k: int, - use_grouped_topk: bool, - renormalize: bool, - topk_group: Optional[int] = None, - num_expert_group: Optional[int] = None, - custom_routing_function: Optional[Callable] = None, - scoring_func: str = "softmax", - e_score_correction_bias: Optional[torch.Tensor] = None, - num_fused_shared_experts: int = 0, - routed_scaling_factor: float = 1.0): - from aiter.fused_moe import fused_topk + def select_experts( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + use_grouped_topk: bool, + renormalize: bool, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: float = 1.0, + ): # DeekSeekv2 uses grouped_top_k if use_grouped_topk: @@ -477,44 +543,27 @@ def select_experts(hidden_states: torch.Tensor, topk_group=topk_group, scoring_func=scoring_func, e_score_correction_bias=e_score_correction_bias, - if is_rocm_aiter_moe_enabled() else {}), - **({"num_fused_shared_experts":num_fused_shared_experts} if is_rocm_aiter_fusion_shared_expert_enabled() else {})) - elif custom_routing_function is None: - topk_weights, topk_ids = fused_topk(hidden_states=hidden_states, - gating_output=router_logits, - topk=top_k, - renormalize=renormalize) + routed_scaling_factor=routed_scaling_factor, + num_fused_shared_experts=num_fused_shared_experts, + ) + else: + topk_weights, topk_ids = fused_topk( + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + ) return topk_weights, topk_ids - def naive_multicast(self, x: torch.Tensor, - cu_tokens_across_dp_cpu: torch.Tensor): - assert (len(x.shape) == 2) - buffer = torch.empty((cu_tokens_across_dp_cpu[-1], x.size(1)), - device=x.device, - dtype=x.dtype) - - start = 0 if self.dp_rank == 0 else cu_tokens_across_dp_cpu[ - self.dp_rank - 1] - end = cu_tokens_across_dp_cpu[self.dp_rank] - buffer[start:end, :].copy_(x) - for idx in range(get_dp_group().world_size): - start = 0 if idx == 0 else cu_tokens_across_dp_cpu[idx - 1] - end = cu_tokens_across_dp_cpu[idx] - get_dp_group().broadcast(buffer[start:end, :], idx) - - return buffer - - def forward(self, hidden_states: torch.Tensor, - router_logits: torch.Tensor): + def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): if self.use_direct_call: return self.forward_impl(hidden_states, router_logits) else: - return torch.ops.vllm.moe_forward(hidden_states, router_logits, - self.layer_name) + return torch.ops.vllm.moe_forward( + hidden_states, router_logits, self.layer_name + ) - def forward_impl(self, hidden_states: torch.Tensor, - router_logits: torch.Tensor): + def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): assert self.quant_method is not None # Matrix multiply. @@ -526,7 +575,7 @@ def forward_impl(self, hidden_states: torch.Tensor, renormalize=self.renormalize, use_grouped_topk=self.use_grouped_topk, global_num_experts=self.global_num_experts, - expert_map=self.expert_map if not is_rocm_aiter_moe_enabled() else self.expert_mask, + expert_map=self.expert_mask, topk_group=self.topk_group, num_expert_group=self.num_expert_group, custom_routing_function=self.custom_routing_function, @@ -538,23 +587,33 @@ def forward_impl(self, hidden_states: torch.Tensor, if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): # Default set to False. (May have to add shared expert outputs.) - final_hidden_states = tensor_model_parallel_all_reduce( - final_hidden_states) + final_hidden_states = get_tp_group.all_reduce(final_hidden_states) return final_hidden_states @classmethod def make_expert_params_mapping( - cls, ckpt_gate_proj_name: str, ckpt_down_proj_name: str, - ckpt_up_proj_name: str, - num_experts: int) -> List[Tuple[str, str, int, str]]: + cls, + ckpt_gate_proj_name: str, + ckpt_down_proj_name: str, + ckpt_up_proj_name: str, + num_experts: int, + ) -> List[Tuple[str, str, int, str]]: return [ # (param_name, weight_name, expert_id, shard_id) - ("experts.w13_" if weight_name - in [ckpt_gate_proj_name, ckpt_up_proj_name] else "experts.w2_", - f"experts.{expert_id}.{weight_name}.", expert_id, shard_id) - for expert_id in range(num_experts) for shard_id, weight_name in [ + ( + ( + "experts.w13_" + if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] + else "experts.w2_" + ), + f"experts.{expert_id}.{weight_name}.", + expert_id, + shard_id, + ) + for expert_id in range(num_experts) + for shard_id, weight_name in [ ("w1", ckpt_gate_proj_name), ("w2", ckpt_down_proj_name), ("w3", ckpt_up_proj_name), @@ -572,7 +631,8 @@ def extra_repr(self) -> str: f"ep_size={self.ep_size}, " f"reduce_results={self.reduce_results}, " f"renormalize={self.renormalize}, " - f"use_grouped_topk={self.use_grouped_topk}") + f"use_grouped_topk={self.use_grouped_topk}" + ) if self.use_grouped_topk: s += f", num_expert_group={self.num_expert_group}, topk_group={self.topk_group}" # noqa: E501 diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index c3492a9fba..40074b8ee3 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -1,10 +1,16 @@ import torch from functools import cache, lru_cache from typing import Optional +from vllm.utils import direct_register_custom_op + + +def is_rocm_aiter_fusion_shared_expert_enabled(): + return False aiter_topK_meta_data = None + @lru_cache(maxsize=1) def init_aiter_topK_meta_data( n_routed_experts: int, @@ -14,44 +20,91 @@ def init_aiter_topK_meta_data( tp_size: int, shared_experts_score: float = 1.0, max_num_tokens: int = 32768, - is_EP: bool = False -): + is_EP: bool = False, +): global aiter_topK_meta_data fake_expertid = n_routed_experts + n_shared_experts - + # all layers reuse same buffer - total_topk_ids = torch.empty((max_num_tokens, top_k + n_shared_experts + is_EP), dtype=torch.int32, device='cuda') - ns_topk_ids, s_topk_ids = total_topk_ids.split([top_k, n_shared_experts + is_EP], dim=1) + total_topk_ids = torch.empty( + (max_num_tokens, top_k + n_shared_experts + is_EP), + dtype=torch.int32, + device="cuda", + ) + ns_topk_ids, s_topk_ids = total_topk_ids.split( + [top_k, n_shared_experts + is_EP], dim=1 + ) shared_expert_ids = [n_routed_experts + i for i in range(n_shared_experts + is_EP)] if is_EP: - s_topk_ids_list = [[fake_expertid]* (n_shared_experts + is_EP)] * max_num_tokens + s_topk_ids_list = [ + [fake_expertid] * (n_shared_experts + is_EP) + ] * max_num_tokens for i in range(tp_rank, max_num_tokens, tp_size): s_topk_ids_list[i] = shared_expert_ids else: s_topk_ids_list = [range(n_routed_experts, fake_expertid)] * max_num_tokens - s_topk_ids[:] = torch.tensor(s_topk_ids_list, dtype=torch.int32, device='cuda') + s_topk_ids[:] = torch.tensor(s_topk_ids_list, dtype=torch.int32, device="cuda") - total_topk_weights = torch.empty((max_num_tokens, top_k + n_shared_experts + is_EP), dtype=torch.float32, device='cuda') - ns_topk_weights, s_topk_weights = total_topk_weights.split([top_k, n_shared_experts + is_EP], dim=1) + total_topk_weights = torch.empty( + (max_num_tokens, top_k + n_shared_experts + is_EP), + dtype=torch.float32, + device="cuda", + ) + ns_topk_weights, s_topk_weights = total_topk_weights.split( + [top_k, n_shared_experts + is_EP], dim=1 + ) s_topk_weights.fill_(shared_experts_score) aiter_topK_meta_data = (total_topk_weights, total_topk_ids) -def rocm_aiter_topk_softmax_impl(topk_weights: torch.Tensor, - topk_indices: torch.Tensor, - token_expert_indices: torch.Tensor, - gating_output: torch.Tensor, - renormalize: bool) -> None: +def rocm_aiter_topk_softmax_impl( + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_fused_shared_experts: int = 0, +) -> None: from aiter import topk_softmax - topk_softmax(topk_weights, topk_indices, token_expert_indices, - gating_output, renormalize) + token = gating_output.shape[0] + device = gating_output.device + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + assert aiter_topK_meta_data is not None, ( + "AITER topK meta data is not initialized. " + "Please ensure that init_aiter_topK_meta_data is called before this function." + ) + total_topk_weights, total_topk_ids = aiter_topK_meta_data + assert total_topk_weights.shape[0] >= token, ( + f"AITER topK meta data support {total_topk_weights.shape[0]} tokens which " + f"is determined by max_num_batched_tokens, but got {token} tokens now." + ) + total_topk_weights = total_topk_weights[:token] + total_topk_ids = total_topk_ids[:token] + topk_weights, _ = total_topk_weights.split( + [topk, total_topk_weights.shape[1] - topk], dim=1 + ) + topk_ids, _ = total_topk_ids.split( + [topk, total_topk_ids.shape[1] - topk], dim=1 + ) + else: + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + token_expert_indicies = torch.empty( + gating_output.shape[0], topk, dtype=torch.int32, device=gating_output.device + ) + topk_softmax( + topk_weights, topk_ids, token_expert_indicies, gating_output, renormalize + ) + if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: + return total_topk_weights, total_topk_ids + return topk_weights, topk_ids -def rocm_aiter_topk_softmax_fake(topk_weights: torch.Tensor, - topk_indices: torch.Tensor, - token_expert_indices: torch.Tensor, - gating_output: torch.Tensor, - renormalize: bool) -> None: + +def rocm_aiter_topk_softmax_fake( + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_fused_shared_experts: int, +) -> None: pass @@ -67,6 +120,7 @@ def rocm_aiter_biased_grouped_topk_impl( ) -> tuple[torch.Tensor, torch.Tensor]: from aiter import biased_grouped_topk + token = gating_output.shape[0] device = gating_output.device if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: @@ -90,9 +144,16 @@ def rocm_aiter_biased_grouped_topk_impl( else: topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) - biased_grouped_topk(gating_output, correction_bias, topk_weights, topk_ids, - num_expert_group, topk_group, need_renorm, - routed_scaling_factor) + biased_grouped_topk( + gating_output, + correction_bias, + topk_weights, + topk_ids, + num_expert_group, + topk_group, + need_renorm, + routed_scaling_factor, + ) if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: return total_topk_weights, total_topk_ids return topk_weights, topk_ids @@ -160,6 +221,7 @@ def rocm_aiter_grouped_topk_impl( ) -> tuple[torch.Tensor, torch.Tensor]: from aiter import grouped_topk + token = gating_output.shape[0] device = gating_output.device if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: @@ -183,8 +245,16 @@ def rocm_aiter_grouped_topk_impl( else: topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) - grouped_topk(gating_output, topk_weights, topk_ids, num_expert_group, - topk_group, need_renorm, scoring_func, routed_scaling_factor) + grouped_topk( + gating_output, + topk_weights, + topk_ids, + num_expert_group, + topk_group, + need_renorm, + scoring_func, + routed_scaling_factor, + ) if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: return total_topk_weights, total_topk_ids return topk_weights, topk_ids @@ -238,3 +308,76 @@ def rocm_aiter_grouped_topk_fake( if is_rocm_aiter_fusion_shared_expert_enabled() and num_fused_shared_experts > 0: return total_topk_weights, total_topk_ids return topk_weights, topk_ids + + +direct_register_custom_op( + op_name="rocm_aiter_topk_softmax", + op_func=rocm_aiter_topk_softmax_impl, + mutates_args=[], + fake_impl=rocm_aiter_topk_softmax_fake, + dispatch_key="CUDA", +) + +direct_register_custom_op( + op_name="rocm_aiter_biased_grouped_topk", + op_func=rocm_aiter_biased_grouped_topk_impl, + mutates_args=[], + fake_impl=rocm_aiter_biased_grouped_topk_fake, + dispatch_key="CUDA", +) + +direct_register_custom_op( + op_name="rocm_aiter_grouped_topk", + op_func=rocm_aiter_grouped_topk_impl, + mutates_args=[], + fake_impl=rocm_aiter_grouped_topk_fake, + dispatch_key="CUDA", +) + + +def rocm_aiter_topk_softmax( + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_fused_shared_experts: int = 0, +) -> tuple[torch.Tensor, ...]: + return torch.ops.vllm.rocm_aiter_topk_softmax( + gating_output, topk, renormalize, num_fused_shared_experts + ) + + +def rocm_aiter_grouped_topk( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: int = 0, + topk_group: int = 0, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: float = 1.0, +) -> tuple[torch.Tensor, torch.Tensor]: + if e_score_correction_bias is not None: + return torch.ops.vllm.rocm_aiter_biased_grouped_topk( + gating_output, + e_score_correction_bias, + num_expert_group, + topk_group, + renormalize, + topk, + routed_scaling_factor, + num_fused_shared_experts, + ) + else: + assert scoring_func == "softmax" or scoring_func == "sigmoid" + return torch.ops.vllm.rocm_aiter_grouped_topk( + gating_output, + num_expert_group, + topk_group, + renormalize, + topk, + scoring_func, + routed_scaling_factor, + num_fused_shared_experts, + ) diff --git a/atom/model_ops/utils.py b/atom/model_ops/utils.py new file mode 100644 index 0000000000..5ede735490 --- /dev/null +++ b/atom/model_ops/utils.py @@ -0,0 +1,74 @@ +from typing import Tuple, Optional, List, Union +import torch +from aiter import per_tensor_quant, dtypes + + +def per_tensor_dequantize( + tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor] +) -> torch.Tensor: + fake_qweight = tensor.to(torch.float) + dq_weight = fake_qweight * inv_scale + return dq_weight + + +def normalize_e4m3fn_to_e4m3fnuz( + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + # assert weight.dtype == torch.float8_e4m3fn + # The bits pattern 10000000(-128) represents zero in e4m3fn + # but NaN in e4m3fnuz. So here we set it to 0. + # https://onnx.ai/onnx/technical/float8.html + weight_as_int8 = weight.view(torch.int8) + ROCM_FP8_NAN_AS_INT = -128 + weight_as_int8[weight_as_int8 == ROCM_FP8_NAN_AS_INT] = 0 + weight = weight_as_int8.view(torch.float8_e4m3fnuz) + + # For the same bits representation, e4m3fnuz value is half of + # the e4m3fn value, so we should double the scaling factor to + # get the same dequantized value. + # https://onnx.ai/onnx/technical/float8.html + weight_scale = weight_scale * 2.0 + if input_scale is not None: + input_scale = input_scale * 2.0 + return weight, weight_scale, input_scale + + +def requantize_with_max_scale( + weight: torch.Tensor, + weight_scale: torch.Tensor, + logical_widths: List[int], + normalize_e4m3fn_to_e4m3fnuz=False, +) -> Tuple[torch.Tensor, torch.Tensor]: + # Max scale to be used for requanitzation. + if normalize_e4m3fn_to_e4m3fnuz: + quant_dtype = torch.float8_e4m3fnuz + weight = weight.view(torch.float8_e4m3fn) + max_w_scale = weight_scale.max() * 2.0 + else: + quant_dtype = weight.dtype + max_w_scale = weight_scale.max() + + # QKV / MLP is fused in the on disk checkpoint if any of the + # weight scales are still set to the default since we initialize + # N weight scales for N shards but we only load 1 weight scale + # from disk in this case. Skip requantization in this case (since) + # we already are quantized with the single scale. + # * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8 + unfused_module_in_checkpoint = ( + weight_scale[-1] > torch.finfo(torch.float8_e4m3fn).min + ) + + # If unfused checkpoint, need requanize with the single scale. + if unfused_module_in_checkpoint or normalize_e4m3fn_to_e4m3fnuz: + start = 0 + for idx, logical_width in enumerate(logical_widths): + end = start + logical_width + weight_dq = per_tensor_dequantize(weight[start:end, :], weight_scale[idx]) + weight.view(quant_dtype)[start:end, :], _ = per_tensor_quant( + weight_dq, max_w_scale, quant_dtype=quant_dtype + ) + start = end + + return max_w_scale, weight.view(quant_dtype) diff --git a/atom/models/llama.py b/atom/models/llama.py index 10fb47ed55..64127bbbe8 100644 --- a/atom/models/llama.py +++ b/atom/models/llama.py @@ -46,7 +46,7 @@ ) from aiter.rotary_embedding import get_rope from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead -from atom.config import QuantizationConfig, get_quant_config +from atom.config import QuantizationConfig, Config from atom.models.utils import ( PPMissingLayer, @@ -302,15 +302,15 @@ def forward( class LlamaModel(nn.Module): def __init__( self, - config: LlamaConfig, - cache_config: str = "bf16", - quant_config: Optional[QuantizationConfig] = None, + atom_config: Config, prefix: str = "", layer_type: type[nn.Module] = LlamaDecoderLayer, ): super().__init__() + config = atom_config.hf_config self.config = config - self.quant_config = quant_config + cache_config = atom_config.kv_cache_dtype + quant_config = atom_config.quant_config self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size if get_pp_group().is_first_rank or ( @@ -395,16 +395,14 @@ class LlamaForCausalLM(nn.Module): def __init__( self, - config: LlamaConfig, - kv_cache_dtype: str = "bf16", + atom_config: Config, prefix: str = "", layer_type: type[nn.Module] = LlamaDecoderLayer, ): super().__init__() - self.config = config + config = atom_config.hf_config self.model = self._init_model( - config=config, - cache_config=kv_cache_dtype, + atom_config=atom_config, prefix=maybe_prefix(prefix, "model"), layer_type=layer_type, ) @@ -418,7 +416,7 @@ def __init__( prefix=maybe_prefix(prefix, "lm_head"), ) if config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) + self.lm_head.weight = self.model.embed_tokens.weight else: self.lm_head = PPMissingLayer() @@ -428,15 +426,12 @@ def __init__( def _init_model( self, - config: LlamaConfig, - cache_config: str = "bf16", + atom_config: Config, prefix: str = "", layer_type: type[nn.Module] = LlamaDecoderLayer, ): return LlamaModel( - config=config, - cache_config=cache_config, - quant_config=get_quant_config(config), + atom_config=atom_config, prefix=prefix, layer_type=layer_type, ) diff --git a/atom/models/mixtral.py b/atom/models/mixtral.py new file mode 100644 index 0000000000..a1909deaf7 --- /dev/null +++ b/atom/models/mixtral.py @@ -0,0 +1,390 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Mixtral model.""" +from collections.abc import Iterable +from typing import Optional, Union + +import torch +from torch import nn +from transformers import MixtralConfig + +from aiter import QuantType +from atom.model_ops.attention import Attention +from aiter.dist.parallel_state import ( + get_pp_group, + get_tp_group, + get_tensor_model_parallel_world_size, +) +from atom.model_ops.activation import SiluAndMul +from atom.model_ops.layernorm import RMSNorm +from atom.model_ops.linear import QKVParallelLinear, RowParallelLinear, ReplicatedLinear +from atom.model_ops.moe import FusedMoE +from aiter.rotary_embedding import get_rope +from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead +from atom.config import QuantizationConfig, Config + +from atom.models.utils import ( + PPMissingLayer, + extract_layer_index, + IntermediateTensors, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) + + +class MixtralMoE(nn.Module): + """A tensor-parallel MoE implementation for Mixtral that shards each expert + across all ranks. + + Each expert's weights are sharded across all ranks and a fused MoE + kernel is used for the forward pass, and finally we reduce the outputs + across ranks. + """ + + def __init__( + self, + config: MixtralConfig, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + quant_config: Optional[QuantizationConfig] = None, + tp_size: Optional[int] = None, + dp_size: Optional[int] = None, + prefix: str = "", + ): + super().__init__() + self.hidden_size = hidden_size + + # Gate always runs at half / full precision for now. + + self.gate = ReplicatedLinear( + hidden_size, + num_experts, + bias=False, + quant_config=None, + prefix=f"{prefix}.gate", + ) + + self.experts = FusedMoE( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + reduce_results=True, + renormalize=True, + quant_config=quant_config, + tp_size=tp_size, + dp_size=dp_size, + prefix=f"{prefix}.experts", + config=config, + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + # NOTE: hidden_states can have either 1D or 2D shape. + orig_shape = hidden_states.shape + hidden_states = hidden_states.view(-1, self.hidden_size) + # router_logits: (num_tokens, n_experts) + router_logits = self.gate(hidden_states) + final_hidden_states = self.experts(hidden_states, router_logits) + return final_hidden_states.view(orig_shape) + + +class MixtralAttention(nn.Module): + + def __init__( + self, + config: MixtralConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + rope_theta: float = 10000, + cache_config: str = "bf16", + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + # MixtralConfig has an optional head_dim argument + self.head_dim = getattr(config, "head_dim", None) + if self.head_dim is None: + self.head_dim = self.hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position, + base=int(self.rope_theta), + is_neox_style=True, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv = self.qkv_proj(hidden_states) + q, k, v = torch.split(qkv, [self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output = self.o_proj(attn_output) + return output + + +class MixtralDecoderLayer(nn.Module): + + def __init__( + self, + config: MixtralConfig, + cache_config: str = "bf16", + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + # Requires transformers > 4.32.0 + rope_theta = getattr(config, "rope_theta", 10000) + self.self_attn = MixtralAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + self.block_sparse_moe = MixtralMoE( + config=config, + num_experts=config.num_local_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + quant_config=quant_config, + prefix=f"{prefix}.block_sparse_moe", + ) + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + ) -> torch.Tensor: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + ) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.block_sparse_moe(hidden_states) + return hidden_states, residual + + +# @support_torch_compile +class MixtralModel(nn.Module): + + def __init__( + self, + *, + atom_config: Config, + prefix: str = "", + ): + super().__init__() + config = atom_config.hf_config + self.config = config + cache_config = atom_config.kv_cache_dtype + quant_config = config.quant_config + self.vocab_size = config.vocab_size + self.org_vocab_size = config.vocab_size + + self.embed_tokens = VocabParallelEmbedding( + self.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: MixtralDecoderLayer( + config, cache_config, quant_config=quant_config, prefix=prefix + ), + prefix=f"{prefix}.layers", + ) + + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for layer in self.layers[self.start_layer : self.end_layer]: + hidden_states, residual = layer(positions, hidden_states, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + return FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="w1", + ckpt_down_proj_name="w2", + ckpt_up_proj_name="w3", + num_experts=self.config.num_local_experts, + ) + + +class MixtralForCausalLM(nn.Module): + packed_modules_mapping = { + "q_proj": ("qkv_proj", "q"), + "k_proj": ("qkv_proj", "k"), + "v_proj": ("qkv_proj", "v"), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + + def __init__(self, atom_config: Config, prefix: str = ""): + super().__init__() + config = atom_config.hf_config + + self.model = MixtralModel( + atom_config=atom_config, prefix=maybe_prefix(prefix, "model") + ) + self.unpadded_vocab_size = config.vocab_size + self.lm_head = ParallelLMHead( + self.unpadded_vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + ) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> Optional[torch.Tensor]: + logits = self.lm_head(hidden_states) + return logits + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + return self.model.get_expert_mapping() diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 801727ba23..660523ff3c 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -3,7 +3,9 @@ # import torch.distributed as dist from aiter.dist.parallel_state import get_tp_group +from typing import Optional from transformers import Qwen3Config +from atom.config import QuantizationConfig, Config from atom.model_ops.activation import SiluAndMul from atom.model_ops.attention import Attention @@ -33,6 +35,7 @@ def __init__( rope_theta: float = 10000, rope_scaling: tuple | None = None, kv_cache_dtype: str = "fp16", + quant_config: Optional[QuantizationConfig] = None, ) -> None: super().__init__() tp_size = get_tp_group().world_size @@ -53,11 +56,13 @@ def __init__( self.total_num_heads, self.total_num_kv_heads, bias=qkv_bias, + quant_config=quant_config, ) self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, hidden_size, bias=False, + quant_config=quant_config, ) self.rotary_emb = get_rope( self.head_dim, @@ -100,17 +105,20 @@ def __init__( hidden_size: int, intermediate_size: int, hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, bias=False, + quant_config=quant_config, ) self.down_proj = RowParallelLinear( intermediate_size, hidden_size, bias=False, + quant_config=quant_config, ) assert hidden_act == "silu" self.act_fn = SiluAndMul() @@ -124,7 +132,12 @@ def forward(self, x): class Qwen3DecoderLayer(nn.Module): - def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: + def __init__( + self, + config: Qwen3Config, + cache_config: str = "bf16", + quant_config: Optional[QuantizationConfig] = None, + ) -> None: super().__init__() self.self_attn = Qwen3Attention( hidden_size=config.hidden_size, @@ -136,12 +149,14 @@ def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: head_dim=getattr(config, "head_dim", None), rope_theta=getattr(config, "rope_theta", 1000000), rope_scaling=getattr(config, "rope_scaling", None), - kv_cache_dtype=kv_cache_dtype, + kv_cache_dtype=cache_config, + quant_config=quant_config, ) self.mlp = Qwen3MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, + quant_config=quant_config, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( @@ -167,14 +182,19 @@ def forward( class Qwen3Model(nn.Module): - def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: + def __init__(self, atom_config: Config) -> None: super().__init__() + config = atom_config.hf_config + cache_config = atom_config.kv_cache_dtype + quant_config = atom_config.quant_config self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size ) self.layers = nn.ModuleList( [ - Qwen3DecoderLayer(config, kv_cache_dtype) + Qwen3DecoderLayer( + config, cache_config=cache_config, quant_config=quant_config + ) for _ in range(config.num_hidden_layers) ] ) @@ -202,9 +222,10 @@ class Qwen3ForCausalLM(nn.Module): "up_proj": ("gate_up_proj", 1), } - def __init__(self, config: Qwen3Config, kv_cache_dtype: str) -> None: + def __init__(self, atom_config: Config) -> None: super().__init__() - self.model = Qwen3Model(config, kv_cache_dtype) + config = atom_config.hf_config + self.model = Qwen3Model(atom_config) self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size) if config.tie_word_embeddings: self.lm_head.weight.data = self.model.embed_tokens.weight.data From 61648655df72ceec825ea18b40641821fc3539bc Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 5 Sep 2025 16:33:31 +0800 Subject: [PATCH 026/119] move outside cudagraph logic before compile --- atom/model_engine/model_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 995dc4d4f9..c08efa1f81 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -44,8 +44,6 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) - if self.config.compilation_config.level == 3: - self.model = torch.compile(self.model, fullgraph=True, backend="eager") load_model(self.model, config.model) self.sampler = Sampler() self.warmup_model() @@ -54,6 +52,8 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.capture_cudagraph() torch.set_default_device("cpu") torch.set_default_dtype(default_dtype) + if self.config.compilation_config.level == 3: + self.model = torch.compile(self.model, fullgraph=True, backend="eager") if self.world_size > 1: if rank == 0: From 38729325e02f85c145cbd62b7c1918af3ca6e6f4 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 5 Sep 2025 17:28:38 +0800 Subject: [PATCH 027/119] enable larger cuda capture graph --- atom/config.py | 2 +- atom/model_engine/model_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/atom/config.py b/atom/config.py index fb1ca7deef..e68e92c913 100644 --- a/atom/config.py +++ b/atom/config.py @@ -37,7 +37,7 @@ def __post_init__(self): if self.level not in {0, 1, 2, 3}: raise ValueError("level must in 0-3") if not self.cuda_graph_sizes: - self.cuda_graph_sizes = [128] + self.cuda_graph_sizes = [512] class QuantizationConfig(dict): diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index c08efa1f81..2a32a6bcdc 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -389,7 +389,7 @@ def capture_cudagraph(self): cuda_graph_sizes = self.config.compilation_config.cuda_graph_sizes if len(cuda_graph_sizes) == 1: self.graph_bs = [1, 2, 4, 8] + [ - i for i in range(64, cuda_graph_sizes[0] + 1, 64) + i for i in range(16, cuda_graph_sizes[0] + 1, 16) ] elif len(cuda_graph_sizes) > 1: self.graph_bs = sorted(cuda_graph_sizes, reverse=True) From 0a57dd5a5e2871c033ec00d1b53a09341bcec382 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Mon, 8 Sep 2025 05:39:02 +0000 Subject: [PATCH 028/119] support profiling for torch compile and tp --- atom/model_engine/llm_engine.py | 67 +++++++++++-------------------- atom/model_engine/model_runner.py | 32 +++++++++++++++ example.py | 27 ++++++++++++- 3 files changed, 82 insertions(+), 44 deletions(-) diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index a7b1faf39a..7180eb92f2 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -4,7 +4,6 @@ from tqdm.auto import tqdm from transformers import AutoTokenizer import torch.multiprocessing as mp -import torch.profiler as torch_profiler from atom.config import Config from atom.sampling_params import SamplingParams @@ -13,14 +12,6 @@ from atom.model_engine.model_runner import ModelRunner -class EmptyContext: - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - class LLMEngine: def __init__(self, model, **kwargs): @@ -40,22 +31,6 @@ def __init__(self, model, **kwargs): self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True) config.eos = self.tokenizer.eos_token_id self.scheduler = Scheduler(config) - self.profiler = ( - EmptyContext() - if config.torch_profiler_dir is None - else torch_profiler.profile( - activities=[ - torch_profiler.ProfilerActivity.CPU, - torch_profiler.ProfilerActivity.CUDA, - ], - record_shapes=True, - with_stack=True, - profile_memory=True, - on_trace_ready=torch_profiler.tensorboard_trace_handler( - str(config.torch_profiler_dir), use_gzip=True - ), - ) - ) atexit.register(self.exit) def exit(self): @@ -88,7 +63,11 @@ def generate( prompts: list[str] | list[list[int]], sampling_params: SamplingParams | list[SamplingParams], use_tqdm: bool = True, + enable_profiling: bool = False, ) -> list[str]: + # Start profiling for all ranks if enabled + if enable_profiling: + self.model_runner.call("start_profiler") if use_tqdm: pbar = tqdm(total=len(prompts), desc="Generating", dynamic_ncols=True) if not isinstance(sampling_params, list): @@ -97,25 +76,27 @@ def generate( self.add_request(prompt, sp) outputs = {} prefill_throughput = decode_throughput = 0.0 - with self.profiler: - while not self.is_finished(): - t = perf_counter() - output, num_tokens = self.step() + while not self.is_finished(): + t = perf_counter() + output, num_tokens = self.step() + if use_tqdm: + if num_tokens > 0: + prefill_throughput = num_tokens / (perf_counter() - t) + else: + decode_throughput = -num_tokens / (perf_counter() - t) + pbar.set_postfix( + { + "Prefill": f"{int(prefill_throughput)}tok/s", + "Decode": f"{int(decode_throughput)}tok/s", + } + ) + for seq_id, token_ids in output: + outputs[seq_id] = token_ids if use_tqdm: - if num_tokens > 0: - prefill_throughput = num_tokens / (perf_counter() - t) - else: - decode_throughput = -num_tokens / (perf_counter() - t) - pbar.set_postfix( - { - "Prefill": f"{int(prefill_throughput)}tok/s", - "Decode": f"{int(decode_throughput)}tok/s", - } - ) - for seq_id, token_ids in output: - outputs[seq_id] = token_ids - if use_tqdm: - pbar.update(1) + pbar.update(1) + # Stop profiling for all ranks if enabled + if enable_profiling: + self.model_runner.call("stop_profiler") outputs = [outputs[seq_id] for seq_id in sorted(outputs)] outputs = [ {"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 2a32a6bcdc..a8363d2dca 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -2,6 +2,7 @@ import os import torch import torch.distributed as dist +import torch.profiler as torch_profiler import tqdm from aiter import dtypes from aiter import init_dist_env, destroy_dist_env @@ -34,6 +35,14 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.world_size = config.tensor_parallel_size self.rank = rank self.event = event + + # Initialize profiler for this rank + self.profiler = None + self.profiler_dir = None + if config.torch_profiler_dir is not None: + # Create rank-specific profiler directory + self.profiler_dir = os.path.join(config.torch_profiler_dir, f"rank_{rank}") + os.makedirs(self.profiler_dir, exist_ok=True) device = torch.device(f"cuda:{rank}") torch.cuda.set_device(device) @@ -104,6 +113,29 @@ def call(self, method_name, *args): self.write_shm(method_name, *args) method = getattr(self, method_name, None) return method(*args) + + def start_profiler(self): + """Start profiling for this rank""" + if self.profiler_dir is not None and self.profiler is None: + self.profiler = torch_profiler.profile( + activities=[ + torch_profiler.ProfilerActivity.CPU, + torch_profiler.ProfilerActivity.CUDA, + ], + record_shapes=True, + with_stack=True, + profile_memory=True, + on_trace_ready=torch_profiler.tensorboard_trace_handler( + self.profiler_dir, use_gzip=True + ), + ) + self.profiler.__enter__() + + def stop_profiler(self): + """Stop profiling for this rank""" + if self.profiler is not None: + self.profiler.__exit__(None, None, None) + self.profiler = None def warmup_model(self): torch.cuda.empty_cache() diff --git a/example.py b/example.py index 1c17b5bf5b..84a2d1dda1 100644 --- a/example.py +++ b/example.py @@ -1,10 +1,12 @@ import os +import torch from atom import LLMEngine, SamplingParams from transformers import AutoTokenizer import argparse from atom.config import CompilationConfig from typing import List + parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="config of test", @@ -42,6 +44,10 @@ parser.add_argument("--level", type=int, default=0, help="The level of compilation") +parser.add_argument( + "--torch-profiler-dir", type=str, default=None, help="Directory to save torch profiler traces" +) + def parse_size_list(size_str: str) -> List[int]: import ast try: @@ -60,6 +66,7 @@ def main(): tensor_parallel_size=args.tensor_parallel_size, kv_cache_dtype=args.kv_cache_dtype, port=args.port, + torch_profiler_dir=args.torch_profiler_dir, compilation_config=CompilationConfig( level = args.level, cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes) @@ -83,7 +90,25 @@ def main(): for prompt in prompts ] print("This is prompts:", prompts) - outputs = llm.generate(prompts, sampling_params) + + # Calculate total token number of prompts + prompt_token_ids = [llm.tokenizer.encode(prompt) for prompt in prompts] + prompt_lens = [len(token_ids) for token_ids in prompt_token_ids] + + # Create warmup inputs with same shapes as all prompts + # Generate random token IDs for each prompt length to match expected input shapes + warmup_prompts = [] + for i, prompt_len in enumerate(prompt_lens): + warmup_prompt = torch.randint( + 0, llm.tokenizer.vocab_size, size=(prompt_len,) + ).tolist() + warmup_prompts.append(warmup_prompt) + + # Run warmup with the same batch structure as the actual prompts (no profiling) + _ = llm.generate(warmup_prompts, sampling_params) + + # generate (with profiling) + outputs = llm.generate(prompts, sampling_params, enable_profiling=True) for prompt, output in zip(prompts, outputs): print("\n") From 26fa369c4f1b064640ca950341a623ed561d7089 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Mon, 15 Sep 2025 11:48:34 +0000 Subject: [PATCH 029/119] match with new aiter code --- atom/model_engine/model_runner.py | 3 ++- atom/model_ops/embed_head.py | 2 +- atom/model_ops/linear.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index a8363d2dca..0c4634ec75 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -424,7 +424,8 @@ def capture_cudagraph(self): i for i in range(16, cuda_graph_sizes[0] + 1, 16) ] elif len(cuda_graph_sizes) > 1: - self.graph_bs = sorted(cuda_graph_sizes, reverse=True) + self.graph_bs = cuda_graph_sizes + self.graph_bs.sort(reverse=True) self.graphs = {} self.graph_pool = None diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 100a031789..179e203919 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -42,7 +42,7 @@ def forward(self, x: torch.Tensor): y = F.embedding(x, self.weight) if self.tp_size > 1: y = mask.unsqueeze(1) * y - y = get_tp_group().all_reduce(y, open_fp8_quant=False) + y = get_tp_group().all_reduce(y, ca_fp8_quant=False) return y diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 1ca72e8213..035fc9d494 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -230,7 +230,7 @@ def forward( dtype=otype, ) if self.tp_dim == 1 and self.tp_size > 1: - y = get_tp_group().all_reduce(y, open_fp8_quant=False) + y = get_tp_group().all_reduce(y, ca_fp8_quant=False) return y From f912c67d81f7598f492b764421140f18795a05f6 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 21 Sep 2025 12:20:32 +0000 Subject: [PATCH 030/119] close to vllm v1 --- atom/config.py | 13 +- atom/model_engine/block_manager.py | 5 +- atom/model_engine/engine_core.py | 187 +++++++++++++++++ atom/model_engine/engine_core_mgr.py | 154 ++++++++++++++ atom/model_engine/llm_engine.py | 177 +++++++++------- atom/model_engine/model_runner.py | 175 ++++++++-------- atom/model_engine/scheduler.py | 42 +++- atom/model_engine/sequence.py | 55 +++-- atom/model_ops/attention.py | 7 +- atom/model_ops/embed_head.py | 20 +- atom/model_ops/sampler.py | 32 ++- atom/utils/__init__.py | 297 +++++++++++++++++++++++++++ atom/utils/context.py | 37 +++- example.py | 37 ++-- 14 files changed, 1001 insertions(+), 237 deletions(-) create mode 100644 atom/model_engine/engine_core.py create mode 100644 atom/model_engine/engine_core_mgr.py create mode 100644 atom/utils/__init__.py diff --git a/atom/config.py b/atom/config.py index e68e92c913..351d41795c 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,13 +1,12 @@ import os -import torch import re -from dataclasses import dataclass -from transformers import AutoConfig +from dataclasses import dataclass, field from typing import Optional -from dataclasses import field + +import torch from aiter import QuantType -from transformers import PretrainedConfig from aiter.utility.dtypes import d_dtypes +from transformers import AutoConfig, PretrainedConfig @dataclass @@ -122,7 +121,7 @@ class Config: tensor_parallel_size: int = 1 enforce_eager: bool = False hf_config: AutoConfig | None = None - eos: int = -1 + eos_token_id: int = -1 kvcache_block_size: int = 16 num_kvcache_blocks: int = -1 kv_cache_dtype: str = "bf16" @@ -131,6 +130,8 @@ class Config: torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) compilation_config: CompilationConfig = field(default_factory=CompilationConfig) quant_config: QuantizationConfig = field(default_factory=lambda: QuantizationConfig()) + asyncio_mode: bool = False + master_addr: str = "127.0.0.1" def __post_init__(self): # assert os.path.isdir(self.model) diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index 669ede6a66..e123073c5f 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -1,9 +1,10 @@ from collections import deque -import xxhash + import numpy as np +import xxhash -from atom.model_engine.sequence import Sequence from atom.config import Config +from atom.model_engine.sequence import Sequence class Block: diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py new file mode 100644 index 0000000000..73602bc4d7 --- /dev/null +++ b/atom/model_engine/engine_core.py @@ -0,0 +1,187 @@ +import enum +import logging +import pickle +import queue +import threading +import weakref +from contextlib import ExitStack +from typing import Any, Optional + +import zmq +import zmq.asyncio + +from atom.config import Config +from atom.model_engine.sequence import Sequence, SequenceStatus, get_exit_sequence + +logger = logging.getLogger("atom") + + +from atom.model_engine.model_runner import ModelRunner +from atom.model_engine.scheduler import Scheduler +from atom.utils import ( + close_sockets, + get_engine_client_zmq_addr, + get_mp_context, + make_zmq_socket, + shutdown_all_processes, + zmq_socket_ctx, +) + + +class EngineCoreRequestType(enum.Enum): + """ + Request types defined as hex byte strings, so it can be sent over sockets + without separate encoding step. + """ + + ADD = b"\x00" + ABORT = b"\x01" + START_DP_WAVE = b"\x02" + UTILITY = b"\x03" + # Sentinel used within EngineCoreProc. + EXECUTOR_FAILED = b"\x04" + # Sentinel used within EngineCore. + SHUTDOWN = b"\x05" + + +class EngineCore: + def __init__(self, config: Config, input_address: str, output_address: str): + self.ps = [] + self.events = [] + ctx = get_mp_context() + if config.tensor_parallel_size > 1: + for i in range(1, config.tensor_parallel_size): + event = ctx.Event() + process = ctx.Process( + target=ModelRunner, name=f"ModelRunner{i}", args=(config, i, event) + ) + process.start() + self.ps.append(process) + self.events.append(event) + self.model_runner = ModelRunner( + config, 0, self.events if config.tensor_parallel_size > 1 else [] + ) + logger.info("Engine core model runner loaded") + self.scheduler = Scheduler(config) + self.input_queue = queue.Queue[Sequence]() + self.output_queue = queue.Queue[Sequence]() + self.input_address = input_address + self.output_address = output_address + logger.info(f"Engine core input address: {self.input_address}") + logger.info(f"Engine core output address: {self.output_address}") + self.input_thread = threading.Thread( + target=self.process_input_sockets, args=(self.input_address,), daemon=True + ) + self.input_thread.start() + self.output_thread = threading.Thread( + target=self.process_output_sockets, args=(self.output_address,), daemon=True + ) + self.output_thread.start() + + self.profile_enbaled = config.torch_profiler_dir is not None + + def exit(self): + self._send_engine_dead() + self.model_runner.call("exit") + del self.model_runner + for p in self.ps: + p.join() + logger.info("Engine core model runner exit") + + @staticmethod + def run_engine(config: Config, input_address: str, output_address: str): + engine: EngineCore = None + try: + engine = EngineCore(config, input_address, output_address) + engine.start_profiler() + engine.busy_loop() + engine.stop_profiler() + finally: + if engine is not None: + engine.exit() + + def _send_engine_dead(self): + logger.info("Send engine core dead signal") + self.output_queue.put_nowait(get_exit_sequence()) + self.output_thread.join(timeout=5.0) + + def busy_loop(self): + shutdown = False + while True: + shutdown = shutdown or self._process_input_queue() + if not self.scheduler.is_finished(): + self._process_engine_step() + elif shutdown: + break + + def _process_engine_step(self): + scheduled_batchs = self.scheduler.schedule() + token_ids = self.model_runner.call("run", scheduled_batchs) + self.scheduler.postprocess(scheduled_batchs.seqs, token_ids) + for seq in scheduled_batchs.seqs: + if seq.is_finished: + self.output_queue.put_nowait(seq) + + def _process_input_queue(self): + while not self.input_queue.empty(): + seq = self.input_queue.get_nowait() + print(f"Engine core processing sequence {seq.id} with status {seq.status}") + if seq.status == SequenceStatus.EXIT_ENGINE: + return True + self.scheduler.add(seq) + return False + + def process_input_sockets(self, input_address: str): + """Input socket IO thread.""" + with ExitStack() as stack, zmq.Context() as ctx: + input_socket = stack.enter_context( + make_zmq_socket(ctx, input_address, zmq.DEALER, bind=False) + ) + poller = zmq.Poller() + # Send initial message to input socket - this is required + # before the front-end ROUTER socket can send input messages + # back to us. + input_socket.send(b"") + poller.register(input_socket, zmq.POLLIN) + logger.info("Engine core input socket connected") + + while True: + for input_socket, _ in poller.poll(): + # (RequestType, RequestData) + serialized_obj = input_socket.recv(copy=False) + request_type, req = pickle.loads(serialized_obj) + if request_type == EngineCoreRequestType.ADD: + logger.info(f"Engine core input: {request_type} {req.id}") + self.input_queue.put_nowait(req) + elif request_type == EngineCoreRequestType.SHUTDOWN: + logger.info("Engine core input thread shutdown") + self.input_queue.put_nowait(get_exit_sequence()) + break + + def process_output_sockets(self, output_address: str): + """Output socket IO thread.""" + with ExitStack() as stack, zmq.Context() as ctx: + socket = stack.enter_context( + make_zmq_socket(ctx, output_address, zmq.PUSH, linger=4000) + ) + logger.info("Engine core output socket connected") + + while True: + seq = self.output_queue.get() + if seq.status == SequenceStatus.EXIT_ENGINE: + socket.send(pickle.dumps((EngineCoreRequestType.SHUTDOWN, None))) + logger.info("Engine core output thread closed") + break + serialized_obj = pickle.dumps((EngineCoreRequestType.ADD, seq)) + socket.send(serialized_obj) + logger.info(f"Engine core output: {seq.id}") + + def start_profiler(self): + if self.profile_enbaled: + self.model_runner.call("start_profiler") + + def stop_profiler(self): + if self.profile_enbaled: + print("Stopping profiler...") + self.model_runner.call("stop_profiler") + print("Profiler stopped.") diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py new file mode 100644 index 0000000000..4c8e655981 --- /dev/null +++ b/atom/model_engine/engine_core_mgr.py @@ -0,0 +1,154 @@ +import logging +import multiprocessing +import pickle +import queue +import weakref +from threading import Thread + +import zmq +import zmq.asyncio + +from atom.config import Config +from atom.model_engine.engine_core import EngineCore, EngineCoreRequestType +from atom.model_engine.sequence import Sequence +from atom.utils import ( + get_open_zmq_inproc_path, + get_open_zmq_ipc_path, + make_zmq_socket, + shutdown_all_processes, +) + +logger = logging.getLogger("atom") + + +class CoreManager: + def __init__(self, config: Config): + sync_ctx = zmq.Context(io_threads=2) + self.ctx = zmq.asyncio.Context(sync_ctx) if config.asyncio_mode else sync_ctx + self.outputs_queue = queue.Queue[Sequence]() + engine_core_process, addresses = launch_engine_core(config) + logger.info("Engine core inited") + self.engine_core_process = engine_core_process + + input_address = addresses["input_address"] + output_address = addresses["output_address"] + self.input_socket = make_zmq_socket( + self.ctx, input_address, zmq.ROUTER, bind=True + ) + identity, _ = self.input_socket.recv_multipart() + logger.info(f"Core engine input socket identity: {identity}") + self.core_engine_identity = identity + self.output_socket = make_zmq_socket(self.ctx, output_address, zmq.PULL) + + shutdown_path = get_open_zmq_inproc_path() + self.shutdown_path = shutdown_path + + def process_outputs_socket(): + assert isinstance(self.output_socket, zmq.Socket) + shutdown_socket = self.ctx.socket(zmq.PAIR) + try: + shutdown_socket.bind(shutdown_path) + poller = zmq.Poller() + poller.register(shutdown_socket, zmq.POLLIN) + poller.register(self.output_socket, zmq.POLLIN) + logger.info("Core client output thread started") + while True: + socks = poller.poll() + if not socks: + continue + if len(socks) == 2 or socks[0][0] == shutdown_socket: + # shutdown signal, exit thread. + logger.info("Core client output thread receive shutdown signal") + break + + obj = self.output_socket.recv(copy=False) + request_type, seq = pickle.loads(obj) + if request_type == EngineCoreRequestType.SHUTDOWN: + logger.info("Core client output thread receive shutdown signal") + self.engine_core_process.join() + self.engine_core_process = None + break + elif request_type == EngineCoreRequestType.ADD: + logger.info(f"Engine core output sequence id: {seq.id}") + self.outputs_queue.put_nowait(seq) + finally: + # Close sockets. + shutdown_socket.close(linger=0) + self.output_socket.close(linger=0) + + self.output_queue_thread = Thread( + target=process_outputs_socket, + name="EngineCoreOutputQueueThread", + daemon=True, + ) + self.output_queue_thread.start() + self._finalizer = weakref.finalize(self, self.close) + + def close(self): + self._shut_down_core_engine() + self.input_socket.close() + logger.info("Core client input socket closed") + if self.shutdown_path and self.output_queue_thread: + with self.ctx.socket(zmq.PAIR) as shutdown_sender: + shutdown_sender.connect(self.shutdown_path) + # Send shutdown signal. + shutdown_sender.send(b"") + self.output_queue_thread.join() + logger.info("Core client output thread is shut down") + + def add_request(self, seq: Sequence): + logger.info(f"Add request to engine core, sequence id: {seq.id}") + self.input_socket.send_multipart( + [self.core_engine_identity, pickle.dumps((EngineCoreRequestType.ADD, seq))], + copy=False, + ) + + def _shut_down_core_engine(self): + logger.info("Shutdown core engine") + if self.engine_core_process is not None: + self.input_socket.send_multipart( + [ + self.core_engine_identity, + pickle.dumps((EngineCoreRequestType.SHUTDOWN, None)), + ], + copy=False, + ) + self.engine_core_process.join() + + # def _shut_down_core_engine(self): + # logger.info("Shutdown core engine") + # if self.engine_core_process is not None and self.engine_core_process.is_alive(): + # shutdown_all_processes([self.engine_core_process]) + + def get_output(self) -> Sequence: + outputs = self.outputs_queue.get() + logger.info(f"Get output from engine core, sequence id: {outputs.id}") + # if isinstance(outputs, Exception): + # raise outputs from None + return outputs + + def is_rest(self): + return not self.outputs_queue.empty() + + def is_alive(self): + return self.engine_core_process is not None + + +def launch_engine_core(config: Config): + input_address = get_open_zmq_ipc_path() + output_address = get_open_zmq_ipc_path() + import torch + + torch.multiprocessing.set_start_method("spawn") + process = multiprocessing.Process( + target=EngineCore.run_engine, + name=f"EngineCore", + kwargs={ + "config": config, + "input_address": input_address, + "output_address": output_address, + }, + ) + process.start() + logger.info("Engine core process started") + return process, {"input_address": input_address, "output_address": output_address} diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 7180eb92f2..bfa36b0f87 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -1,15 +1,17 @@ -import atexit +import logging +import time from dataclasses import fields -from time import perf_counter + +import torch.multiprocessing as mp from tqdm.auto import tqdm from transformers import AutoTokenizer -import torch.multiprocessing as mp from atom.config import Config -from atom.sampling_params import SamplingParams +from atom.model_engine.engine_core_mgr import CoreManager from atom.model_engine.sequence import Sequence -from atom.model_engine.scheduler import Scheduler -from atom.model_engine.model_runner import ModelRunner +from atom.sampling_params import SamplingParams + +logger = logging.getLogger("atom") class LLMEngine: @@ -18,90 +20,115 @@ def __init__(self, model, **kwargs): config_fields = {field.name for field in fields(Config)} config_kwargs = {k: v for k, v in kwargs.items() if k in config_fields} config = Config(model, **config_kwargs) - self.ps = [] - self.events = [] - ctx = mp.get_context("spawn") - for i in range(1, config.tensor_parallel_size): - event = ctx.Event() - process = ctx.Process(target=ModelRunner, args=(config, i, event)) - process.start() - self.ps.append(process) - self.events.append(event) - self.model_runner = ModelRunner(config, 0, self.events) self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True) - config.eos = self.tokenizer.eos_token_id - self.scheduler = Scheduler(config) - atexit.register(self.exit) - - def exit(self): - self.model_runner.call("exit") - del self.model_runner - for p in self.ps: - p.join() - - def add_request(self, prompt: str | list[int], sampling_params: SamplingParams): - if isinstance(prompt, str): - prompt = self.tokenizer.encode(prompt) - seq = Sequence(prompt, self.model_runner.block_size, sampling_params) - self.scheduler.add(seq) + config.eos_token_id = self.tokenizer.eos_token_id + + self.rquest_ids = set() + self.io_processor = InputOutputProcessor( + self.tokenizer, config.kvcache_block_size + ) + self.core_mgr = CoreManager(config) + logger.info("LLMEngine init") + + def add_request( + self, prompt_or_tokens: str | list[int], sampling_params: SamplingParams + ): + # 1. convet prompt to request + req = self.io_processor.preprocess(prompt_or_tokens, sampling_params) + + # 2. add req to scheduler + self.core_mgr.add_request(req) def step(self): - seqs, is_prefill = self.scheduler.schedule() - token_ids = self.model_runner.call("run", seqs, is_prefill) - self.scheduler.postprocess(seqs, token_ids) - outputs = [ - (seq.seq_id, seq.completion_token_ids) for seq in seqs if seq.is_finished - ] - num_tokens = sum(len(seq) for seq in seqs) if is_prefill else -len(seqs) - return outputs, num_tokens + # seqs, is_prefill = self.scheduler.schedule() + # token_ids = self.model_runner.call("run", seqs, is_prefill) + # self.scheduler.postprocess(seqs, token_ids) + # outputs = [ + # (seq.seq_id, seq.completion_token_ids) for seq in seqs if seq.is_finished + # ] + # num_tokens = sum(len(seq) for seq in seqs) if is_prefill else -len(seqs) + # return outputs, num_tokens + + seq = self.core_mgr.get_output() + return seq def is_finished(self): - return self.scheduler.is_finished() + return not self.io_processor.has_pending_requests() def generate( self, - prompts: list[str] | list[list[int]], + # prompts: list[str] | list[list[int]], + prompts: list[str], sampling_params: SamplingParams | list[SamplingParams], - use_tqdm: bool = True, enable_profiling: bool = False, ) -> list[str]: - # Start profiling for all ranks if enabled - if enable_profiling: - self.model_runner.call("start_profiler") - if use_tqdm: - pbar = tqdm(total=len(prompts), desc="Generating", dynamic_ncols=True) + # # Start profiling for all ranks if enabled + # if enable_profiling: + # self.model_runner.call("start_profiler") if not isinstance(sampling_params, list): sampling_params = [sampling_params] * len(prompts) for prompt, sp in zip(prompts, sampling_params): self.add_request(prompt, sp) outputs = {} - prefill_throughput = decode_throughput = 0.0 - while not self.is_finished(): - t = perf_counter() - output, num_tokens = self.step() - if use_tqdm: - if num_tokens > 0: - prefill_throughput = num_tokens / (perf_counter() - t) - else: - decode_throughput = -num_tokens / (perf_counter() - t) - pbar.set_postfix( - { - "Prefill": f"{int(prefill_throughput)}tok/s", - "Decode": f"{int(decode_throughput)}tok/s", - } - ) - for seq_id, token_ids in output: - outputs[seq_id] = token_ids - if use_tqdm: - pbar.update(1) - # Stop profiling for all ranks if enabled - if enable_profiling: - self.model_runner.call("stop_profiler") + while not self.is_finished() and ( + self.core_mgr.is_alive() or self.core_mgr.is_rest() + ): + seq = self.step() + out = self.io_processor.postprocess(seq) + outputs[seq.id] = out + # # Stop profiling for all ranks if enabled + # if enable_profiling: + # self.model_runner.call("stop_profiler") outputs = [outputs[seq_id] for seq_id in sorted(outputs)] - outputs = [ - {"text": self.tokenizer.decode(token_ids), "token_ids": token_ids} - for token_ids in outputs - ] - if use_tqdm: - pbar.close() return outputs + + +class InputOutputProcessor: + + def __init__(self, tokenizer, block_size): + self.tokenizer = tokenizer + self.block_size = block_size + self.requests = {} + + def preprocess( + self, prompt_or_tokens: str | list[int], sampling_params: SamplingParams + ): + """responsible for: + 1) Tokenize + 2) Create Sequence object""" + tokens = ( + self.tokenizer.encode(prompt_or_tokens) + if isinstance(prompt_or_tokens, str) + else prompt_or_tokens + ) + seq = Sequence(tokens, self.block_size, sampling_params) + seq.arrive_time = time.time() + self.requests[seq.id] = seq + print( + f"Request {seq.id} arrived, input tokens: {len(tokens)}, pending requests: {len(self.requests)}" + ) + return seq + + def postprocess(self, req: Sequence): + """responsible for: + 1) Compute stats for logging + 2) Detokenize""" + self.requests.pop(req.id) + output_str = self.tokenizer.decode(req.completion_token_ids) + req.leave_time = time.time() + print( + f"Request {req.id} finished with reason {req.leave_reason}. " + f"Input tokens: {req.num_prompt_tokens}, output tokens: {req.num_completion_tokens}, " + f"latency: {req.leave_time - req.arrive_time:.2f}s" + ) + return { + "text": output_str, + "token_ids": req.completion_token_ids, + "latency": req.leave_time - req.arrive_time, + "finish_reason": req.leave_reason, + "num_tokens_input": req.num_prompt_tokens, + "num_tokens_output": req.num_completion_tokens, + } + + def has_pending_requests(self): + return len(self.requests) > 0 diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 0c4634ec75..5f95373bb0 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -1,22 +1,30 @@ -import pickle +import logging import os +import pickle +import signal +import weakref +from multiprocessing.shared_memory import SharedMemory +from multiprocessing.synchronize import Event + import torch import torch.distributed as dist import torch.profiler as torch_profiler import tqdm -from aiter import dtypes -from aiter import init_dist_env, destroy_dist_env -from aiter.dist.parallel_state import graph_capture, get_tensor_model_parallel_rank -from multiprocessing.synchronize import Event -from multiprocessing.shared_memory import SharedMemory +from aiter import destroy_dist_env, dtypes, init_dist_env +from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture + from atom.config import Config +from atom.model_engine.scheduler import ScheduledBatchs from atom.model_engine.sequence import Sequence +from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler -from atom.models.qwen3 import Qwen3ForCausalLM from atom.models.llama import LlamaForCausalLM +from atom.models.qwen3 import Qwen3ForCausalLM + # from atom.models.mixtral import MixtralForCausalLM -from atom.utils.context import set_context, get_context, reset_context -from atom.model_loader.loader import load_model +from atom.utils.context import get_context, reset_context, set_context + +logger = logging.getLogger("atom") suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -35,7 +43,7 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.world_size = config.tensor_parallel_size self.rank = rank self.event = event - + # Initialize profiler for this rank self.profiler = None self.profiler_dir = None @@ -46,12 +54,14 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): device = torch.device(f"cuda:{rank}") torch.cuda.set_device(device) - os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_ADDR"] = self.config.master_addr os.environ["MASTER_PORT"] = str(self.config.port) init_dist_env(self.world_size, rankID=rank) default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") + self.graph_bs = [0] # for eager fallback + self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) load_model(self.model, config.model) self.sampler = Sampler() @@ -73,14 +83,25 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.shm = SharedMemory(name="atom") self.loop() - def exit(self): - if self.world_size > 1: + def signal_handler(signum, frame): + raise SystemExit( + f"Rank{self.rank}/{self.world_size}: received signal {signum}, exiting..." + ) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + logger.info(f"Initializing ModelRunner for rank {rank}") + self._finalizer = weakref.finalize(self, self.exit) + + def exit(self, msg=None): + logger.info(f"Exiting ModelRunner for rank {self.rank}/{self.world_size} {msg}") + if dist.is_initialized(): self.shm.close() dist.barrier() if self.rank == 0: self.shm.unlink() if not self.enforce_eager: - del self.graphs, self.graph_pool + self.graphs = self.graph_pool = None torch.cuda.synchronize() destroy_dist_env() @@ -113,7 +134,7 @@ def call(self, method_name, *args): self.write_shm(method_name, *args) method = getattr(self, method_name, None) return method(*args) - + def start_profiler(self): """Start profiling for this rank""" if self.profiler_dir is not None and self.profiler is None: @@ -130,7 +151,7 @@ def start_profiler(self): ), ) self.profiler.__enter__() - + def stop_profiler(self): """Stop profiling for this rank""" if self.profiler is not None: @@ -151,7 +172,8 @@ def warmup_model(self): Sequence([0] * max_model_len, block_size=self.block_size) for _ in range(num_seqs) ] - self.run(seqs, True) + dummy_batch = ScheduledBatchs(seqs, True, False) + self.run(dummy_batch) torch.cuda.empty_cache() def allocate_kv_cache(self): @@ -230,7 +252,7 @@ def prepare_block_tables(self, seqs: list[Sequence]): ).cuda(non_blocking=True) return block_tables - def prepare_prefill(self, seqs: list[Sequence]): + def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): input_ids = [] positions = [] cu_seqlens_q = [0] @@ -239,7 +261,7 @@ def prepare_prefill(self, seqs: list[Sequence]): max_seqlen_k = 0 slot_mapping = [] block_tables = None - for seq in seqs: + for seq in scheduled_batchs.seqs: seqlen = len(seq) input_ids.extend(seq[seq.num_cached_tokens :]) positions.extend(list(range(seq.num_cached_tokens, seqlen))) @@ -259,7 +281,7 @@ def prepare_prefill(self, seqs: list[Sequence]): end = start + seq.last_block_num_tokens slot_mapping.extend(list(range(start, end))) if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache - block_tables = self.prepare_block_tables(seqs) + block_tables = self.prepare_block_tables(scheduled_batchs.seqs) input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda( non_blocking=True ) @@ -279,19 +301,23 @@ def prepare_prefill(self, seqs: list[Sequence]): dropout_p = 0.0 set_context( True, - cu_seqlens_q, - cu_seqlens_k, - max_seqlen_q, - max_seqlen_k, - min_seqlen_q, - slot_mapping, - None, - block_tables, - dropout_p, + len(scheduled_batchs.seqs), + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + min_seqlen_q=min_seqlen_q, + slot_mapping=slot_mapping, + block_tables=block_tables, + dropout_p=dropout_p, ) return input_ids, positions - def prepare_decode(self, seqs: list[Sequence]): + @torch.inference_mode() + def prepare_decode(self, scheduled_batchs: ScheduledBatchs): + bs = len(scheduled_batchs.seqs) + graph_bs = next(x for x in self.graph_bs if x >= bs) + assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" input_ids = [] positions = [] slot_mapping = [] @@ -303,7 +329,7 @@ def prepare_decode(self, seqs: list[Sequence]): min_seqlen_q = 0 dropout_p = 0.0 - for seq in seqs: + for seq in scheduled_batchs.seqs: current_seq_len = len(seq) cu_seqlens_q.append(cu_seqlens_q[-1] + 1) @@ -316,39 +342,39 @@ def prepare_decode(self, seqs: list[Sequence]): slot_mapping.append( seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 ) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda( - non_blocking=True - ) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda( - non_blocking=True + input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) + positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) + context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True) + block_tables = self.prepare_block_tables(scheduled_batchs.seqs) + cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) + + graph_vars = self.graph_vars + graph_vars["slot_mapping"][bs:graph_bs] = -1 + graph_vars["slot_mapping"][:bs].copy_(slot_mapping, non_blocking=True) + graph_vars["input_ids"][:bs].copy_(input_ids, non_blocking=True) + graph_vars["positions"][:bs].copy_(positions, non_blocking=True) + graph_vars["context_lens"][:bs].copy_(context_lens, non_blocking=True) + graph_vars["block_tables"][:bs, : block_tables.size(1)].copy_( + block_tables, non_blocking=True ) - slot_mapping = torch.tensor( - slot_mapping, dtype=torch.int64, pin_memory=True - ).cuda(non_blocking=True) - context_lens = torch.tensor( - context_lens, dtype=torch.int32, pin_memory=True - ).cuda(non_blocking=True) - block_tables = self.prepare_block_tables(seqs) - cu_seqlens_q = torch.tensor( - cu_seqlens_q, dtype=torch.int32, pin_memory=True - ).cuda(non_blocking=True) - cu_seqlens_k = torch.tensor( - cu_seqlens_k, dtype=torch.int32, pin_memory=True - ).cuda(non_blocking=True) set_context( False, + batch_size=bs, + graph_bs=graph_bs, cu_seqlens_q=cu_seqlens_q, cu_seqlens_k=cu_seqlens_k, max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping, - context_lens=context_lens, - block_tables=block_tables, + slot_mapping=graph_vars["slot_mapping"][:bs], + context_lens=graph_vars["context_lens"][:bs], + block_tables=graph_vars["block_tables"][:bs, : block_tables.size(1)], dropout_p=dropout_p, ) - return input_ids, positions + return graph_vars["input_ids"][:bs], graph_vars["positions"][:bs] def prepare_sample(self, seqs: list[Sequence]): temperatures = [] @@ -364,34 +390,20 @@ def run_model( self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool ): torch.cuda.empty_cache() - if is_prefill or self.enforce_eager or input_ids.size(0) > 512: + ctx = get_context() + bs = ctx.batch_size + if is_prefill or self.enforce_eager or bs > self.graph_bs[-1]: return self.model.compute_logits(self.model(input_ids, positions)) else: - bs = input_ids.size(0) - graph_bs = next(x for x in self.graph_bs if x >= bs) - assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" - - context = get_context() - graph = self.graphs[graph_bs] - graph_vars = self.graph_vars - # for k, v in graph_vars.items(): - # if k != "outputs": - # v.zero_() - graph_vars["slot_mapping"][bs:graph_bs] = -1 - graph_vars["input_ids"][:bs] = input_ids - graph_vars["positions"][:bs] = positions - graph_vars["slot_mapping"][:bs] = context.slot_mapping - graph_vars["context_lens"][:bs] = context.context_lens - graph_vars["block_tables"][ - :bs, : context.block_tables.size(1) - ] = context.block_tables - graph.replay() - return self.model.compute_logits(graph_vars["outputs"][:bs]) - - def run(self, seqs: list[Sequence], is_prefill: bool) -> list[int]: - input_ids, positions = ( - self.prepare_prefill(seqs) if is_prefill else self.prepare_decode(seqs) - ) + graph_bs = ctx.graph_bs + self.graphs[graph_bs].replay() + return self.model.compute_logits(self.graph_vars["outputs"][:bs]) + + def run(self, scheduled_batchs: ScheduledBatchs) -> list[int]: + seqs = scheduled_batchs.seqs + is_prefill = scheduled_batchs.is_prefill + prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode + input_ids, positions = prepare_func(scheduled_batchs) temperatures = self.prepare_sample(seqs) if self.rank == 0 else None logits = self.run_model(input_ids, positions, is_prefill) token_ids = ( @@ -414,7 +426,6 @@ def capture_cudagraph(self): outputs = torch.zeros(max_bs, hf_config.hidden_size) # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) - self.graph_bs = [] if self.config.compilation_config.cudagraph_capture_sizes: self.graph_bs = self.config.compilation_config.cudagraph_capture_sizes else: @@ -427,7 +438,7 @@ def capture_cudagraph(self): self.graph_bs = cuda_graph_sizes self.graph_bs.sort(reverse=True) - self.graphs = {} + self.graphs: dict[int, torch.cuda.CUDAGraph] = dict() self.graph_pool = None with graph_capture() as gc: @@ -442,6 +453,7 @@ def capture_cudagraph(self): graph = torch.cuda.CUDAGraph() set_context( False, + batch_size=bs, slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs], @@ -456,6 +468,7 @@ def capture_cudagraph(self): self.graphs[bs] = graph torch.cuda.synchronize() reset_context() + self.graph_bs.sort(reverse=False) self.graph_vars = dict( input_ids=input_ids, diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index edb6962e9a..2dfd2e2c78 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -1,8 +1,16 @@ from collections import deque +from dataclasses import dataclass from atom.config import Config -from atom.model_engine.sequence import Sequence, SequenceStatus from atom.model_engine.block_manager import BlockManager +from atom.model_engine.sequence import Sequence, SequenceStatus + + +@dataclass +class ScheduledBatchs: + seqs: list[Sequence] + is_prefill: bool + has_seq_out: bool class Scheduler: @@ -10,10 +18,11 @@ class Scheduler: def __init__(self, config: Config): self.max_num_seqs = config.max_num_seqs self.max_num_batched_tokens = config.max_num_batched_tokens - self.eos = config.eos + self.eos_token_id = config.eos_token_id self.block_manager = BlockManager(config) self.waiting: deque[Sequence] = deque() self.running: deque[Sequence] = deque() + self.has_seq_out = False def is_finished(self): return not self.waiting and not self.running @@ -21,7 +30,7 @@ def is_finished(self): def add(self, seq: Sequence): self.waiting.append(seq) - def schedule(self) -> tuple[list[Sequence], bool]: + def schedule(self) -> ScheduledBatchs: # prefill scheduled_seqs = [] num_seqs = 0 @@ -29,6 +38,7 @@ def schedule(self) -> tuple[list[Sequence], bool]: if not self.running and not self.waiting: self.block_manager.reset() + has_seq_out = self.has_seq_out while self.waiting and num_seqs < self.max_num_seqs: seq = self.waiting[0] if num_batched_tokens + len( @@ -43,7 +53,11 @@ def schedule(self) -> tuple[list[Sequence], bool]: self.running.append(seq) scheduled_seqs.append(seq) if scheduled_seqs: - return scheduled_seqs, True + return ScheduledBatchs( + seqs=scheduled_seqs, + is_prefill=True, + has_seq_out=has_seq_out, + ) # decode while self.running and num_seqs < self.max_num_seqs: @@ -60,19 +74,29 @@ def schedule(self) -> tuple[list[Sequence], bool]: scheduled_seqs.append(seq) assert scheduled_seqs self.running.extendleft(reversed(scheduled_seqs)) - return scheduled_seqs, False + self.has_seq_out = False + return ScheduledBatchs( + seqs=scheduled_seqs, + is_prefill=False, + has_seq_out=has_seq_out, + ) def preempt(self, seq: Sequence): seq.status = SequenceStatus.WAITING self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], token_ids: list[int]) -> list[bool]: + def postprocess(self, seqs: list[Sequence], token_ids: list[int]): for seq, token_id in zip(seqs, token_ids): seq.append_token(token_id) - if ( - not seq.ignore_eos and token_id == self.eos - ) or seq.num_completion_tokens == seq.max_tokens: + leave_reason = None + if not seq.ignore_eos and token_id == self.eos_token_id: + leave_reason = "eos" + elif seq.num_completion_tokens == seq.max_tokens: + leave_reason = "max_tokens" + if leave_reason is not None: + seq.leave_reason = leave_reason seq.status = SequenceStatus.FINISHED self.block_manager.deallocate(seq) self.running.remove(seq) + self.has_seq_out = True diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index dabf8e9103..b1c48f3da6 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -9,16 +9,22 @@ class SequenceStatus(Enum): WAITING = auto() RUNNING = auto() FINISHED = auto() + EXIT_ENGINE = auto() + + +def get_exit_sequence(): + exit_seq = Sequence([-1], 0) + exit_seq.status = SequenceStatus.EXIT_ENGINE + return exit_seq class Sequence: counter = count() - block_size = 16 def __init__( self, token_ids: list[int], block_size: int, sampling_params=SamplingParams() ): - self.seq_id = next(Sequence.counter) + self.id = next(Sequence.counter) self.status = SequenceStatus.WAITING self.token_ids = copy(token_ids) self.last_token = token_ids[-1] @@ -31,6 +37,11 @@ def __init__( self.ignore_eos = sampling_params.ignore_eos self.block_size = block_size + # statistics fields + self.arrive_time = 0.0 + self.leave_time = 0.0 + self.leave_reason = "" + def __len__(self): return self.num_tokens @@ -74,23 +85,23 @@ def append_token(self, token_id: int): self.last_token = token_id self.num_tokens += 1 - def __getstate__(self): - return ( - self.num_tokens, - self.num_prompt_tokens, - self.num_cached_tokens, - self.block_table, - self.token_ids if self.num_completion_tokens == 0 else self.last_token, - ) - - def __setstate__(self, state): - ( - self.num_tokens, - self.num_prompt_tokens, - self.num_cached_tokens, - self.block_table, - ) = state[:-1] - if self.num_completion_tokens == 0: - self.token_ids = state[-1] - else: - self.last_token = state[-1] + # def __getstate__(self): + # return ( + # self.num_tokens, + # self.num_prompt_tokens, + # self.num_cached_tokens, + # self.block_table, + # self.token_ids if self.num_completion_tokens == 0 else self.last_token, + # ) + + # def __setstate__(self, state): + # ( + # self.num_tokens, + # self.num_prompt_tokens, + # self.num_cached_tokens, + # self.block_table, + # ) = state[:-1] + # if self.num_completion_tokens == 0: + # self.token_ids = state[-1] + # else: + # self.last_token = state[-1] diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index a78c0a823a..005121f127 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -1,11 +1,12 @@ +import aiter import torch -from torch import nn import triton import triton.language as tl - -import aiter from aiter.paged_attn import PagedAttention +from torch import nn + from atom.utils.context import get_context + # from flash_attn import flash_attn_with_kvcache diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 179e203919..124cf23464 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -1,8 +1,9 @@ import torch -from torch import nn -import torch.nn.functional as F import torch.distributed as dist +import torch.nn.functional as F +from aiter.dist.communication_op import tensor_model_parallel_all_gather from aiter.dist.parallel_state import get_tp_group +from torch import nn from atom.utils.context import get_context @@ -69,11 +70,12 @@ def forward(self, x: torch.Tensor): x = x[last_indices].contiguous() logits = F.linear(x, self.weight, self.bias) if self.tp_size > 1: - all_logits = ( - [torch.empty_like(logits) for _ in range(self.tp_size)] - if self.tp_rank == 0 - else None - ) - dist.gather(logits, all_logits, 0) - logits = torch.cat(all_logits, -1) if self.tp_rank == 0 else None + logits = tensor_model_parallel_all_gather(logits) + # all_logits = ( + # [torch.empty_like(logits) for _ in range(self.tp_size)] + # if self.tp_rank == 0 + # else None + # ) + # dist.gather(logits, all_logits, 0) + # logits = torch.cat(all_logits, -1) if self.tp_rank == 0 else None return logits diff --git a/atom/model_ops/sampler.py b/atom/model_ops/sampler.py index e4b9816e70..5d67556a0e 100644 --- a/atom/model_ops/sampler.py +++ b/atom/model_ops/sampler.py @@ -1,4 +1,5 @@ import torch +from aiter.ops.triton.topk import topk from torch import nn @@ -6,13 +7,28 @@ class Sampler(nn.Module): def __init__(self): super().__init__() + self.eps = 1e-10 - def forward(self, logits: torch.Tensor, temperatures: torch.Tensor): - logits = logits.to(torch.float) - greedy_tokens = logits.argmax(dim=-1) - logits.div_(temperatures.unsqueeze(dim=1)) + def forward( + self, + logits: torch.Tensor, # (token_num, vocab_size) + temperatures: torch.Tensor, # (token_num,) + ) -> torch.Tensor: # (token_num,) + logits = logits.float() + return torch.where( + temperatures == 0, self.greedy_sample(logits), self.random_sample(logits) + ) + + def greedy_sample( + self, logits: torch.Tensor # (token_num, vocab_size) + ) -> torch.Tensor: # (token_num,) + _, sampled_tokens = topk(logits, 1) + return sampled_tokens.view(-1) + + def random_sample( + self, logits: torch.Tensor # (token_num, vocab_size) + ) -> torch.Tensor: # (token_num,) probs = torch.softmax(logits, dim=-1, dtype=torch.float) - # logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float) - epsilon = 1e-10 - sample_tokens = probs.div_(torch.empty_like(probs).exponential_(1) + epsilon).argmax(dim=-1) - return torch.where(temperatures == 0, greedy_tokens, sample_tokens) + logits = probs.div_(torch.empty_like(probs).exponential_(1) + self.eps) + _, sampled_tokens = topk(logits, 1) + return sampled_tokens.view(-1) diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py new file mode 100644 index 0000000000..9b97a3f747 --- /dev/null +++ b/atom/utils/__init__.py @@ -0,0 +1,297 @@ +import contextlib +import ipaddress +import logging +import multiprocessing +import os +import signal +import socket +import tempfile +import time +from functools import lru_cache +from multiprocessing.context import ForkContext, SpawnContext +from multiprocessing.process import BaseProcess +from typing import Any, Iterator, Optional, Sequence, Union +from urllib.parse import urlparse +from uuid import uuid4 + +import psutil +import zmq +import zmq.asyncio + +logger = logging.getLogger("atom") + + +def get_mp_context() -> Union[ForkContext, SpawnContext]: + """Get a multiprocessing context with 'spawn' start method.""" + return multiprocessing.get_context("spawn") + + +def shutdown_all_processes(procs: list[BaseProcess]): + # Shutdown the process. + for proc in procs: + if proc.is_alive(): + proc.terminate() + + # Allow 5 seconds for remaining procs to terminate. + deadline = time.monotonic() + 5 + for proc in procs: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + if proc.is_alive(): + proc.join(remaining) + + for proc in procs: + if proc.is_alive() and (pid := proc.pid) is not None: + kill_process_tree(pid) + + +def kill_process_tree(pid: int): + """ + Kills all descendant processes of the given pid by sending SIGKILL. + + Args: + pid (int): Process ID of the parent process + """ + try: + parent = psutil.Process(pid) + except psutil.NoSuchProcess: + return + + # Get all children recursively + children = parent.children(recursive=True) + + # Send SIGKILL to all children first + for child in children: + with contextlib.suppress(ProcessLookupError): + os.kill(child.pid, signal.SIGKILL) + + # Finally kill the parent + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + + +def is_valid_ipv6_address(address: str) -> bool: + try: + ipaddress.IPv6Address(address) + return True + except ValueError: + return False + + +def split_host_port(host_port: str) -> tuple[str, int]: + # ipv6 + if host_port.startswith("["): + host, port = host_port.rsplit("]", 1) + host = host[1:] + port = port.split(":")[1] + return host, int(port) + else: + host, port = host_port.split(":") + return host, int(port) + + +def join_host_port(host: str, port: int) -> str: + if is_valid_ipv6_address(host): + return f"[{host}]:{port}" + else: + return f"{host}:{port}" + + +def get_distributed_init_method(ip: str, port: int) -> str: + return get_tcp_uri(ip, port) + + +def get_tcp_uri(ip: str, port: int) -> str: + if is_valid_ipv6_address(ip): + return f"tcp://[{ip}]:{port}" + else: + return f"tcp://{ip}:{port}" + + +def get_open_zmq_inproc_path() -> str: + return f"inproc://{uuid4()}" + + +def get_open_port() -> int: + """ + Get an open port for the vLLM process to listen on. + An edge case to handle, is when we run data parallel, + we need to avoid ports that are potentially used by + the data parallel master process. + Right now we reserve 10 ports for the data parallel master + process. Currently it uses 2 ports. + """ + return _get_open_port() + + +def get_open_ports_list(count: int = 5) -> list[int]: + """Get a list of open ports.""" + ports = set() + while len(ports) < count: + ports.add(get_open_port()) + return list(ports) + + +def _get_open_port() -> int: + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +@lru_cache() +def get_zmq_base_path() -> str: + return tempfile.gettempdir() + + +def get_open_zmq_ipc_path() -> str: + base_rpc_path = get_zmq_base_path() + return f"ipc://{base_rpc_path}/{uuid4()}" + + +def get_engine_client_zmq_addr(local_only: bool, host: str, port: int = 0) -> str: + """Assign a new ZMQ socket address. + + If local_only is True, participants are colocated and so a unique IPC + address will be returned. + + Otherwise, the provided host and port will be used to construct a TCP + address (port == 0 means assign an available port).""" + + return ( + get_open_zmq_ipc_path() + if local_only + else (get_tcp_uri(host, port or get_open_port())) + ) + + +def close_sockets(sockets: Sequence[Union[zmq.Socket, zmq.asyncio.Socket]]): + for sock in sockets: + if sock is not None: + sock.close(linger=0) + + +def split_zmq_path(path: str) -> tuple[str, str, str]: + """Split a zmq path into its parts.""" + parsed = urlparse(path) + if not parsed.scheme: + raise ValueError(f"Invalid zmq path: {path}") + + scheme = parsed.scheme + host = parsed.hostname or "" + port = str(parsed.port or "") + + if scheme == "tcp" and not all((host, port)): + # The host and port fields are required for tcp + raise ValueError(f"Invalid zmq path: {path}") + + if scheme != "tcp" and port: + # port only makes sense with tcp + raise ValueError(f"Invalid zmq path: {path}") + + return scheme, host, port + + +def make_zmq_path(scheme: str, host: str, port: Optional[int] = None) -> str: + """Make a ZMQ path from its parts. + + Args: + scheme: The ZMQ transport scheme (e.g. tcp, ipc, inproc). + host: The host - can be an IPv4 address, IPv6 address, or hostname. + port: Optional port number, only used for TCP sockets. + + Returns: + A properly formatted ZMQ path string. + """ + if port is None: + return f"{scheme}://{host}" + if is_valid_ipv6_address(host): + return f"{scheme}://[{host}]:{port}" + return f"{scheme}://{host}:{port}" + + +# Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L783 # noqa: E501 +def make_zmq_socket( + ctx: Union[zmq.asyncio.Context, zmq.Context], # type: ignore[name-defined] + path: str, + socket_type: Any, + bind: Optional[bool] = None, + identity: Optional[bytes] = None, + linger: Optional[int] = None, +) -> Union[zmq.Socket, zmq.asyncio.Socket]: # type: ignore[name-defined] + """Make a ZMQ socket with the proper bind/connect semantics.""" + + mem = psutil.virtual_memory() + socket = ctx.socket(socket_type) + + # Calculate buffer size based on system memory + total_mem = mem.total / 1024**3 + available_mem = mem.available / 1024**3 + # For systems with substantial memory (>32GB total, >16GB available): + # - Set a large 0.5GB buffer to improve throughput + # For systems with less memory: + # - Use system default (-1) to avoid excessive memory consumption + if total_mem > 32 and available_mem > 16: + buf_size = int(0.5 * 1024**3) # 0.5GB in bytes + else: + buf_size = -1 # Use system default buffer size + + if bind is None: + bind = socket_type not in (zmq.PUSH, zmq.SUB, zmq.XSUB) + + if socket_type in (zmq.PULL, zmq.DEALER, zmq.ROUTER): + socket.setsockopt(zmq.RCVHWM, 0) + socket.setsockopt(zmq.RCVBUF, buf_size) + + if socket_type in (zmq.PUSH, zmq.DEALER, zmq.ROUTER): + socket.setsockopt(zmq.SNDHWM, 0) + socket.setsockopt(zmq.SNDBUF, buf_size) + + if identity is not None: + socket.setsockopt(zmq.IDENTITY, identity) + + if linger is not None: + socket.setsockopt(zmq.LINGER, linger) + + if socket_type == zmq.XPUB: + socket.setsockopt(zmq.XPUB_VERBOSE, True) + + # Determine if the path is a TCP socket with an IPv6 address. + # Enable IPv6 on the zmq socket if so. + scheme, host, _ = split_zmq_path(path) + if scheme == "tcp" and is_valid_ipv6_address(host): + socket.setsockopt(zmq.IPV6, 1) + + if bind: + socket.bind(path) + else: + socket.connect(path) + + return socket + + +@contextlib.contextmanager +def zmq_socket_ctx( + path: str, + socket_type: Any, + bind: Optional[bool] = None, + linger: int = 0, + identity: Optional[bytes] = None, +) -> Iterator[zmq.Socket]: + """Context manager for a ZMQ socket""" + + ctx = zmq.Context() # type: ignore[attr-defined] + try: + yield make_zmq_socket(ctx, path, socket_type, bind=bind, identity=identity) + except KeyboardInterrupt: + logger.debug("Got Keyboard Interrupt.") + + finally: + ctx.destroy(linger=linger) diff --git a/atom/utils/context.py b/atom/utils/context.py index 660045f926..72187faa39 100644 --- a/atom/utils/context.py +++ b/atom/utils/context.py @@ -1,10 +1,13 @@ from dataclasses import dataclass + import torch @dataclass class Context: is_prefill: bool = False + batch_size: int = 0 + graph_bs: int = 0 cu_seqlens_q: torch.Tensor | None = None cu_seqlens_k: torch.Tensor | None = None max_seqlen_q: int = 0 @@ -18,13 +21,41 @@ class Context: _CONTEXT = Context() + def get_context(): return _CONTEXT -def set_context(is_prefill, cu_seqlens_q=None, cu_seqlens_k=None, max_seqlen_q=0, max_seqlen_k=0, min_seqlen_q=0, slot_mapping=None, context_lens=None, block_tables=None, - dropout_p=0.0): + +def set_context( + is_prefill: bool, + batch_size: int, + graph_bs=0, + cu_seqlens_q=None, + cu_seqlens_k=None, + max_seqlen_q=0, + max_seqlen_k=0, + min_seqlen_q=0, + slot_mapping=None, + context_lens=None, + block_tables=None, + dropout_p=0.0, +): global _CONTEXT - _CONTEXT = Context(is_prefill, cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, min_seqlen_q, slot_mapping, context_lens, block_tables, dropout_p) + _CONTEXT = Context( + is_prefill, + batch_size, + graph_bs, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + min_seqlen_q, + slot_mapping, + context_lens, + block_tables, + dropout_p, + ) + def reset_context(): global _CONTEXT diff --git a/example.py b/example.py index 84a2d1dda1..1a1a224bf4 100644 --- a/example.py +++ b/example.py @@ -1,11 +1,12 @@ +import argparse import os +from typing import List + import torch -from atom import LLMEngine, SamplingParams from transformers import AutoTokenizer -import argparse -from atom.config import CompilationConfig -from typing import List +from atom import LLMEngine, SamplingParams +from atom.config import CompilationConfig parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, @@ -91,21 +92,19 @@ def main(): ] print("This is prompts:", prompts) - # Calculate total token number of prompts - prompt_token_ids = [llm.tokenizer.encode(prompt) for prompt in prompts] - prompt_lens = [len(token_ids) for token_ids in prompt_token_ids] - - # Create warmup inputs with same shapes as all prompts - # Generate random token IDs for each prompt length to match expected input shapes - warmup_prompts = [] - for i, prompt_len in enumerate(prompt_lens): - warmup_prompt = torch.randint( - 0, llm.tokenizer.vocab_size, size=(prompt_len,) - ).tolist() - warmup_prompts.append(warmup_prompt) - - # Run warmup with the same batch structure as the actual prompts (no profiling) - _ = llm.generate(warmup_prompts, sampling_params) + # # Calculate total token number of prompts + # prompt_token_ids = [llm.tokenizer.encode(prompt) for prompt in prompts] + # prompt_lens = [len(token_ids) for token_ids in prompt_token_ids] + # # Create warmup inputs with same shapes as all prompts + # # Generate random token IDs for each prompt length to match expected input shapes + # warmup_prompts = [] + # for i, prompt_len in enumerate(prompt_lens): + # warmup_prompt = torch.randint( + # 0, llm.tokenizer.vocab_size, size=(prompt_len,) + # ).tolist() + # warmup_prompts.append(warmup_prompt) + # # Run warmup with the same batch structure as the actual prompts (no profiling) + # _ = llm.generate(warmup_prompts, sampling_params) # generate (with profiling) outputs = llm.generate(prompts, sampling_params, enable_profiling=True) From 104cfcc9c01ba4ae116299121e0c3a4b95e0d96e Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 21 Sep 2025 12:49:48 +0000 Subject: [PATCH 031/119] using aiter's softmax --- atom/model_ops/sampler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atom/model_ops/sampler.py b/atom/model_ops/sampler.py index 5d67556a0e..2fa501f2f5 100644 --- a/atom/model_ops/sampler.py +++ b/atom/model_ops/sampler.py @@ -1,4 +1,5 @@ import torch +from aiter.ops.triton.softmax import softmax from aiter.ops.triton.topk import topk from torch import nn @@ -28,7 +29,7 @@ def greedy_sample( def random_sample( self, logits: torch.Tensor # (token_num, vocab_size) ) -> torch.Tensor: # (token_num,) - probs = torch.softmax(logits, dim=-1, dtype=torch.float) + probs = softmax(logits) logits = probs.div_(torch.empty_like(probs).exponential_(1) + self.eps) _, sampled_tokens = topk(logits, 1) return sampled_tokens.view(-1) From 2d5f75efc586caaacc15c76344ef91f9f8d8f9ad Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 21 Sep 2025 13:28:53 +0000 Subject: [PATCH 032/119] minor refine --- atom/model_engine/model_runner.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 5f95373bb0..291df7d205 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -313,7 +313,6 @@ def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): ) return input_ids, positions - @torch.inference_mode() def prepare_decode(self, scheduled_batchs: ScheduledBatchs): bs = len(scheduled_batchs.seqs) graph_bs = next(x for x in self.graph_bs if x >= bs) @@ -385,6 +384,15 @@ def prepare_sample(self, seqs: list[Sequence]): ).cuda(non_blocking=True) return temperatures + @torch.inference_mode() + def prepare_model(self, scheduled_batchs: ScheduledBatchs): + seqs = scheduled_batchs.seqs + is_prefill = scheduled_batchs.is_prefill + prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode + input_ids, positions = prepare_func(scheduled_batchs) + temperatures = self.prepare_sample(seqs) if self.rank == 0 else None + return input_ids, positions, temperatures + @torch.inference_mode() def run_model( self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool @@ -400,12 +408,8 @@ def run_model( return self.model.compute_logits(self.graph_vars["outputs"][:bs]) def run(self, scheduled_batchs: ScheduledBatchs) -> list[int]: - seqs = scheduled_batchs.seqs - is_prefill = scheduled_batchs.is_prefill - prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode - input_ids, positions = prepare_func(scheduled_batchs) - temperatures = self.prepare_sample(seqs) if self.rank == 0 else None - logits = self.run_model(input_ids, positions, is_prefill) + input_ids, positions, temperatures = self.prepare_model(scheduled_batchs) + logits = self.run_model(input_ids, positions, scheduled_batchs.is_prefill) token_ids = ( self.sampler(logits, temperatures).tolist() if self.rank == 0 else None ) From 1f7a9931b41258cbf1c96a5d62a19de7d1fb88f9 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Mon, 22 Sep 2025 08:01:14 +0000 Subject: [PATCH 033/119] support fused_moe and Mixtral --- atom/model_engine/model_runner.py | 7 +- atom/model_loader/loader.py | 54 ++- atom/model_loader/weight_utils.py | 34 +- atom/model_ops/base_config.py | 55 +++ atom/model_ops/moe.py | 537 ++++++++++++++++++++++++++++-- atom/model_ops/topK.py | 12 +- atom/model_ops/utils.py | 95 +++++- atom/models/mixtral.py | 7 +- 8 files changed, 751 insertions(+), 50 deletions(-) create mode 100644 atom/model_ops/base_config.py diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 291df7d205..5ab1541823 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -20,8 +20,7 @@ from atom.model_ops.sampler import Sampler from atom.models.llama import LlamaForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM - -# from atom.models.mixtral import MixtralForCausalLM +from atom.models.mixtral import MixtralForCausalLM from atom.utils.context import get_context, reset_context, set_context logger = logging.getLogger("atom") @@ -29,7 +28,7 @@ suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, "LlamaForCausalLM": LlamaForCausalLM, - # "MixtralForCausalLM": MixtralForCausalLM, + "MixtralForCausalLM": MixtralForCausalLM, } @@ -179,6 +178,8 @@ def warmup_model(self): def allocate_kv_cache(self): config = self.config hf_config = config.hf_config + if not hasattr(hf_config, "head_dim"): + hf_config.head_dim = hf_config.hidden_size // hf_config.num_attention_heads free, total = torch.cuda.mem_get_info() used = total - free peak = torch.cuda.memory_stats()["allocated_bytes.all.peak"] diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index be84b51d11..49ff738949 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -4,6 +4,7 @@ from torch import nn from safetensors import safe_open from atom.model_loader.weight_utils import download_weights_from_hf +from atom.model_ops.base_config import QuantizeMethodBase from tqdm import tqdm @@ -24,30 +25,59 @@ def load_model(model: nn.Module, model_name_or_path: str): for file in tqdm(glob(os.path.join(path, "*.safetensors"))): print(f"Loading weights from {file}") with safe_open(file, "pt", "cpu") as f: - for weight_name in f.keys(): - if weight_name.endswith( + for name in f.keys(): + if name.endswith( "kv_scale" ): continue - weight_tensor = f.get_tensor(weight_name) - if "weight_scale_inv" in weight_name: - weight_name = weight_name.replace( + weight_tensor = f.get_tensor(name) + if "weight_scale_inv" in name: + name = name.replace( "weight_scale_inv", "weight_scale" ) for k in packed_modules_mapping: - if k in weight_name: + if k in name: v, shard_id = packed_modules_mapping[k] - param_name = weight_name.replace(k, v) + param_name = name.replace(k, v) param = model.get_parameter(param_name) weight_loader = getattr(param, "weight_loader") weight_loader(param, weight_tensor, shard_id) break else: - param = model.get_parameter(weight_name) - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, weight_tensor) + # Check if model has expert mapping before processing + if hasattr(model, "get_expert_mapping"): + for k in model.get_expert_mapping(): + param_name, weight_name, expert_id, shard_id = k + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if ((name.endswith(".bias") or name.endswith("_bias")) + and name not in dict(model.named_parameters())): + continue + param = model.get_parameter(name) + weight_loader = getattr(param, "weight_loader") + weight_loader(param, + weight_tensor, + name, + shard_id=shard_id, + expert_id=expert_id) + break + else: + param = model.get_parameter(name) + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, weight_tensor) + else: + # Model doesn't have expert mapping, use generic loading + param = model.get_parameter(name) + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, weight_tensor) for _, module in model.named_modules(): if hasattr(module, "process_weights_after_loading"): module.process_weights_after_loading() + quant_method = getattr(module, "quant_method", None) + if isinstance(quant_method, QuantizeMethodBase): + quant_method.process_weights_after_loading(module) diff --git a/atom/model_loader/weight_utils.py b/atom/model_loader/weight_utils.py index cdb0ce34be..d2161acc3d 100644 --- a/atom/model_loader/weight_utils.py +++ b/atom/model_loader/weight_utils.py @@ -10,6 +10,7 @@ import tempfile import hashlib import fnmatch +import torch from pathlib import Path from typing import Any, Callable, Optional, Union @@ -112,4 +113,35 @@ def download_weights_from_hf( if time_taken > 0.5: logger.info("Time spent downloading weights for %s: %.6f seconds", model_name_or_path, time_taken) - return hf_folder \ No newline at end of file + return hf_folder + + +def set_weight_attrs( + weight: torch.Tensor, + weight_attrs: Optional[dict[str, Any]], +): + """Set attributes on a weight tensor. + + This method is used to set attributes on a weight tensor. This method + will not overwrite existing attributes. + + Args: + weight: The weight tensor. + weight_attrs: A dictionary of attributes to set on the weight tensor. + """ + if weight_attrs is None: + return + for key, value in weight_attrs.items(): + assert not hasattr( + weight, key), (f"Overwriting existing tensor attribute: {key}") + + # NOTE(woosuk): During weight loading, we often do something like: + # narrowed_tensor = param.data.narrow(0, offset, len) + # narrowed_tensor.copy_(real_weight) + # expecting narrowed_tensor and param.data to share the same storage. + # However, on TPUs, narrowed_tensor will lazily propagate to the base + # tensor, which is param.data, leading to the redundant memory usage. + # This sometimes causes OOM errors during model loading. To avoid this, + # we sync the param tensor after its weight loader is called. + # TODO(woosuk): Remove this hack once we have a better solution. + setattr(weight, key, value) \ No newline at end of file diff --git a/atom/model_ops/base_config.py b/atom/model_ops/base_config.py new file mode 100644 index 0000000000..a17179ec13 --- /dev/null +++ b/atom/model_ops/base_config.py @@ -0,0 +1,55 @@ +import inspect +from abc import ABC, abstractmethod +from typing import Type + +import torch +from torch import nn + + +class QuantizeMethodBase(ABC): + """Base class for different quantized methods.""" + + @abstractmethod + def create_weights(self, layer: torch.nn.Module, *weight_args, + **extra_weight_attrs): + """Create weights for a layer. + + The weights will be set as attributes of the layer.""" + raise NotImplementedError + + @abstractmethod + def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + # Not required functions + def embedding(self, layer: torch.nn.Module, *args, + **kwargs) -> torch.Tensor: + """Gather embeddings in the layer based on indices in the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError + + def process_weights_after_loading(self, layer: nn.Module) -> None: + """Process the weight after loading. + + This can be used for example, to transpose weights for computation. + """ + return + + +def method_has_implemented_embedding( + method_class: Type[QuantizeMethodBase]) -> bool: + """ + Not all quant methods have embedding implemented, so we need to check that + it exists for our given method. We check this by making sure the function + has been changed from the base implementation. + """ + base_embedding = inspect.getattr_static(QuantizeMethodBase, "embedding", + None) + class_embedding = inspect.getattr_static(method_class, "embedding", None) + + return (class_embedding is not None + and class_embedding is not base_embedding) diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 55e73fff30..9836e5cf14 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -1,4 +1,5 @@ from functools import lru_cache +from abc import abstractmethod import torch from torch import nn from typing import Callable, List, Optional, Tuple @@ -6,12 +7,500 @@ from transformers import PretrainedConfig from aiter import QuantType, dtypes from atom.config import QuantizationConfig +from atom.model_ops.base_config import QuantizeMethodBase from atom.model_ops.topK import rocm_aiter_grouped_topk as grouped_topk from atom.model_ops.topK import rocm_aiter_topk_softmax as fused_topk from atom.model_ops.topK import ( init_aiter_topK_meta_data, is_rocm_aiter_fusion_shared_expert_enabled, ) +from atom.model_ops.utils import ( + shuffle_weights, + normalize_e4m3fn_to_e4m3fnuz, + per_tensor_dequantize, + scaled_fp8_quant, + all_close_1d, +) +from atom.model_loader.weight_utils import set_weight_attrs +from aiter.fused_moe import fused_moe +from aiter import ActivationType + +class FusedMoEMethodBase(QuantizeMethodBase): + + @abstractmethod + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size_per_partition: int, + params_dtype: torch.dtype, **extra_weight_attrs): + raise NotImplementedError + + @abstractmethod + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + use_grouped_topk: bool = False, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, + activation: str = "silu", + ) -> torch.Tensor: + raise NotImplementedError + + +class UnquantizedFusedMoEMethod(FusedMoEMethodBase): + """MoE method without quantization.""" + + def create_weights(self, layer: torch.nn.Module, num_experts: int, + hidden_size: int, intermediate_size_per_partition: int, + params_dtype: torch.dtype, **extra_weight_attrs): + # Fused gate_up_proj (column parallel) + w13_weight = torch.nn.Parameter(torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + # down_proj (row parallel) + w2_weight = torch.nn.Parameter(torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor: + # Pad the weight tensor. This is an optimization on ROCm platform, which + # can benefit from tensors located far enough from one another in memory + # if (envs.VLLM_ROCM_MOE_PADDING and current_platform.is_rocm() + # and weight.stride(-1) == 1 + # and (weight.stride(-2) * weight.element_size()) % 512 == 0): + # num_pad = 256 // weight.element_size() + # weight = F.pad(weight, (0, num_pad), "constant", 0)[..., :-num_pad] + # torch.cuda.empty_cache() + return weight + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + super().process_weights_after_loading(layer) + + layer.w13_weight = torch.nn.Parameter(self._maybe_pad_weight( + layer.w13_weight.data), + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(self._maybe_pad_weight( + layer.w2_weight.data), + requires_grad=False) + # reshaping weights is required for aiter moe kernel. + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight.data, layer.w2_weight.data) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + use_grouped_topk: bool = False, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, + activation: ActivationType = ActivationType.Silu, + ) -> torch.Tensor: + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function, + scoring_func=scoring_func, + e_score_correction_bias=e_score_correction_bias) + return fused_moe( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_weight=topk_weights, + topk_ids=topk_ids, + expert_mask=expert_map, + activation=activation) + + +class Fp8MoEMethod(FusedMoEMethodBase): + """MoE method for FP8. + Supports loading FP8 checkpoints with static weight scale and + dynamic/static activation scale. + + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: QuantizationConfig): + self.quant_config = quant_config + self.quant_type = self.quant_config["quant_type"] + self.quant_dtype = self.quant_config["quant_dtype"] + self.block_quant = self.quant_type == QuantType.per_1x128 or self.quant_type == QuantType.per_1x32 + self.need_normalize_e4m3fn_to_e4m3fnuz = self.quant_dtype == torch.float8_e4m3fnuz + + def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, **extra_weight_attrs): + + #TODO hard code for now + params_dtype = torch.float8_e4m3fn + + if self.block_quant: + if self.quant_type == QuantType.per_1x128: + block_n = 1 + block_k = 128 + elif self.quant_type == QuantType.per_1x32: + block_n = 1 + block_k = 32 + tp_size = get_tp_group().world_size + # NOTE: To ensure proper alignment of the block-wise quantization + # scales, the output_size of the weights for both the gate and up + # layers must be divisible by block_n. + # Required by column parallel or enabling merged weights + if intermediate_size_per_partition % block_n != 0: + raise ValueError( + f"The output_size of gate's and up's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_n = {block_n}.") + if (tp_size > 1 + and intermediate_size_per_partition % block_k != 0): + # Required by row parallel + raise ValueError( + f"The input_size of down's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}.") + + # WEIGHTS + w13_weight = torch.nn.Parameter(torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter(torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype), + requires_grad=False) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + if not self.block_quant: + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter(torch.ones( + num_experts, 2, dtype=torch.float32), + requires_grad=False) + w2_weight_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + else: + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + 2 * ((intermediate_size_per_partition + block_n - 1) // + block_n), + (hidden_size + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + (hidden_size + block_n - 1) // block_n, + (intermediate_size_per_partition + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale_inv", w13_weight_scale) + layer.register_parameter("w2_weight_scale_inv", w2_weight_scale) + assert self.quant_config["is_dynamic"] == True + + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + # extra_weight_attrs.update( + # {"quant_method": FusedMoeWeightScaleSupported.BLOCK. + # value} if self.block_quant else + # {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}) + # If loading fp8 checkpoint, pass the weight loaders. + # If loading an fp16 checkpoint, do not (we will quantize in + # process_weights_after_loading() + if self.quant_config["quant_dtype"] == torch.float8_e4m3fnuz: + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # INPUT_SCALES + if not self.quant_config["is_dynamic"]: + w13_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + w2_input_scale = torch.nn.Parameter(torch.ones( + num_experts, dtype=torch.float32), + requires_grad=False) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + else: + layer.w13_input_scale = None + layer.w2_input_scale = None + + def process_weights_after_loading(self, layer: nn.Module) -> None: + # Lazy import to avoid importing triton too early. + # from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + # is_rocm_aiter_moe_enabled, shuffle_weights) + + # self.rocm_aiter_moe_enabled = is_rocm_aiter_moe_enabled() + # self.rocm_aiter_use_asm = (self.rocm_aiter_moe_enabled + # and envs.VLLM_ROCM_USE_AITER_ASMMOE) + + # TODO (rob): refactor block quant into separate class. + if self.block_quant: + assert self.quant_config.activation_scheme == "dynamic" + if self.need_normalize_e4m3fn_to_e4m3fnuz: + w13_weight, w13_weight_scale_inv, w13_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w13_weight, layer.w13_weight_scale_inv, + layer.w13_input_scale) + w2_weight, w2_weight_scale_inv, w2_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w2_weight, layer.w2_weight_scale_inv, + layer.w2_input_scale) + else: + w13_weight = layer.w13_weight.data + w13_weight_scale_inv = layer.w13_weight_scale_inv.data + w2_weight = layer.w2_weight + w2_weight_scale_inv = layer.w2_weight_scale_inv + + # torch.compile() cannot use Parameter subclasses. + layer.w13_weight = nn.Parameter(w13_weight, requires_grad=False) + layer.w13_weight_scale_inv = nn.Parameter(w13_weight_scale_inv, + requires_grad=False) + layer.w2_weight = nn.Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale_inv = nn.Parameter(w2_weight_scale_inv, + requires_grad=False) + + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight.data, layer.w2_weight.data) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) + + # # DeepGemm scales need to be transposed and aligned. We try to do + # # it ahead of time for performance reasons. + # if self.allow_deep_gemm: + # # Lazy import to avoid CUDA initialization problems. + # import deep_gemm as dg + # if _is_col_major(layer.w13_weight_scale_inv): + # layer.w13_weight_scale_inv = \ + # dg.get_col_major_tma_aligned_tensor(layer.w13_weight_scale_inv).contiguous() + # if _is_col_major(layer.w2_weight_scale_inv): + # layer.w2_weight_scale_inv = \ + # dg.get_col_major_tma_aligned_tensor(layer.w2_weight_scale_inv).contiguous() + return + + # # If checkpoint is fp16, quantize in place. + # if not self.quant_config.is_checkpoint_fp8_serialized: + # fp8_dtype = current_platform.fp8_dtype() + # w13_weight = torch.empty_like(layer.w13_weight.data, + # dtype=fp8_dtype) + # w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype) + + # # Re-initialize w13_scale because we directly quantize + # # merged w13 weights and generate a single scaling factor. + # layer.w13_weight_scale = torch.nn.Parameter(torch.ones( + # layer.local_num_experts, + # dtype=torch.float32, + # device=w13_weight.device), + # requires_grad=False) + # for expert in range(layer.local_num_experts): + # w13_weight[expert, :, :], layer.w13_weight_scale[ + # expert] = ops.scaled_fp8_quant( + # layer.w13_weight.data[expert, :, :]) + # w2_weight[expert, :, :], layer.w2_weight_scale[ + # expert] = ops.scaled_fp8_quant( + # layer.w2_weight.data[expert, :, :]) + # layer.w13_weight = torch.nn.Parameter(w13_weight, + # requires_grad=False) + # layer.w2_weight = torch.nn.Parameter(w2_weight, + # requires_grad=False) + # if self.rocm_aiter_moe_enabled: + + # shuffled_w13, shuffled_w2 = shuffle_weights( + # layer.w13_weight, layer.w2_weight) + + # layer.w13_weight = torch.nn.Parameter(shuffled_w13, + # requires_grad=False) + # layer.w2_weight = torch.nn.Parameter(shuffled_w2, + # requires_grad=False) + # return + + # If checkpoint is fp8, we need to handle that the + # MoE kernels require single activation scale and single weight + # scale for w13 per expert. + else: + # Fp8 moe kernels require a single activation scale. + # We take the max of all the scales in case they differ. + if not self.quant_config["is_dynamic"]: + if (layer.w13_input_scale is None + or layer.w2_input_scale is None): + raise ValueError( + "QuantConfig has static quantization, but found " + "activation scales are None.") + # if (not all_close_1d(layer.w13_input_scale) + # or not all_close_1d(layer.w2_input_scale)): + # print( + # "Found input_scales that are not equal for " + # "fp8 MoE layer. Using the maximum across experts " + # "for each layer.") + layer.w13_input_scale = torch.nn.Parameter( + layer.w13_input_scale.max(), requires_grad=False) + layer.w2_input_scale = torch.nn.Parameter( + layer.w2_input_scale.max(), requires_grad=False) + if self.need_normalize_e4m3fn_to_e4m3fnuz: + # Normalize the weights and scales + w13_weight, w13_weight_scale, w13_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w13_weight, layer.w13_weight_scale, + layer.w13_input_scale) + w2_weight, w2_weight_scale, w2_input_scale = \ + normalize_e4m3fn_to_e4m3fnuz( + layer.w2_weight, layer.w2_weight_scale, + layer.w2_input_scale) + # Reset the parameter + layer.w13_weight = torch.nn.Parameter(w13_weight, + requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter( + w13_weight_scale, requires_grad=False) + if w13_input_scale is not None: + layer.w13_input_scale = torch.nn.Parameter( + w13_input_scale, requires_grad=False) + layer.w2_weight = torch.nn.Parameter(w2_weight, + requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, + requires_grad=False) + if w2_input_scale is not None: + layer.w2_input_scale = torch.nn.Parameter( + w2_input_scale, requires_grad=False) + + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max then dequant and requant each expert. + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.local_num_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start:start + + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id]) + layer.w13_weight[expert_id][ + start:start + shard_size, :], _ = scaled_fp8_quant( + dq_weight, max_w13_scales[expert_id]) + start += shard_size + + shuffled_w13, shuffled_w2 = shuffle_weights( + layer.w13_weight, layer.w2_weight) + + layer.w13_weight = torch.nn.Parameter(shuffled_w13, + requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, + requires_grad=False) + + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, + requires_grad=False) + return + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + router_logits: torch.Tensor, + top_k: int, + renormalize: bool, + use_grouped_topk: bool = False, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + e_score_correction_bias: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, + activation: ActivationType = ActivationType.Silu, + ) -> torch.Tensor: + topk_weights, topk_ids = FusedMoE.select_experts( + hidden_states=x, + router_logits=router_logits, + use_grouped_topk=use_grouped_topk, + top_k=top_k, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=custom_routing_function, + scoring_func=scoring_func, + e_score_correction_bias=e_score_correction_bias, + num_fused_shared_experts=layer.num_fused_shared_experts, + routed_scaling_factor=layer.routed_scaling_factor, + ) + return fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weight=topk_weights, + topk_ids=topk_ids, + expert_mask=expert_map, + activation=activation, + quant_type=self.quant_type, + w1_scale=(layer.w13_weight_scale_inv + if self.block_quant else layer.w13_weight_scale), + w2_scale=(layer.w2_weight_scale_inv + if self.block_quant else layer.w2_weight_scale), + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale) def determine_expert_map( @@ -87,6 +576,7 @@ def __init__( top_k: int, hidden_size: int, intermediate_size: int, + params_dtype: Optional[torch.dtype] = None, reduce_results: bool = False, renormalize: bool = True, use_grouped_topk: bool = False, @@ -101,7 +591,7 @@ def __init__( scoring_func: str = "softmax", e_score_correction_bias: Optional[torch.Tensor] = None, apply_router_weight_on_input: bool = False, - activation: str = "silu", + activation: ActivationType = ActivationType.Silu, config: Optional[PretrainedConfig] = None, ): super().__init__() @@ -109,7 +599,7 @@ def __init__( self.params_dtype = ( quant_config["quant_dtype"] if quant_config else torch.get_default_dtype() ) - + self.quant_config = quant_config # Note: here we guard against accessing the TP and DP groups when # uninitialized (this happens when testing) self.tp_size = tp_size if tp_size is not None else get_tp_group().world_size @@ -121,10 +611,9 @@ def __init__( if use_ep: # Set TP size to 1 to adjust for EP and adjust EP size and rank - # for DP attention. - self.ep_rank = tp_rank + self.tp_size * self.dp_rank + self.ep_rank = tp_rank self.tp_rank = 0 - self.ep_size = self.tp_size * self.dp_size + self.ep_size = self.tp_size self.tp_size = 1 self.local_num_experts, self.expert_map = determine_expert_map( @@ -134,9 +623,9 @@ def __init__( ) else: # Adjust TP size for DP attention - self.tp_rank = tp_rank + self.tp_size * self.dp_rank + self.tp_rank = tp_rank self.ep_rank = 0 - self.tp_size = self.tp_size * self.dp_size + self.tp_size = self.tp_size self.ep_size = 1 self.local_num_experts = self.global_num_experts self.expert_map = None @@ -149,7 +638,7 @@ def __init__( else 0 ) self.routed_scaling_factor = ( - config.routed_scaling_factor + getattr(config, 'routed_scaling_factor', 1.0) if config is not None and config.torch_dtype != torch.float16 else 1.0 ) @@ -186,7 +675,7 @@ def __init__( tp_rank=self.ep_rank if use_ep else tp_rank, tp_size=self.ep_size if use_ep else tp_size, shared_experts_score=1.0, - max_num_tokens=c, + max_num_tokens=config.max_num_batched_tokens, # Default max tokens is_EP=use_ep, ) if is_rocm_aiter_fusion_shared_expert_enabled(): @@ -212,12 +701,24 @@ def __init__( "Only softmax scoring function is supported for " "non-grouped topk." ) + # Note: get_quant_method will look at the layer's local_num_experts + # for heuristic purposes, so it must be initialized first. + if quant_config["quant_type"] == QuantType.No: + self.quant_method: Optional[QuantizeMethodBase] = ( + UnquantizedFusedMoEMethod()) + elif quant_config["quant_dtype"] == torch.float8_e4m3fnuz or quant_config["quant_dtype"] == torch.float8_e4m3fn: + self.quant_method = Fp8MoEMethod(quant_config) + else: + raise ValueError(f"Unsupported quant dtype: {quant_config['quant_dtype']}") + + assert self.quant_method is not None + self.apply_router_weight_on_input = apply_router_weight_on_input moe_quant_params = { "num_experts": self.local_num_experts, "hidden_size": hidden_size, "intermediate_size_per_partition": self.intermediate_size_per_partition, - "params_dtype": params_dtype, + "params_dtype": self.params_dtype, "weight_loader": self.weight_loader, } self.quant_method.create_weights(layer=self, **moe_quant_params) @@ -407,13 +908,6 @@ def weight_loader( # dimension intermediate_size_per_partition is used. SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0} - is_gguf_weight = getattr(param, "is_gguf_weight", False) - is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) - if is_gguf_weight_type: - param.weight_type = loaded_weight.item() - param.data.copy_(loaded_weight) - return - # is_transposed: if the dim to shard the weight # should be flipped. Required by GPTQ, compressed-tensors # should be whatever dimension intermediate_size_per_partition is @@ -556,12 +1050,7 @@ def select_experts( return topk_weights, topk_ids def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): - if self.use_direct_call: - return self.forward_impl(hidden_states, router_logits) - else: - return torch.ops.vllm.moe_forward( - hidden_states, router_logits, self.layer_name - ) + return self.forward_impl(hidden_states, router_logits) def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): assert self.quant_method is not None @@ -587,7 +1076,7 @@ def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor) if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): # Default set to False. (May have to add shared expert outputs.) - final_hidden_states = get_tp_group.all_reduce(final_hidden_states) + final_hidden_states = get_tp_group().all_reduce(final_hidden_states, ca_fp8_quant=False) return final_hidden_states diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index 40074b8ee3..31d1cebac9 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -62,7 +62,7 @@ def rocm_aiter_topk_softmax_impl( topk: int, renormalize: bool, num_fused_shared_experts: int = 0, -) -> None: +) -> tuple[torch.Tensor, torch.Tensor]: from aiter import topk_softmax token = gating_output.shape[0] @@ -103,9 +103,13 @@ def rocm_aiter_topk_softmax_fake( gating_output: torch.Tensor, topk: int, renormalize: bool, - num_fused_shared_experts: int, -) -> None: - pass + num_fused_shared_experts: int = 0, +) -> tuple[torch.Tensor, torch.Tensor]: + token = gating_output.shape[0] + device = gating_output.device + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + return topk_weights, topk_ids def rocm_aiter_biased_grouped_topk_impl( diff --git a/atom/model_ops/utils.py b/atom/model_ops/utils.py index 5ede735490..8be039d58b 100644 --- a/atom/model_ops/utils.py +++ b/atom/model_ops/utils.py @@ -1,7 +1,7 @@ from typing import Tuple, Optional, List, Union import torch from aiter import per_tensor_quant, dtypes - +from aiter.ops.shuffle import shuffle_weight def per_tensor_dequantize( tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor] @@ -72,3 +72,96 @@ def requantize_with_max_scale( start = end return max_w_scale, weight.view(quant_dtype) + + +def shuffle_weights( + *tensors: torch.Tensor, layout: tuple[int, int] = (16, 16) +) -> tuple[torch.Tensor, ...]: + """ + Applies shuffle_weight function from AITER to each + input tensor and returns them. + + Rearranges (shuffles) the input tensor/s + into a specified block layout for optimized computation. + + Args: + *tensors: Variable number of torch.Tensor objects. + layout: A pair of integers specifying the + block sizes used to divide the tensors during shuffling. + Default is (16, 16). + + Returns: + A Tuple of shuffled tensors. + """ + return tuple(shuffle_weight(tensor, layout=layout) for tensor in tensors) + + +def all_close_1d(x: torch.Tensor) -> bool: + assert len(x.shape) == 1 + return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0])) + + +def per_tensor_dequantize( + tensor: torch.Tensor, inv_scale: Union[float, + torch.Tensor]) -> torch.Tensor: + fake_qweight = tensor.to(torch.float16) + dq_weight = fake_qweight * inv_scale + return dq_weight + + +def scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, + num_token_padding: Optional[int] = None, + scale_ub: Optional[torch.Tensor] = None, + use_per_token_if_dynamic: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP8 and return quantized tensor and scale. + + This function supports both static and dynamic quantization: If you + provide the scale, it will use static scaling and if you omit it, + the scale will be determined dynamically. The function also allows + optional padding of the output tensors for downstream kernels that + will benefit from padding. + + Args: + input: The input tensor to be quantized to FP8 + scale: Optional scaling factor for the FP8 quantization + scale_ub: Optional upper bound for scaling factor in dynamic + per token case + num_token_padding: If specified, pad the first dimension + of the output to at least this value. + use_per_token_if_dynamic: Whether to do per_tensor or per_token + in the dynamic quantization case. + + Returns: + tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and + scaling factor. + """ + # This code assumes batch_dim and num_tokens are flattened + assert (input.ndim == 2) + shape: Union[tuple[int, int], torch.Size] = input.shape + # For ROCm on MI300, the output fp8 dtype is torch.float_e4m3fnuz + # TODO check rocm platfom, float_e4m3fnuz for now + out_dtype: torch.dtype = torch.float8_e4m3fnuz + if num_token_padding: + shape = (max(num_token_padding, input.shape[0]), shape[1]) + output = torch.empty(shape, device=input.device, dtype=out_dtype) + + if scale is None: + if use_per_token_if_dynamic: + scale = torch.empty((shape[0], 1), + device=input.device, + dtype=torch.float32) + torch.ops._C.dynamic_per_token_scaled_fp8_quant( + output, input.contiguous(), scale, scale_ub) + else: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale) + else: + # num_token_padding not implemented for this case + assert (scale.numel() == 1 or num_token_padding is None) + torch.ops._C.static_scaled_fp8_quant(output, input, scale) + + return output, scale \ No newline at end of file diff --git a/atom/models/mixtral.py b/atom/models/mixtral.py index a1909deaf7..65d8a13141 100644 --- a/atom/models/mixtral.py +++ b/atom/models/mixtral.py @@ -272,14 +272,13 @@ def __init__( config = atom_config.hf_config self.config = config cache_config = atom_config.kv_cache_dtype - quant_config = config.quant_config + quant_config = atom_config.quant_config self.vocab_size = config.vocab_size self.org_vocab_size = config.vocab_size self.embed_tokens = VocabParallelEmbedding( self.vocab_size, config.hidden_size, - org_num_embeddings=config.vocab_size, ) self.start_layer, self.end_layer, self.layers = make_layers( @@ -340,8 +339,6 @@ class MixtralForCausalLM(nn.Module): "q_proj": ("qkv_proj", "q"), "k_proj": ("qkv_proj", "k"), "v_proj": ("qkv_proj", "v"), - "gate_proj": ("gate_up_proj", 0), - "up_proj": ("gate_up_proj", 1), } def __init__(self, atom_config: Config, prefix: str = ""): @@ -357,7 +354,7 @@ def __init__(self, atom_config: Config, prefix: str = ""): config.hidden_size, org_num_embeddings=config.vocab_size, ) - if self.config.tie_word_embeddings: + if config.tie_word_embeddings: self.lm_head.weight = self.model.embed_tokens.weight self.make_empty_intermediate_tensors = ( From ca99a4f5d33b9f422dfe03eea088e2170b3b7744 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Tue, 23 Sep 2025 04:22:51 +0000 Subject: [PATCH 034/119] remove vllm dependency --- atom/model_engine/model_runner.py | 2 +- atom/model_ops/moe.py | 5 +- atom/model_ops/topK.py | 8 +-- atom/model_ops/utils.py | 97 ++++++++++++++----------------- 4 files changed, 53 insertions(+), 59 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 5ab1541823..af1e36bce5 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -178,7 +178,7 @@ def warmup_model(self): def allocate_kv_cache(self): config = self.config hf_config = config.hf_config - if not hasattr(hf_config, "head_dim"): + if not hasattr(hf_config, "head_dim") or hf_config.head_dim is None: hf_config.head_dim = hf_config.hidden_size // hf_config.num_attention_heads free, total = torch.cuda.mem_get_info() used = total - free diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 9836e5cf14..9a65f58202 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -6,6 +6,7 @@ from aiter.dist.parallel_state import get_tp_group from transformers import PretrainedConfig from aiter import QuantType, dtypes +from aiter import get_hip_quant from atom.config import QuantizationConfig from atom.model_ops.base_config import QuantizeMethodBase from atom.model_ops.topK import rocm_aiter_grouped_topk as grouped_topk @@ -18,7 +19,6 @@ shuffle_weights, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize, - scaled_fp8_quant, all_close_1d, ) from atom.model_loader.weight_utils import set_weight_attrs @@ -437,8 +437,9 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: layer.w13_weight[expert_id][start:start + shard_size, :], layer.w13_weight_scale[expert_id][shard_id]) + quant_func = get_hip_quant(self.quant_type) layer.w13_weight[expert_id][ - start:start + shard_size, :], _ = scaled_fp8_quant( + start:start + shard_size, :], _ = quant_func( dq_weight, max_w13_scales[expert_id]) start += shard_size diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index 31d1cebac9..1953199223 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -1,7 +1,7 @@ import torch from functools import cache, lru_cache from typing import Optional -from vllm.utils import direct_register_custom_op +from atom.model_ops.utils import direct_register_custom_op def is_rocm_aiter_fusion_shared_expert_enabled(): @@ -345,7 +345,7 @@ def rocm_aiter_topk_softmax( renormalize: bool, num_fused_shared_experts: int = 0, ) -> tuple[torch.Tensor, ...]: - return torch.ops.vllm.rocm_aiter_topk_softmax( + return torch.ops.aiter.rocm_aiter_topk_softmax( gating_output, topk, renormalize, num_fused_shared_experts ) @@ -363,7 +363,7 @@ def rocm_aiter_grouped_topk( routed_scaling_factor: float = 1.0, ) -> tuple[torch.Tensor, torch.Tensor]: if e_score_correction_bias is not None: - return torch.ops.vllm.rocm_aiter_biased_grouped_topk( + return torch.ops.aiter.rocm_aiter_biased_grouped_topk( gating_output, e_score_correction_bias, num_expert_group, @@ -375,7 +375,7 @@ def rocm_aiter_grouped_topk( ) else: assert scoring_func == "softmax" or scoring_func == "sigmoid" - return torch.ops.vllm.rocm_aiter_grouped_topk( + return torch.ops.aiter.rocm_aiter_grouped_topk( gating_output, num_expert_group, topk_group, diff --git a/atom/model_ops/utils.py b/atom/model_ops/utils.py index 8be039d58b..ce5eb95f03 100644 --- a/atom/model_ops/utils.py +++ b/atom/model_ops/utils.py @@ -1,7 +1,10 @@ from typing import Tuple, Optional, List, Union import torch +from torch.library import Library +from typing import Callable, Optional, Tuple from aiter import per_tensor_quant, dtypes from aiter.ops.shuffle import shuffle_weight +from aiter.ops.quant import per_tensor_quant def per_tensor_dequantize( tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor] @@ -109,59 +112,49 @@ def per_tensor_dequantize( return dq_weight -def scaled_fp8_quant( - input: torch.Tensor, - scale: Optional[torch.Tensor] = None, - num_token_padding: Optional[int] = None, - scale_ub: Optional[torch.Tensor] = None, - use_per_token_if_dynamic: bool = False, -) -> tuple[torch.Tensor, torch.Tensor]: - """ - Quantize input tensor to FP8 and return quantized tensor and scale. - - This function supports both static and dynamic quantization: If you - provide the scale, it will use static scaling and if you omit it, - the scale will be determined dynamically. The function also allows - optional padding of the output tensors for downstream kernels that - will benefit from padding. +aiter_lib = Library("aiter", "FRAGMENT") - Args: - input: The input tensor to be quantized to FP8 - scale: Optional scaling factor for the FP8 quantization - scale_ub: Optional upper bound for scaling factor in dynamic - per token case - num_token_padding: If specified, pad the first dimension - of the output to at least this value. - use_per_token_if_dynamic: Whether to do per_tensor or per_token - in the dynamic quantization case. - Returns: - tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and - scaling factor. +def direct_register_custom_op( + op_name: str, + op_func: Callable, + mutates_args: list[str], + fake_impl: Optional[Callable] = None, + target_lib: Optional[Library] = None, + dispatch_key: str = "CUDA", + tags: Tuple[torch.Tag, ...] = (), +): """ - # This code assumes batch_dim and num_tokens are flattened - assert (input.ndim == 2) - shape: Union[tuple[int, int], torch.Size] = input.shape - # For ROCm on MI300, the output fp8 dtype is torch.float_e4m3fnuz - # TODO check rocm platfom, float_e4m3fnuz for now - out_dtype: torch.dtype = torch.float8_e4m3fnuz - if num_token_padding: - shape = (max(num_token_padding, input.shape[0]), shape[1]) - output = torch.empty(shape, device=input.device, dtype=out_dtype) - - if scale is None: - if use_per_token_if_dynamic: - scale = torch.empty((shape[0], 1), - device=input.device, - dtype=torch.float32) - torch.ops._C.dynamic_per_token_scaled_fp8_quant( - output, input.contiguous(), scale, scale_ub) - else: - scale = torch.zeros(1, device=input.device, dtype=torch.float32) - torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale) - else: - # num_token_padding not implemented for this case - assert (scale.numel() == 1 or num_token_padding is None) - torch.ops._C.static_scaled_fp8_quant(output, input, scale) + `torch.library.custom_op` can have significant overhead because it + needs to consider complicated dispatching logic. This function + directly registers a custom op and dispatches it to the CUDA backend. + See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5 + for more details. + + By default, the custom op is registered to the vLLM library. If you + want to register it to a different library, you can pass the library + object to the `target_lib` argument. + + IMPORTANT: the lifetime of the operator is tied to the lifetime of the + library object. If you want to bind the operator to a different library, + make sure the library object is alive when the operator is used. + """ + import torch.library - return output, scale \ No newline at end of file + def _op_func(*args, **kwargs): + # if AITER_LOG_MORE >= 2: + # log_args(op_func, *args, **kwargs) + return op_func(*args, **kwargs) + + if hasattr(torch.library, "infer_schema"): + schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args) + else: + # for pytorch 2.4 + import torch._custom_op.impl + + schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args) + my_lib = target_lib or aiter_lib + my_lib.define(op_name + schema_str, tags=tags) + my_lib.impl(op_name, _op_func, dispatch_key=dispatch_key) + if fake_impl is not None: + my_lib._register_fake(op_name, fake_impl) \ No newline at end of file From d078d3517f858e7ad02e1be815798080b68c3b83 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Tue, 23 Sep 2025 07:05:51 +0000 Subject: [PATCH 035/119] filter duplicated safetensors file --- atom/model_loader/loader.py | 6 ++++-- atom/model_loader/weight_utils.py | 29 +++++++++++++++++++++++++++-- atom/models/mixtral.py | 2 +- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index 49ff738949..43b6d56498 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -3,7 +3,8 @@ import torch from torch import nn from safetensors import safe_open -from atom.model_loader.weight_utils import download_weights_from_hf +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME +from atom.model_loader.weight_utils import download_weights_from_hf, filter_duplicate_safetensors_files from atom.model_ops.base_config import QuantizeMethodBase from tqdm import tqdm @@ -22,7 +23,8 @@ def load_model(model: nn.Module, model_name_or_path: str): model_name_or_path, None, ["*.safetensors"], ignore_patterns=["original/*"] ) ) - for file in tqdm(glob(os.path.join(path, "*.safetensors"))): + hf_weights_files = filter_duplicate_safetensors_files(glob(os.path.join(path, "*.safetensors")), path, SAFE_WEIGHTS_INDEX_NAME) + for file in tqdm(hf_weights_files): print(f"Loading weights from {file}") with safe_open(file, "pt", "cpu") as f: for name in f.keys(): diff --git a/atom/model_loader/weight_utils.py b/atom/model_loader/weight_utils.py index d2161acc3d..e3b0cfffd6 100644 --- a/atom/model_loader/weight_utils.py +++ b/atom/model_loader/weight_utils.py @@ -12,7 +12,8 @@ import fnmatch import torch from pathlib import Path -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Optional, Union, List +import json logger = logging.getLogger(__name__) @@ -144,4 +145,28 @@ def set_weight_attrs( # This sometimes causes OOM errors during model loading. To avoid this, # we sync the param tensor after its weight loader is called. # TODO(woosuk): Remove this hack once we have a better solution. - setattr(weight, key, value) \ No newline at end of file + setattr(weight, key, value) + + +def filter_duplicate_safetensors_files(hf_weights_files: List[str], + hf_folder: str, + index_file: str) -> List[str]: + # model.safetensors.index.json is a mapping from keys in the + # torch state_dict to safetensors file holding that weight. + index_file_name = os.path.join(hf_folder, index_file) + if not os.path.isfile(index_file_name): + return hf_weights_files + + # Iterate through the weight_map (weight_name: safetensors files) + # to identify weights that we should use. + with open(index_file_name) as f: + weight_map = json.load(f)["weight_map"] + weight_files_in_index = set() + for weight_name in weight_map: + weight_files_in_index.add( + os.path.join(hf_folder, weight_map[weight_name])) + # Filter out any fields that are not found in the index file. + hf_weights_files = [ + f for f in hf_weights_files if f in weight_files_in_index + ] + return hf_weights_files \ No newline at end of file diff --git a/atom/models/mixtral.py b/atom/models/mixtral.py index 65d8a13141..b90c693446 100644 --- a/atom/models/mixtral.py +++ b/atom/models/mixtral.py @@ -180,7 +180,7 @@ def __init__( self.head_dim, self.scaling, num_kv_heads=self.num_kv_heads, - cache_config=cache_config, + kv_cache_dtype=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", ) From b636e7983837c7aa574b9314b8889f8069c626f1 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Tue, 23 Sep 2025 10:02:19 +0000 Subject: [PATCH 036/119] remove op register for topk --- atom/model_ops/topK.py | 32 +++----------------------- atom/model_ops/utils.py | 50 +---------------------------------------- 2 files changed, 4 insertions(+), 78 deletions(-) diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index 1953199223..625f6d5881 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -1,7 +1,6 @@ import torch from functools import cache, lru_cache from typing import Optional -from atom.model_ops.utils import direct_register_custom_op def is_rocm_aiter_fusion_shared_expert_enabled(): @@ -314,38 +313,13 @@ def rocm_aiter_grouped_topk_fake( return topk_weights, topk_ids -direct_register_custom_op( - op_name="rocm_aiter_topk_softmax", - op_func=rocm_aiter_topk_softmax_impl, - mutates_args=[], - fake_impl=rocm_aiter_topk_softmax_fake, - dispatch_key="CUDA", -) - -direct_register_custom_op( - op_name="rocm_aiter_biased_grouped_topk", - op_func=rocm_aiter_biased_grouped_topk_impl, - mutates_args=[], - fake_impl=rocm_aiter_biased_grouped_topk_fake, - dispatch_key="CUDA", -) - -direct_register_custom_op( - op_name="rocm_aiter_grouped_topk", - op_func=rocm_aiter_grouped_topk_impl, - mutates_args=[], - fake_impl=rocm_aiter_grouped_topk_fake, - dispatch_key="CUDA", -) - - def rocm_aiter_topk_softmax( gating_output: torch.Tensor, topk: int, renormalize: bool, num_fused_shared_experts: int = 0, ) -> tuple[torch.Tensor, ...]: - return torch.ops.aiter.rocm_aiter_topk_softmax( + return rocm_aiter_topk_softmax_impl( gating_output, topk, renormalize, num_fused_shared_experts ) @@ -363,7 +337,7 @@ def rocm_aiter_grouped_topk( routed_scaling_factor: float = 1.0, ) -> tuple[torch.Tensor, torch.Tensor]: if e_score_correction_bias is not None: - return torch.ops.aiter.rocm_aiter_biased_grouped_topk( + return rocm_aiter_biased_grouped_topk_impl( gating_output, e_score_correction_bias, num_expert_group, @@ -375,7 +349,7 @@ def rocm_aiter_grouped_topk( ) else: assert scoring_func == "softmax" or scoring_func == "sigmoid" - return torch.ops.aiter.rocm_aiter_grouped_topk( + return rocm_aiter_grouped_topk_impl( gating_output, num_expert_group, topk_group, diff --git a/atom/model_ops/utils.py b/atom/model_ops/utils.py index ce5eb95f03..6caf87e756 100644 --- a/atom/model_ops/utils.py +++ b/atom/model_ops/utils.py @@ -109,52 +109,4 @@ def per_tensor_dequantize( torch.Tensor]) -> torch.Tensor: fake_qweight = tensor.to(torch.float16) dq_weight = fake_qweight * inv_scale - return dq_weight - - -aiter_lib = Library("aiter", "FRAGMENT") - - -def direct_register_custom_op( - op_name: str, - op_func: Callable, - mutates_args: list[str], - fake_impl: Optional[Callable] = None, - target_lib: Optional[Library] = None, - dispatch_key: str = "CUDA", - tags: Tuple[torch.Tag, ...] = (), -): - """ - `torch.library.custom_op` can have significant overhead because it - needs to consider complicated dispatching logic. This function - directly registers a custom op and dispatches it to the CUDA backend. - See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5 - for more details. - - By default, the custom op is registered to the vLLM library. If you - want to register it to a different library, you can pass the library - object to the `target_lib` argument. - - IMPORTANT: the lifetime of the operator is tied to the lifetime of the - library object. If you want to bind the operator to a different library, - make sure the library object is alive when the operator is used. - """ - import torch.library - - def _op_func(*args, **kwargs): - # if AITER_LOG_MORE >= 2: - # log_args(op_func, *args, **kwargs) - return op_func(*args, **kwargs) - - if hasattr(torch.library, "infer_schema"): - schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args) - else: - # for pytorch 2.4 - import torch._custom_op.impl - - schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args) - my_lib = target_lib or aiter_lib - my_lib.define(op_name + schema_str, tags=tags) - my_lib.impl(op_name, _op_func, dispatch_key=dispatch_key) - if fake_impl is not None: - my_lib._register_fake(op_name, fake_impl) \ No newline at end of file + return dq_weight \ No newline at end of file From b3c66d21e80c2d44a525cd55e579ac19b5f34b5e Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Wed, 24 Sep 2025 06:33:34 +0000 Subject: [PATCH 037/119] fix eager --- atom/model_engine/model_runner.py | 71 +++++++++++++++++-------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index af1e36bce5..56410215dd 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -19,8 +19,8 @@ from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler from atom.models.llama import LlamaForCausalLM -from atom.models.qwen3 import Qwen3ForCausalLM from atom.models.mixtral import MixtralForCausalLM +from atom.models.qwen3 import Qwen3ForCausalLM from atom.utils.context import get_context, reset_context, set_context logger = logging.getLogger("atom") @@ -66,6 +66,7 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.sampler = Sampler() self.warmup_model() self.allocate_kv_cache() + self.allocate_decode_vars() if not self.enforce_eager: self.capture_cudagraph() torch.set_default_device("cpu") @@ -175,6 +176,21 @@ def warmup_model(self): self.run(dummy_batch) torch.cuda.empty_cache() + def allocate_decode_vars(self): + config = self.config + hidden_size = config.hf_config.hidden_size + hidden_type = config.hf_config.torch_dtype + max_bs = min(self.config.max_num_seqs, 512) + max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size + self.decode_vars = { + "input_ids": torch.zeros(max_bs, dtype=torch.int64), + "positions": torch.zeros(max_bs, dtype=torch.int64), + "slot_mapping": torch.zeros(max_bs, dtype=torch.int64), + "context_lens": torch.zeros(max_bs, dtype=torch.int32), + "block_tables": torch.zeros(max_bs, max_num_blocks, dtype=torch.int32), + "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), + } + def allocate_kv_cache(self): config = self.config hf_config = config.hf_config @@ -316,7 +332,9 @@ def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): def prepare_decode(self, scheduled_batchs: ScheduledBatchs): bs = len(scheduled_batchs.seqs) - graph_bs = next(x for x in self.graph_bs if x >= bs) + graph_bs = ( + bs if self.enforce_eager else next(x for x in self.graph_bs if x >= bs) + ) assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" input_ids = [] positions = [] @@ -350,13 +368,13 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - graph_vars = self.graph_vars - graph_vars["slot_mapping"][bs:graph_bs] = -1 - graph_vars["slot_mapping"][:bs].copy_(slot_mapping, non_blocking=True) - graph_vars["input_ids"][:bs].copy_(input_ids, non_blocking=True) - graph_vars["positions"][:bs].copy_(positions, non_blocking=True) - graph_vars["context_lens"][:bs].copy_(context_lens, non_blocking=True) - graph_vars["block_tables"][:bs, : block_tables.size(1)].copy_( + decode_vars = self.decode_vars + decode_vars["slot_mapping"][bs : graph_bs + 1] = -1 + decode_vars["slot_mapping"][:bs].copy_(slot_mapping, non_blocking=True) + decode_vars["input_ids"][:bs].copy_(input_ids, non_blocking=True) + decode_vars["positions"][:bs].copy_(positions, non_blocking=True) + decode_vars["context_lens"][:bs].copy_(context_lens, non_blocking=True) + decode_vars["block_tables"][:bs, : block_tables.size(1)].copy_( block_tables, non_blocking=True ) set_context( @@ -368,13 +386,13 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, - slot_mapping=graph_vars["slot_mapping"][:bs], - context_lens=graph_vars["context_lens"][:bs], - block_tables=graph_vars["block_tables"][:bs, : block_tables.size(1)], + slot_mapping=decode_vars["slot_mapping"][:bs], + context_lens=decode_vars["context_lens"][:bs], + block_tables=decode_vars["block_tables"][:bs, : block_tables.size(1)], dropout_p=dropout_p, ) - return graph_vars["input_ids"][:bs], graph_vars["positions"][:bs] + return decode_vars["input_ids"][:bs], decode_vars["positions"][:bs] def prepare_sample(self, seqs: list[Sequence]): temperatures = [] @@ -406,7 +424,7 @@ def run_model( else: graph_bs = ctx.graph_bs self.graphs[graph_bs].replay() - return self.model.compute_logits(self.graph_vars["outputs"][:bs]) + return self.model.compute_logits(self.decode_vars["outputs"][:bs]) def run(self, scheduled_batchs: ScheduledBatchs) -> list[int]: input_ids, positions, temperatures = self.prepare_model(scheduled_batchs) @@ -419,16 +437,12 @@ def run(self, scheduled_batchs: ScheduledBatchs) -> list[int]: @torch.inference_mode() def capture_cudagraph(self): - config = self.config - hf_config = config.hf_config - max_bs = min(self.config.max_num_seqs, 512) - max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size - input_ids = torch.zeros(max_bs, dtype=torch.int64) - positions = torch.zeros(max_bs, dtype=torch.int64) - slot_mapping = torch.zeros(max_bs, dtype=torch.int64) - context_lens = torch.zeros(max_bs, dtype=torch.int32) - block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32) - outputs = torch.zeros(max_bs, hf_config.hidden_size) + input_ids = self.decode_vars["input_ids"] + positions = self.decode_vars["positions"] + slot_mapping = self.decode_vars["slot_mapping"] + context_lens = self.decode_vars["context_lens"] + block_tables = self.decode_vars["block_tables"] + outputs = self.decode_vars["outputs"] # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) if self.config.compilation_config.cudagraph_capture_sizes: @@ -474,12 +488,3 @@ def capture_cudagraph(self): torch.cuda.synchronize() reset_context() self.graph_bs.sort(reverse=False) - - self.graph_vars = dict( - input_ids=input_ids, - positions=positions, - slot_mapping=slot_mapping, - context_lens=context_lens, - block_tables=block_tables, - outputs=outputs, - ) From 42f6bde285402940bc76104b8275d7a647089b36 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Wed, 24 Sep 2025 07:50:56 +0000 Subject: [PATCH 038/119] fix tp1 --- atom/model_engine/model_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 56410215dd..b49a25ca68 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -95,7 +95,7 @@ def signal_handler(signum, frame): def exit(self, msg=None): logger.info(f"Exiting ModelRunner for rank {self.rank}/{self.world_size} {msg}") - if dist.is_initialized(): + if dist.is_initialized() and self.world_size > 1: self.shm.close() dist.barrier() if self.rank == 0: From cfe90ea96dbc03c78513f96dcaced873a4081257 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 19 Sep 2025 17:49:26 +0800 Subject: [PATCH 039/119] support torch compile in atom --- atom/config.py | 224 +++++++++ atom/model_engine/model_runner.py | 10 +- atom/model_ops/linear.py | 751 ++++++++++++++++++------------ atom/models/qwen3.py | 16 +- atom/utils/__init__.py | 139 ++++++ atom/utils/backends.py | 661 ++++++++++++++++++++++++++ atom/utils/compiler_inferface.py | 464 ++++++++++++++++++ atom/utils/cuda_graph.py | 218 +++++++++ atom/utils/decorators.py | 312 +++++++++++++ bench.py | 6 +- 10 files changed, 2498 insertions(+), 303 deletions(-) create mode 100644 atom/utils/backends.py create mode 100644 atom/utils/compiler_inferface.py create mode 100644 atom/utils/cuda_graph.py create mode 100644 atom/utils/decorators.py diff --git a/atom/config.py b/atom/config.py index 351d41795c..0710381ba0 100644 --- a/atom/config.py +++ b/atom/config.py @@ -4,9 +4,67 @@ from typing import Optional import torch +from dataclasses import dataclass +from transformers import AutoConfig +from typing import Optional, Any, ClassVar +from dataclasses import field from aiter import QuantType from aiter.utility.dtypes import d_dtypes from transformers import AutoConfig, PretrainedConfig +import enum +import hashlib + +CUDA_CAPTURE = True + +def get_capture_status() -> bool: + return CUDA_CAPTURE + +def set_capture_status(capture): + global CUDA_CAPTURE + CUDA_CAPTURE = capture + + +class CUDAGraphMode(enum.Enum): + """ Constants for the cudagraph mode in CompilationConfig. + Meanwhile, the subset enum `NONE`, `PIECEWISE` and `FULL` are also + treated as concrete runtime mode for cudagraph runtime dispatching. + """ + NONE = 0 + PIECEWISE = 1 + FULL = 2 + FULL_DECODE_ONLY = (FULL, NONE) + FULL_AND_PIECEWISE = (FULL, PIECEWISE) + + def decode_mode(self) -> 'CUDAGraphMode': + return CUDAGraphMode(self.value[0]) if \ + self.separate_routine() else self + + def mixed_mode(self) -> 'CUDAGraphMode': + return CUDAGraphMode(self.value[1]) if \ + self.separate_routine() else self + + def requires_piecewise_compilation(self) -> bool: + return (self.decode_mode() == CUDAGraphMode.PIECEWISE + or self.mixed_mode() == CUDAGraphMode.PIECEWISE) + + def max_cudagraph_mode(self) -> 'CUDAGraphMode': + return CUDAGraphMode(max( + self.value)) if self.separate_routine() else self + + def has_full_cudagraphs(self) -> bool: + return self.max_cudagraph_mode() == CUDAGraphMode.FULL + + def separate_routine(self) -> bool: + return isinstance(self.value, tuple) + + + +class CompilationLevel: + # constants for the levels of the compilation process + NO_COMPILATION = 0 + DYNAMO_AS_IS = 1 + DYNAMO_ONCE = 2 + PIECEWISE = 3 @dataclass @@ -30,7 +88,80 @@ class CompilationConfig: pattern: [1, 2, 4] + [i for i in range(8, cuda_graph_sizes + 1, 8)] 3. more than one value (e.g. 1 2 128) is provided, then the capture list will follow the provided list.""" + debug_dump_path: str = "" + """The path to dump the debug information.""" + + """custom ops that are disabled""" + traced_files: set[str] = field(default_factory=set, init=False) + + cache_dir: str = "" + + # CudaGraph compilation + cudagraph_mode: Optional[CUDAGraphMode] = None + """ + The mode of the cudagraph: + + - NONE, no cudagraph capture. + - PIECEWISE. (v1 default) + - FULL. + - FULL_DECODE_ONLY. + - FULL_AND_PIECEWISE. + + PIECEWISE mode build piecewise cudagraph only, keeping the cudagraph + incompatiable ops (i.e. some attention ops) outside the cudagraph + for general flexibility. + This is the default mode. + + FULL mode: Capture full cudagraph for all batches. Can be good for small + models or workloads with small prompts; not supported by many backends. + Generally for performance FULL_AND_PIECEWISE is better. + + FULL_DECODE_ONLY mode: Capture full cudagraph for decode batches only. + Mixed prefill-decode batches are run without cudagraphs. Can be good for + decode instances in a P/D setup where prefill is not as important so we + can save some memory. + + FULL_AND_PIECEWISE mode: Capture full cudagraph for decode batches and + piecewise cudagraph for prefill and mixed prefill-decode batches. + This is like the most performant mode for most models. + + Currently, the cudagraph mode is only used for the v1 engine. + Note that the cudagraph logic is generally orthogonal to the + compilation logic. While piecewise cudagraphs require piecewise + compilation (level=PIECEWISE and non-empty splitting_ops), full + cudagraphs are supported with and without compilation. + + Warning: This flag is new and subject to change in addition + more modes may be added. + """ + compilation_time: float = field(default=0.0, init=False) + + splitting_ops: Optional[list[str]] = None + """A list of ops to split the full graph into subgraphs, used in piecewise + compilation.""" + + # splitting_ops: Optional[list[str]] = field(default_factory=list) + + + cudagraph_copy_inputs: bool = False + """Whether to copy input tensors for + cudagraph. If the caller can guarantee that the same input buffers + are always used, it can set this to False. Otherwise, it should + set this to True, and the compiler will copy the input to an + internally managed buffer. Default is False. + Note that this flag is only effective when cudagraph_mode is PIECEWISE. + """ + + _attention_ops: ClassVar[list[str]] = [ + "aiter.wrapper_fmha_v3_varlen_fwd", + # "vllm.unified_attention", + # "vllm.unified_attention_with_output", + # "vllm.mamba_mixer2", + # "vllm.mamba_mixer", + # "vllm.short_conv", + # "vllm.linear_attention", + ] def __post_init__(self): if self.level not in {0, 1, 2, 3}: @@ -38,6 +169,47 @@ def __post_init__(self): if not self.cuda_graph_sizes: self.cuda_graph_sizes = [512] + def compute_hash(self) -> str: + """ + WARNING: Whenever a new field is added to this config, + ensure that it is included in the factors list if + it affects the computation graph. + + Provide a hash that uniquely identifies all the configs + that affect the structure of the computation + graph from input ids/embeddings to the final hidden states, + excluding anything before input ids/embeddings and after + the final hidden states. + """ + factors: list[Any] = [] + factors.append(self.level) + factors.append(self.use_cudagraph) + factors.append(self.local_cache_dir) + factors.append(self.cudagraph_capture_sizes) + factors.append(self.cuda_graph_sizes) + factors.append(self.debug_dump_path) + factors.append(self.traced_files) + factors.append(self.cache_dir) + return hashlib.sha256(str(factors).encode()).hexdigest() + + + def set_splitting_ops_for_v1(self): + # NOTE: this function needs to be called only when level is + # CompilationLevel.PIECEWISE + assert self.level == CompilationLevel.PIECEWISE, ( + "set_splitting_ops_for_v1 should only be called when " + "level is CompilationLevel.PIECEWISE") + + if self.splitting_ops is None: + # NOTE: When using full cudagraph, instead of setting an empty + # list and capture the full cudagraph inside the flattened fx + # graph, we keep the piecewise fx graph structure but capture the + # full cudagraph outside the fx graph. This reduces some cpu + # overhead when the runtime batch_size is not cudagraph captured. + # see https://github.com/vllm-project/vllm/pull/20059 for details. + self.splitting_ops = self._attention_ops + + class QuantizationConfig(dict): def __init__( @@ -52,6 +224,27 @@ def __init__( def get_name(self): return self["quant_name"] + def compute_hash(self) -> str: + """ + WARNING: Whenever a new field is added to this config, + ensure that it is included in the factors list if + it affects the computation graph. + + Provide a hash that uniquely identifies all the configs + that affect the structure of the computation + graph from input ids/embeddings to the final hidden states, + excluding anything before input ids/embeddings and after + the final hidden states. + """ + factors: list[Any] = [] + factors.append(self["quant_type"]) + factors.append(self["quant_dtype"]) + factors.append(self["quant_name"]) + factors.append(self["is_dynamic"]) + str_factors = str(factors) + # assert_hashable(str_factors) + return hashlib.sha256(str(factors).encode()).hexdigest() + def get_quant_config(config: PretrainedConfig) -> QuantizationConfig: torch_dtype = getattr(config, "torch_dtype", "bf16") @@ -146,3 +339,34 @@ def __post_init__(self): assert self.torch_profiler_dir is None or os.path.isdir( self.torch_profiler_dir ), f"torch_profiler_dir {self.torch_profiler_dir} is not a valid directory" + if self.compilation_config.level == CompilationLevel.PIECEWISE: + self.compilation_config.set_splitting_ops_for_v1() + + def compute_hash(self) -> str: + """ + WARNING: Whenever a new field is added to this config, + ensure that it is included in the factors list if + it affects the computation graph. + + Provide a hash that uniquely identifies all the configs + that affect the structure of the computation + graph from input ids/embeddings to the final hidden states, + excluding anything before input ids/embeddings and after + the final hidden states. + """ + factors: list[Any] = [] + + # summarize vllm config + vllm_factors: list[Any] = [] + if self.quant_config: + vllm_factors.append(self.quant_config.compute_hash()) + + if self.compilation_config: + vllm_factors.append(self.compilation_config.compute_hash()) + + factors.append(vllm_factors) + + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] + return hash_str + diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index b49a25ca68..264b57a339 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -71,8 +71,8 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.capture_cudagraph() torch.set_default_device("cpu") torch.set_default_dtype(default_dtype) - if self.config.compilation_config.level == 3: - self.model = torch.compile(self.model, fullgraph=True, backend="eager") + # if self.config.compilation_config.level == 1: + # self.model = torch.compile(self.model, fullgraph=True, backend="eager") if self.world_size > 1: if rank == 0: @@ -169,7 +169,7 @@ def warmup_model(self): max_num_batched_tokens // max_model_len, self.config.max_num_seqs ) seqs = [ - Sequence([0] * max_model_len, block_size=self.block_size) + Sequence([0] * max_num_batched_tokens, block_size=self.block_size) for _ in range(num_seqs) ] dummy_batch = ScheduledBatchs(seqs, True, False) @@ -470,6 +470,10 @@ def capture_cudagraph(self): if get_tensor_model_parallel_rank() == 0: capture_range.set_description(f"Capturing {bs=}") graph = torch.cuda.CUDAGraph() + + cu_seqlens_q = torch.arange(0, bs + 1, dtype=torch.int32, device='cuda') + cu_seqlens_k = cu_seqlens_q.clone() + set_context( False, batch_size=bs, diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 035fc9d494..48cdfdcd65 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -26,212 +26,419 @@ def divide(numerator, denominator): return numerator // denominator +# class LinearBase(nn.Module): + +# def __init__( +# self, +# input_size: int, +# output_size: int | list[int], +# tp_dim: int | None = None, +# bias: bool = False, +# quant_config: Optional[QuantizationConfig] = None, +# ): +# if quant_config is None: +# quant_config = QuantizationConfig() +# quant_type = quant_config["quant_type"] +# params_dtype = quant_config["quant_dtype"] +# super().__init__() +# self.input_size = input_size +# self.output_size = ( +# output_size if isinstance(output_size, int) else sum(output_size) +# ) +# self.tp_dim = tp_dim +# self.tp_rank = get_tp_group().rank +# self.tp_size = get_tp_group().world_size +# self.output_partition_sizes = ( +# output_size if isinstance(output_size, list) else [output_size] +# ) +# if tp_dim == 1: +# self.input_size = divide(input_size, self.tp_size) +# elif tp_dim == 0: +# self.output_size = divide(self.output_size, self.tp_size) +# self.output_partition_sizes = [ +# divide(s, self.tp_size) for s in self.output_partition_sizes +# ] +# self.weight = nn.Parameter( +# torch.empty((self.output_size, self.input_size), dtype=params_dtype), +# requires_grad=False, +# ) +# if bias: +# self.bias = nn.Parameter( +# torch.empty(self.output_size, dtype=params_dtype), requires_grad=False +# ) +# self.bias.weight_loader_process = self.weight_loader_process +# else: +# self.register_parameter("bias", None) +# self.quant_type = quant_type +# self.params_dtype = params_dtype +# if quant_type != QuantType.No: +# if quant_type == QuantType.per_Tensor: +# self.weight_scale = nn.Parameter( +# torch.empty(len(self.output_partition_sizes), 1, dtype=dtypes.fp32), +# requires_grad=False, +# ) +# if not quant_config["is_dynamic"]: +# self.input_scale = nn.Parameter( +# torch.empty( +# len(self.output_partition_sizes), 1, dtype=dtypes.fp32 +# ), +# requires_grad=False, +# ) +# self.input_scale.weight_loader_process = self.weight_loader_process +# self.input_scale.weight_loader = self.weight_loader +# elif quant_type == QuantType.per_Token: +# self.weight_scale = nn.Parameter( +# torch.empty(self.output_size, 1, dtype=dtypes.fp32), +# requires_grad=False, +# ) +# elif quant_type == QuantType.per_1x128: +# self.weight_scale = nn.Parameter( +# torch.empty( +# divide(self.output_size, 128), +# (self.input_size + 127) // 128, +# dtype=dtypes.fp32, +# ), +# requires_grad=False, +# ) +# elif quant_type == QuantType.per_1x32: +# self.weight_scale = nn.Parameter( +# torch.empty( +# self.output_size, +# (self.input_size + 31) // 32, +# dtype=dtypes.fp8_e8m0, +# ), +# requires_grad=False, +# ) +# self.weight.weight_loader_process = self.weight_loader_process +# self.weight_scale.weight_loader_process = self.weight_loader_process +# else: +# self.weight.weight_loader_process = self.weight_loader_process +# self.register_parameter("weight_scale", None) +# self.weight.weight_loader = self.weight_loader +# if self.bias is not None: +# self.bias.weight_loader = self.weight_loader +# if self.weight_scale is not None: +# self.weight_scale.weight_loader = self.weight_loader +# self.need_normalize_e4m3fn_to_e4m3fnuz = params_dtype == torch.float8_e4m3fnuz + +# @staticmethod +# def weight_loader_process( +# param: nn.Parameter, +# loaded_weight: torch.Tensor, +# post_process_func: Callable = lambda a: a, +# ): +# if ( +# param.data.dtype != loaded_weight.dtype +# and param.data.element_size() == loaded_weight.element_size() +# ): +# param.data = param.data.view(loaded_weight.dtype) +# param.data.copy_(post_process_func(loaded_weight)) + +# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): +# param_data = param.data +# param.weight_loader_process(param_data, loaded_weight) + +# def process_weights_after_loading(self): +# if ( +# self.quant_type == QuantType.per_Tensor +# and len(self.output_partition_sizes) > 1 +# ): +# weight_scale, weight = requantize_with_max_scale( +# weight=self.weight.data, +# weight_scale=self.weight_scale.data, +# logical_widths=self.output_partition_sizes, +# normalize_e4m3fn_to_e4m3fnuz=self.need_normalize_e4m3fn_to_e4m3fnuz, +# ) +# self.weight.data = weight +# self.weight_scale.data = weight_scale.view(-1) +# if hasattr(self, "input_scale"): +# self.input_scale.data = ( +# self.input_scale.data.max() * 2.0 +# if self.need_normalize_e4m3fn_to_e4m3fnuz +# else self.input_scale.data.max() +# ) +# elif self.need_normalize_e4m3fn_to_e4m3fnuz: +# self.weight.data, self.weight_scale.data, _ = normalize_e4m3fn_to_e4m3fnuz( +# self.weight.data, self.weight_scale.data +# ) +# if ( +# self.quant_type == QuantType.per_Token and self.params_dtype == dtypes.fp8 +# ) or (self.quant_type == QuantType.per_1x32): +# self.weight.data = shuffle_weight(self.weight.data, (16, 16)) + +# def forward( +# self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 +# ) -> torch.Tensor: +# if self.quant_type.value == QuantType.No.value: +# y = tgemm.mm(x, self.weight, self.bias) +# else: +# if x_scale is None: +# quant_func = get_hip_quant(self.quant_type) +# x, x_scale = quant_func( +# x, +# quant_dtype=self.params_dtype, +# scale=getattr(self, "input_scale", None), +# ) +# if self.quant_type.value == QuantType.per_Tensor.value: +# y = tgemm.mm( +# x, +# self.weight, +# self.bias, +# otype=otype, +# scale_a=x_scale, +# scale_b=self.weight_scale, +# ) +# elif self.quant_type.value == QuantType.per_Token.value: +# if self.params_dtype == dtypes.i8: +# y = gemm_a8w8( +# x, +# self.weight, +# x_scale, +# self.weight_scale, +# self.bias, +# dtype=otype, +# ) +# else: +# y = gemm_a8w8_bpreshuffle( +# x, +# self.weight, +# x_scale, +# self.weight_scale, +# self.bias, +# dtype=otype, +# ) +# elif self.quant_type.value == QuantType.per_1x128.value: +# y = gemm_a8w8_blockscale( +# x, self.weight, x_scale, self.weight_scale, dtype=otype +# ) +# if self.bias is not None: +# y += self.bias +# elif self.quant_type.value == QuantType.per_1x32.value: +# m = x.view(-1, x.size(-1)).shape[0] +# y = torch.empty( +# ((m + 31) // 32 * 32, self.output_size), +# dtype=otype, +# device=x.device, +# ) +# y = gemm_a4w4( +# x, +# self.weight, +# x_scale, +# self.weight_scale, +# y, +# self.bias, +# dtype=otype, +# ) +# if self.tp_dim == 1 and self.tp_size > 1: +# y = get_tp_group().all_reduce(y, open_fp8_quant=False) +# return y + + +# class ReplicatedLinear(LinearBase): + +# def __init__( +# self, +# input_size: int, +# output_size: int, +# bias: bool = False, +# quant_config: Optional[QuantizationConfig] = None, +# **kwargs, +# ): +# super().__init__( +# input_size, +# output_size, +# tp_dim=None, +# bias=bias, +# quant_config=quant_config, +# ) + +# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): +# param_data = param.data +# param.weight_loader_process(param_data, loaded_weight) + + +# class ColumnParallelLinear(LinearBase): + +# def __init__( +# self, +# input_size: int, +# output_size: int, +# bias: bool = False, +# quant_config: Optional[QuantizationConfig] = None, +# **kwargs, +# ): +# self.tp_dim = 0 +# super().__init__( +# input_size, +# output_size, +# self.tp_dim, +# bias, +# quant_config=quant_config, +# ) + +# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): +# param_data = param.data +# shard_size = param_data.size(self.tp_dim) +# start_idx = self.tp_rank * shard_size +# loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) +# param.weight_loader_process(param_data, loaded_weight) + + +# class MergedColumnParallelLinear(LinearBase): + +# def __init__( +# self, +# input_size: int, +# output_sizes: list[int], +# bias: bool = False, +# quant_config: Optional[QuantizationConfig] = None, +# **kwargs, +# ): +# self.output_sizes = output_sizes +# super().__init__( +# input_size, +# output_sizes, +# tp_dim=0, +# bias=bias, +# quant_config=quant_config, +# ) + +# def weight_loader( +# self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int +# ): +# param_data = param.data +# shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size +# shard_size = self.output_sizes[loaded_shard_id] // self.tp_size +# if param is getattr(self, "weight_scale", None) or param is getattr( +# self, "input_scale", None +# ): +# if self.quant_type == QuantType.per_1x128: +# shard_offset //= 128 +# shard_size //= 128 +# elif self.quant_type == QuantType.per_Tensor: +# loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) +# shard_offset = loaded_shard_id +# shard_size = 1 + +# param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) +# loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] +# param.weight_loader_process(param_data, loaded_weight) + + +# class QKVParallelLinear(ColumnParallelLinear): + +# def __init__( +# self, +# hidden_size: int, +# head_size: int, +# total_num_heads: int, +# total_num_kv_heads: int | None = None, +# bias: bool = False, +# quant_config: Optional[QuantizationConfig] = None, +# **kwargs, +# ): +# self.head_size = head_size +# self.total_num_heads = total_num_heads +# self.total_num_kv_heads = total_num_kv_heads or total_num_heads +# tp_size = get_tp_group().world_size +# self.num_heads = divide(self.total_num_heads, tp_size) +# self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) +# input_size = hidden_size +# output_sizes = [ +# self.total_num_heads * self.head_size, +# self.total_num_kv_heads * self.head_size, +# self.total_num_kv_heads * self.head_size, +# ] + +# super().__init__( +# input_size, +# output_sizes, +# bias=bias, +# quant_config=quant_config, +# ) + +# def weight_loader( +# self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str +# ): +# param_data = param.data +# assert loaded_shard_id in ["q", "k", "v"] +# if loaded_shard_id == "q": +# shard_size = self.num_heads * self.head_size +# shard_offset = 0 +# elif loaded_shard_id == "k": +# shard_size = self.num_kv_heads * self.head_size +# shard_offset = self.num_heads * self.head_size +# else: +# shard_size = self.num_kv_heads * self.head_size +# shard_offset = ( +# self.num_heads * self.head_size + self.num_kv_heads * self.head_size +# ) +# if param is getattr(self, "weight_scale", None) or param is getattr( +# self, "input_scale", None +# ): +# if self.quant_type == QuantType.per_1x128: +# shard_offset //= 128 +# shard_size //= 128 +# elif self.quant_type == QuantType.per_Tensor: +# loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) +# shard_offset = ["q", "k", "v"].index(loaded_shard_id) +# shard_size = 1 + +# param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) +# loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] +# param.weight_loader_process(param_data, loaded_weight) + + +# class RowParallelLinear(LinearBase): + +# def __init__( +# self, +# input_size: int, +# output_size: int, +# bias: bool = False, +# quant_config: Optional[QuantizationConfig] = None, +# **kwargs, +# ): +# self.tp_rank = get_tp_group().rank +# super().__init__( +# input_size, +# output_size, +# tp_dim=1, +# bias=bias if self.tp_rank == 0 else False, +# quant_config=quant_config, +# ) + +# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): +# param_data = param.data +# shard_size = param_data.size(self.tp_dim) +# if len(loaded_weight.shape) == 0: +# loaded_weight = loaded_weight.view(1, 1) +# if loaded_weight.size(self.tp_dim) == 1 and self.tp_size > 1: +# loaded_weight = loaded_weight.repeat(1, self.tp_size) +# start_idx = self.tp_rank * shard_size +# loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) +# param.weight_loader_process(param_data, loaded_weight) + + +import torch.distributed as dist + class LinearBase(nn.Module): def __init__( self, input_size: int, - output_size: int | list[int], + output_size: int, tp_dim: int | None = None, - bias: bool = False, - quant_config: Optional[QuantizationConfig] = None, ): - if quant_config is None: - quant_config = QuantizationConfig() - quant_type = quant_config["quant_type"] - params_dtype = quant_config["quant_dtype"] super().__init__() self.input_size = input_size - self.output_size = ( - output_size if isinstance(output_size, int) else sum(output_size) - ) + self.output_size = output_size self.tp_dim = tp_dim - self.tp_rank = get_tp_group().rank - self.tp_size = get_tp_group().world_size - self.output_partition_sizes = ( - output_size if isinstance(output_size, list) else [output_size] - ) - if tp_dim == 1: - self.input_size = divide(input_size, self.tp_size) - elif tp_dim == 0: - self.output_size = divide(self.output_size, self.tp_size) - self.output_partition_sizes = [ - divide(s, self.tp_size) for s in self.output_partition_sizes - ] - self.weight = nn.Parameter( - torch.empty((self.output_size, self.input_size), dtype=params_dtype), - requires_grad=False, - ) - if bias: - self.bias = nn.Parameter( - torch.empty(self.output_size, dtype=params_dtype), requires_grad=False - ) - self.bias.weight_loader_process = self.weight_loader_process - else: - self.register_parameter("bias", None) - self.quant_type = quant_type - self.params_dtype = params_dtype - if quant_type != QuantType.No: - if quant_type == QuantType.per_Tensor: - self.weight_scale = nn.Parameter( - torch.empty(len(self.output_partition_sizes), 1, dtype=dtypes.fp32), - requires_grad=False, - ) - if not quant_config["is_dynamic"]: - self.input_scale = nn.Parameter( - torch.empty( - len(self.output_partition_sizes), 1, dtype=dtypes.fp32 - ), - requires_grad=False, - ) - self.input_scale.weight_loader_process = self.weight_loader_process - self.input_scale.weight_loader = self.weight_loader - elif quant_type == QuantType.per_Token: - self.weight_scale = nn.Parameter( - torch.empty(self.output_size, 1, dtype=dtypes.fp32), - requires_grad=False, - ) - elif quant_type == QuantType.per_1x128: - self.weight_scale = nn.Parameter( - torch.empty( - divide(self.output_size, 128), - (self.input_size + 127) // 128, - dtype=dtypes.fp32, - ), - requires_grad=False, - ) - elif quant_type == QuantType.per_1x32: - self.weight_scale = nn.Parameter( - torch.empty( - self.output_size, - (self.input_size + 31) // 32, - dtype=dtypes.fp8_e8m0, - ), - requires_grad=False, - ) - self.weight.weight_loader_process = self.weight_loader_process - self.weight_scale.weight_loader_process = self.weight_loader_process - else: - self.weight.weight_loader_process = self.weight_loader_process - self.register_parameter("weight_scale", None) - self.weight.weight_loader = self.weight_loader - if self.bias is not None: - self.bias.weight_loader = self.weight_loader - if self.weight_scale is not None: - self.weight_scale.weight_loader = self.weight_loader - self.need_normalize_e4m3fn_to_e4m3fnuz = params_dtype == torch.float8_e4m3fnuz - - @staticmethod - def weight_loader_process( - param: nn.Parameter, - loaded_weight: torch.Tensor, - post_process_func: Callable = lambda a: a, - ): - if ( - param.data.dtype != loaded_weight.dtype - and param.data.element_size() == loaded_weight.element_size() - ): - param.data = param.data.view(loaded_weight.dtype) - param.data.copy_(post_process_func(loaded_weight)) + self.tp_rank = dist.get_rank() + self.tp_size = dist.get_world_size() - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): - param_data = param.data - param.weight_loader_process(param_data, loaded_weight) - - def process_weights_after_loading(self): - if ( - self.quant_type == QuantType.per_Tensor - and len(self.output_partition_sizes) > 1 - ): - weight_scale, weight = requantize_with_max_scale( - weight=self.weight.data, - weight_scale=self.weight_scale.data, - logical_widths=self.output_partition_sizes, - normalize_e4m3fn_to_e4m3fnuz=self.need_normalize_e4m3fn_to_e4m3fnuz, - ) - self.weight.data = weight - self.weight_scale.data = weight_scale.view(-1) - if hasattr(self, "input_scale"): - self.input_scale.data = ( - self.input_scale.data.max() * 2.0 - if self.need_normalize_e4m3fn_to_e4m3fnuz - else self.input_scale.data.max() - ) - elif self.need_normalize_e4m3fn_to_e4m3fnuz: - self.weight.data, self.weight_scale.data, _ = normalize_e4m3fn_to_e4m3fnuz( - self.weight.data, self.weight_scale.data - ) - if ( - self.quant_type == QuantType.per_Token and self.params_dtype == dtypes.fp8 - ) or (self.quant_type == QuantType.per_1x32): - self.weight.data = shuffle_weight(self.weight.data, (16, 16)) - - def forward( - self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 - ) -> torch.Tensor: - if self.quant_type.value == QuantType.No.value: - y = tgemm.mm(x, self.weight, self.bias) - else: - if x_scale is None: - quant_func = get_hip_quant(self.quant_type) - x, x_scale = quant_func( - x, - quant_dtype=self.params_dtype, - scale=getattr(self, "input_scale", None), - ) - if self.quant_type.value == QuantType.per_Tensor.value: - y = tgemm.mm( - x, - self.weight, - self.bias, - otype=otype, - scale_a=x_scale, - scale_b=self.weight_scale, - ) - elif self.quant_type.value == QuantType.per_Token.value: - if self.params_dtype == dtypes.i8: - y = gemm_a8w8( - x, - self.weight, - x_scale, - self.weight_scale, - self.bias, - dtype=otype, - ) - else: - y = gemm_a8w8_bpreshuffle( - x, - self.weight, - x_scale, - self.weight_scale, - self.bias, - dtype=otype, - ) - elif self.quant_type.value == QuantType.per_1x128.value: - y = gemm_a8w8_blockscale( - x, self.weight, x_scale, self.weight_scale, dtype=otype - ) - if self.bias is not None: - y += self.bias - elif self.quant_type.value == QuantType.per_1x32.value: - m = x.view(-1, x.size(-1)).shape[0] - y = torch.empty( - ((m + 31) // 32 * 32, self.output_size), - dtype=otype, - device=x.device, - ) - y = gemm_a4w4( - x, - self.weight, - x_scale, - self.weight_scale, - y, - self.bias, - dtype=otype, - ) - if self.tp_dim == 1 and self.tp_size > 1: - y = get_tp_group().all_reduce(y, ca_fp8_quant=False) - return y + def forward(self, x: torch.Tensor) -> torch.Tensor: + raise NotImplementedError class ReplicatedLinear(LinearBase): @@ -241,20 +448,21 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - quant_config: Optional[QuantizationConfig] = None, - **kwargs, ): - super().__init__( - input_size, - output_size, - tp_dim=None, - bias=bias, - quant_config=quant_config, - ) + super().__init__(input_size, output_size) + self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): - param_data = param.data - param.weight_loader_process(param_data, loaded_weight) + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) class ColumnParallelLinear(LinearBase): @@ -264,65 +472,48 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - quant_config: Optional[QuantizationConfig] = None, - **kwargs, ): - self.tp_dim = 0 - super().__init__( - input_size, - output_size, - self.tp_dim, - bias, - quant_config=quant_config, - ) + super().__init__(input_size, output_size, 0) + self.input_size_per_partition = input_size + self.output_size_per_partition = divide(output_size, self.tp_size) + + self.weight = nn.Parameter(torch.empty(self.output_size_per_partition, self.input_size)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size_per_partition)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param.weight_loader_process(param_data, loaded_weight) + param_data.copy_(loaded_weight) + def forward(self, x: torch.Tensor) -> torch.Tensor: + return F.linear(x, self.weight, self.bias) -class MergedColumnParallelLinear(LinearBase): + +class MergedColumnParallelLinear(ColumnParallelLinear): def __init__( self, input_size: int, output_sizes: list[int], bias: bool = False, - quant_config: Optional[QuantizationConfig] = None, - **kwargs, ): self.output_sizes = output_sizes - super().__init__( - input_size, - output_sizes, - tp_dim=0, - bias=bias, - quant_config=quant_config, - ) - - def weight_loader( - self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int - ): + super().__init__(input_size, sum(output_sizes), bias=bias) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int): param_data = param.data shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size shard_size = self.output_sizes[loaded_shard_id] // self.tp_size - if param is getattr(self, "weight_scale", None) or param is getattr( - self, "input_scale", None - ): - if self.quant_type == QuantType.per_1x128: - shard_offset //= 128 - shard_size //= 128 - elif self.quant_type == QuantType.per_Tensor: - loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) - shard_offset = loaded_shard_id - shard_size = 1 - param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] - param.weight_loader_process(param_data, loaded_weight) + param_data.copy_(loaded_weight) class QKVParallelLinear(ColumnParallelLinear): @@ -334,32 +525,18 @@ def __init__( total_num_heads: int, total_num_kv_heads: int | None = None, bias: bool = False, - quant_config: Optional[QuantizationConfig] = None, - **kwargs, ): self.head_size = head_size self.total_num_heads = total_num_heads self.total_num_kv_heads = total_num_kv_heads or total_num_heads - tp_size = get_tp_group().world_size + tp_size = dist.get_world_size() self.num_heads = divide(self.total_num_heads, tp_size) self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) input_size = hidden_size - output_sizes = [ - self.total_num_heads * self.head_size, - self.total_num_kv_heads * self.head_size, - self.total_num_kv_heads * self.head_size, - ] - - super().__init__( - input_size, - output_sizes, - bias=bias, - quant_config=quant_config, - ) - - def weight_loader( - self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str - ): + output_size = (self.total_num_heads + 2 * self.total_num_kv_heads) * self.head_size + super().__init__(input_size, output_size, bias) + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str): param_data = param.data assert loaded_shard_id in ["q", "k", "v"] if loaded_shard_id == "q": @@ -370,23 +547,10 @@ def weight_loader( shard_offset = self.num_heads * self.head_size else: shard_size = self.num_kv_heads * self.head_size - shard_offset = ( - self.num_heads * self.head_size + self.num_kv_heads * self.head_size - ) - if param is getattr(self, "weight_scale", None) or param is getattr( - self, "input_scale", None - ): - if self.quant_type == QuantType.per_1x128: - shard_offset //= 128 - shard_size //= 128 - elif self.quant_type == QuantType.per_Tensor: - loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) - shard_offset = ["q", "k", "v"].index(loaded_shard_id) - shard_size = 1 - + shard_offset = self.num_heads * self.head_size + self.num_kv_heads * self.head_size param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] - param.weight_loader_process(param_data, loaded_weight) + param_data.copy_(loaded_weight) class RowParallelLinear(LinearBase): @@ -396,25 +560,28 @@ def __init__( input_size: int, output_size: int, bias: bool = False, - quant_config: Optional[QuantizationConfig] = None, - **kwargs, ): - self.tp_rank = get_tp_group().rank - super().__init__( - input_size, - output_size, - tp_dim=1, - bias=bias if self.tp_rank == 0 else False, - quant_config=quant_config, - ) + super().__init__(input_size, output_size, 1) + self.input_size_per_partition = divide(input_size, self.tp_size) + self.output_size_per_partition = output_size + + self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size_per_partition)) + self.weight.weight_loader = self.weight_loader + if bias: + self.bias = nn.Parameter(torch.empty(self.output_size)) + self.bias.weight_loader = self.weight_loader + else: + self.register_parameter("bias", None) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) - if len(loaded_weight.shape) == 0: - loaded_weight = loaded_weight.view(1, 1) - if loaded_weight.size(self.tp_dim) == 1 and self.tp_size > 1: - loaded_weight = loaded_weight.repeat(1, self.tp_size) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param.weight_loader_process(param_data, loaded_weight) + param_data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None) + if self.tp_size > 1: + dist.all_reduce(y) + return y diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 660523ff3c..2515c6e5a7 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -15,6 +15,7 @@ MergedColumnParallelLinear, RowParallelLinear, ) +from atom.utils.decorators import support_torch_compile # from atom.model_ops.rotary_embedding import get_rope from aiter.rotary_embedding import get_rope @@ -56,13 +57,13 @@ def __init__( self.total_num_heads, self.total_num_kv_heads, bias=qkv_bias, - quant_config=quant_config, + # quant_config=quant_config, ) self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, hidden_size, bias=False, - quant_config=quant_config, + # quant_config=quant_config, ) self.rotary_emb = get_rope( self.head_dim, @@ -112,13 +113,13 @@ def __init__( hidden_size, [intermediate_size] * 2, bias=False, - quant_config=quant_config, + # quant_config=quant_config, ) self.down_proj = RowParallelLinear( intermediate_size, hidden_size, bias=False, - quant_config=quant_config, + # quant_config=quant_config, ) assert hidden_act == "silu" self.act_fn = SiluAndMul() @@ -150,13 +151,13 @@ def __init__( rope_theta=getattr(config, "rope_theta", 1000000), rope_scaling=getattr(config, "rope_scaling", None), kv_cache_dtype=cache_config, - quant_config=quant_config, + # quant_config=quant_config, ) self.mlp = Qwen3MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, - quant_config=quant_config, + # quant_config=quant_config, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( @@ -174,12 +175,13 @@ def forward( hidden_states = self.input_layernorm(hidden_states) else: hidden_states, residual = self.input_layernorm(hidden_states, residual) - hidden_states = self.self_attn(positions, hidden_states) + # hidden_states = self.self_attn(positions, hidden_states) hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) hidden_states = self.mlp(hidden_states) return hidden_states, residual +@support_torch_compile class Qwen3Model(nn.Module): def __init__(self, atom_config: Config) -> None: diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index 9b97a3f747..e15bfc5bcf 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -295,3 +295,142 @@ def zmq_socket_ctx( finally: ctx.destroy(linger=linger) +from packaging import version +from packaging.version import Version +import importlib +import torch +import copy +import dataclasses +from contextlib import contextmanager +from typing import Union, Any +from atom.config import get_capture_status, CompilationConfig, Config +from atom.config import CompilationLevel +import time + +context_manager = None +torch_compile_start_time: float = 0.0 + + +def is_torch_equal_or_newer(target: str) -> bool: + """Check if the installed torch version is >= the target version. + + Args: + target: a version string, like "2.6.0". + + Returns: + Whether the condition meets. + """ + try: + return _is_torch_equal_or_newer(str(torch.__version__), target) + except Exception: + # Fallback to PKG-INFO to load the package info, needed by the doc gen. + return Version(importlib.metadata.version('torch')) >= Version(target) + + +# Helper function used in testing. +def _is_torch_equal_or_newer(torch_version: str, target: str) -> bool: + torch_version = version.parse(torch_version) + return torch_version >= version.parse(target) + + +def weak_ref_tensor(tensor: Any) -> Any: + """ + Create a weak reference to a tensor. + The new tensor will share the same data as the original tensor, + but will not keep the original tensor alive. + """ + if isinstance(tensor, torch.Tensor): + return torch.ops._C.weak_ref_tensor(tensor) + else: + return tensor + + +def weak_ref_tensors( + tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]] +) -> Union[torch.Tensor, list[Any], tuple[Any], Any]: + """ + Convenience function to create weak references to tensors, + for single tensor, list of tensors or tuple of tensors. + """ + if isinstance(tensors, torch.Tensor): + return weak_ref_tensor(tensors) + if isinstance(tensors, list): + return [weak_ref_tensor(t) for t in tensors] + if isinstance(tensors, tuple): + return tuple(weak_ref_tensor(t) for t in tensors) + raise ValueError("Invalid type for tensors") + + +@dataclasses.dataclass +class CompilationCounter: + num_models_seen: int = 0 + num_graphs_seen: int = 0 + # including the splitting ops + num_piecewise_graphs_seen: int = 0 + # not including the splitting ops + num_piecewise_capturable_graphs_seen: int = 0 + num_backend_compilations: int = 0 + # Number of gpu_model_runner attempts to trigger CUDAGraphs capture + num_gpu_runner_capture_triggers: int = 0 + # Number of CUDAGraphs captured + num_cudagraph_captured: int = 0 + # InductorAdapter.compile calls + num_inductor_compiles: int = 0 + # EagerAdapter.compile calls + num_eager_compiles: int = 0 + # The number of time vLLM's compiler cache entry was updated + num_cache_entries_updated: int = 0 + # The number of standalone_compile compiled artifacts saved + num_compiled_artifacts_saved: int = 0 + # Number of times a model was loaded with CompilationLevel.DYNAMO_AS_IS + dynamo_as_is_count: int = 0 + + def clone(self) -> "CompilationCounter": + return copy.deepcopy(self) + + @contextmanager + def expect(self, **kwargs): + old = self.clone() + yield + for k, v in kwargs.items(): + assert getattr(self, k) - getattr(old, k) == v, ( + f"{k} not as expected, before it is {getattr(old, k)}" + f", after it is {getattr(self, k)}, " + f"expected diff is {v}") + + +compilation_counter = CompilationCounter() + + + +def resolve_obj_by_qualname(qualname: str) -> Any: + """ + Resolve an object by its fully-qualified class name. + """ + module_name, obj_name = qualname.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, obj_name) + + +def start_monitoring_torch_compile(vllm_config: Config): + global torch_compile_start_time + torch_compile_start_time = time.time() + + compilation_config: CompilationConfig = vllm_config.compilation_config + if compilation_config.level == CompilationLevel.PIECEWISE and \ + compilation_config.debug_dump_path: + import depyf + path = os.path.join(compilation_config.debug_dump_path,"rank_0") + # f"rank_{vllm_config.parallel_config.rank}") + global context_manager + context_manager = depyf.prepare_debug(path) + context_manager.__enter__() + + +def end_monitoring_torch_compile(vllm_config: Config): + compilation_config: CompilationConfig = vllm_config.compilation_config + if compilation_config.level == CompilationLevel.PIECEWISE: + global context_manager + if context_manager is not None: + context_manager.__exit__(None, None, None) + context_manager = None diff --git a/atom/utils/backends.py b/atom/utils/backends.py new file mode 100644 index 0000000000..8afa6f4b35 --- /dev/null +++ b/atom/utils/backends.py @@ -0,0 +1,661 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import ast +import dataclasses +import os +import pprint +import time +from collections.abc import Sequence +from contextlib import contextmanager +from typing import Any, Callable, Optional + + +import torch +import torch.fx as fx +from torch._dispatch.python import enable_python_dispatcher + +from .compiler_inferface import InductorAdaptor, CompilerInterface +from atom.utils import is_torch_equal_or_newer +from aiter import logger +from atom.config import CompilationConfig, Config, CUDAGraphMode +from atom.utils import compilation_counter, resolve_obj_by_qualname + + +def get_static_graph_wrapper_cls(cls) -> str: + return "atom.utils.cuda_graph.CUDAGraphWrapper" + + +# import vllm.envs as envs +# from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig +# from vllm.logger import init_logger +# from vllm.platforms import current_platform +# from atom.utils import is_torch_equal_or_newer, resolve_obj_by_qualname + +# from .compiler_interface import (CompilerInterface, EagerAdaptor, +# InductorAdaptor, InductorStandaloneAdaptor) +# from .counter import compilation_counter +# from .inductor_pass import InductorPass +# from .pass_manager import PostGradPassManager + +# logger = init_logger(__name__) + + + +def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface: + return InductorAdaptor() + # if compilation_config.use_inductor: + # if is_torch_equal_or_newer( + # "2.8.0.dev"): + # logger.debug("Using InductorStandaloneAdaptor") + # return InductorStandaloneAdaptor() + # else: + # logger.debug("Using InductorAdaptor") + # return InductorAdaptor() + # else: + # logger.debug("Using EagerAdaptor") + # return EagerAdaptor() + + +class CompilerManager: + """ + A manager to manage the compilation process, including + caching the compiled graph, loading the compiled graph, + and compiling the graph. + + The cache is a dict mapping + `(runtime_shape, graph_index, backend_name)` + to `any_data` returned from the compiler. + + When serializing the cache, we save it to a Python file + for readability. We don't use json here because json doesn't + support int as key. + """ + + def __init__(self, compilation_config: CompilationConfig): + self.cache: dict[tuple[Optional[int], int, str], Any] = dict() + self.is_cache_updated = False + self.compilation_config = compilation_config + self.compiler = make_compiler(compilation_config) + + def compute_hash(self, vllm_config: Config) -> str: + return self.compiler.compute_hash(vllm_config) + + def initialize_cache(self, + cache_dir: str, + disable_cache: bool = False, + prefix: str = ""): + """ + Initialize the cache directory for the compiler. + + The organization of the cache directory is as follows: + cache_dir=/path/to/hash_str/rank_i_j/prefix/ + inside cache_dir, there will be: + - vllm_compile_cache.py + - computation_graph.py + - transformed_code.py + + for multiple prefixes, they can share the same + base cache dir of /path/to/hash_str/rank_i_j/ , + to store some common compilation artifacts. + """ + + self.disable_cache = disable_cache + self.cache_dir = cache_dir + self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py") + + if not disable_cache and os.path.exists(self.cache_file_path): + # load the cache from the file + with open(self.cache_file_path) as f: + # we use ast.literal_eval to parse the data + # because it is a safe way to parse Python literals. + # do not use eval(), it is unsafe. + self.cache = ast.literal_eval(f.read()) + + self.compiler.initialize_cache(cache_dir=cache_dir, + disable_cache=disable_cache, + prefix=prefix) + + def save_to_file(self): + if self.disable_cache or not self.is_cache_updated: + return + printer = pprint.PrettyPrinter(indent=4) + data = printer.pformat(self.cache) + with open(self.cache_file_path, "w") as f: + f.write(data) + + def load(self, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None) -> Optional[Callable]: + if (runtime_shape, graph_index, self.compiler.name) not in self.cache: + return None + handle = self.cache[(runtime_shape, graph_index, self.compiler.name)] + compiled_graph = self.compiler.load(handle, graph, example_inputs, + graph_index, runtime_shape) + if runtime_shape is None: + logger.debug( + "Directly load the %s-th graph for dynamic shape from %s via " + "handle %s", graph_index, self.compiler.name, handle) + else: + logger.debug( + "Directly load the %s-th graph for shape %s from %s via " + "handle %s", graph_index, str(runtime_shape), + self.compiler.name, handle) + return compiled_graph + + def compile(self, + graph: fx.GraphModule, + example_inputs, + additional_inductor_config, + compilation_config: CompilationConfig, + graph_index: int = 0, + num_graphs: int = 1, + runtime_shape: Optional[int] = None) -> Any: + # print("runtime_shape", runtime_shape) + if graph_index == 0: + # before compiling the first graph, record the start time + global compilation_start_time + compilation_start_time = time.time() + + compilation_counter.num_backend_compilations += 1 + + compiled_graph = None + + # try to load from the cache + compiled_graph = self.load(graph, example_inputs, graph_index, + runtime_shape) + if compiled_graph is not None: + if graph_index == num_graphs - 1: + # after loading the last graph for this shape, record the time. + # there can be multiple graphs due to piecewise compilation. + now = time.time() + elapsed = now - compilation_start_time + if runtime_shape is None: + logger.info( + "Directly load the compiled graph(s) for dynamic shape " + "from the cache, took %.3f s", elapsed) + else: + logger.info( + "Directly load the compiled graph(s) for shape %s " + "from the cache, took %.3f s", str(runtime_shape), + elapsed) + return compiled_graph + + # no compiler cached the graph, or the cache is disabled, + # we need to compile it + if isinstance(self.compiler, InductorAdaptor): + # Let compile_fx generate a key for us + maybe_key = None + else: + maybe_key = \ + f"artifact_shape_{runtime_shape}_subgraph_{graph_index}" + compiled_graph, handle = self.compiler.compile( + graph, example_inputs, additional_inductor_config, runtime_shape, + maybe_key) + + assert compiled_graph is not None, "Failed to compile the graph" + + # store the artifact in the cache + # if not envs.VLLM_DISABLE_COMPILE_CACHE and handle is not None: + if True and handle is not None: + self.cache[(runtime_shape, graph_index, + self.compiler.name)] = handle + compilation_counter.num_cache_entries_updated += 1 + self.is_cache_updated = True + if graph_index == 0: + # adds some info logging for the first graph + if runtime_shape is None: + logger.info( + "Cache the graph for dynamic shape for later use") + else: + logger.info("Cache the graph of shape %s for later use", + str(runtime_shape)) + if runtime_shape is None: + logger.debug( + "Store the %s-th graph for dynamic shape from %s via " + "handle %s", graph_index, self.compiler.name, handle) + else: + logger.debug( + "Store the %s-th graph for shape %s from %s via handle %s", + graph_index, str(runtime_shape), self.compiler.name, + handle) + + # after compiling the last graph, record the end time + if graph_index == num_graphs - 1: + now = time.time() + elapsed = now - compilation_start_time + compilation_config.compilation_time += elapsed + if runtime_shape is None: + logger.info("Compiling a graph for dynamic shape takes %.2f s", + elapsed) + else: + logger.info("Compiling a graph for shape %s takes %.2f s", + runtime_shape, elapsed) + + return compiled_graph + + +@dataclasses.dataclass +class SplitItem: + submod_name: str + graph_id: int + is_splitting_graph: bool + graph: fx.GraphModule + + +def split_graph(graph: fx.GraphModule, + ops: list[str]) -> tuple[fx.GraphModule, list[SplitItem]]: + # split graph by ops + subgraph_id = 0 + node_to_subgraph_id = {} + split_op_graphs = [] + for node in graph.graph.nodes: + if node.op in ("output", "placeholder"): + continue + # print(str(node.target)) + if node.op == 'call_function' and str(node.target) in ops: + subgraph_id += 1 + node_to_subgraph_id[node] = subgraph_id + split_op_graphs.append(subgraph_id) + subgraph_id += 1 + else: + node_to_subgraph_id[node] = subgraph_id + + # `keep_original_order` is important! + # otherwise pytorch might reorder the nodes and + # the semantics of the graph will change when we + # have mutations in the graph + split_gm = torch.fx.passes.split_module.split_module( + graph, + None, + lambda node: node_to_subgraph_id[node], + keep_original_order=True) + + outputs = [] + + names = [name for (name, module) in split_gm.named_modules()] + + for name in names: + if "." in name or name == "": + # recursive child module or the root module + continue + + module = getattr(split_gm, name) + + graph_id = int(name.replace("submod_", "")) + outputs.append( + SplitItem(name, graph_id, (graph_id in split_op_graphs), module)) + + # sort by integer graph_id, rather than string name + outputs.sort(key=lambda x: x.graph_id) + + return split_gm, outputs + + +compilation_start_time = 0.0 + + +# class PiecewiseCompileInterpreter(torch.fx.Interpreter): +# """Code adapted from `torch.fx.passes.shape_prop.ShapeProp`. +# It runs the given graph with fake inputs, and compile some +# submodules specified by `compile_submod_names` with the given +# compilation configs. + +# NOTE: the order in `compile_submod_names` matters, because +# it will be used to determine the order of the compiled piecewise +# graphs. The first graph will handle logging, and the last graph +# has some special cudagraph output handling. +# """ + +# def __init__(self, module: torch.fx.GraphModule, +# compile_submod_names: list[str], vllm_config: Config, +# vllm_backend: "VllmBackend"): +# super().__init__(module) +# from torch._guards import detect_fake_mode +# self.fake_mode = detect_fake_mode() +# self.compile_submod_names = compile_submod_names +# self.compilation_config = vllm_config.compilation_config +# self.vllm_config = vllm_config +# self.vllm_backend = vllm_backend +# # When True, it annoyingly dumps the torch.fx.Graph on errors. +# self.extra_traceback = False + +# def run(self, *args): +# fake_args = [ +# self.fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t +# for t in args +# ] +# with self.fake_mode, enable_python_dispatcher(): +# return super().run(*fake_args) + +# def call_module(self, target: torch.fx.node.Target, +# args: tuple[torch.fx.node.Argument, +# ...], kwargs: dict[str, Any]) -> Any: +# assert isinstance(target, str) +# output = super().call_module(target, args, kwargs) + +# if target in self.compile_submod_names: +# index = self.compile_submod_names.index(target) +# submod = self.fetch_attr(target) +# sym_shape_indices = [ +# i for i, x in enumerate(args) if isinstance(x, torch.SymInt) +# ] +# global compilation_start_time +# compiled_graph_for_dynamic_shape = self.vllm_backend.\ +# compiler_manager.compile( +# submod, +# args, +# self.compilation_config.inductor_compile_config, +# self.compilation_config, +# graph_index=index, +# num_graphs=len(self.compile_submod_names), +# runtime_shape=None) +# # Lazy import here to avoid circular import +# from .cuda_graph import CUDAGraphOptions +# from .cuda_piecewise_backend import PiecewiseBackend + +# piecewise_backend = PiecewiseBackend( +# submod, self.vllm_config, index, +# len(self.compile_submod_names), sym_shape_indices, +# compiled_graph_for_dynamic_shape, self.vllm_backend) + +# if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: +# # resolve the static graph wrapper class (e.g. CUDAGraphWrapper +# # class) as platform dependent. +# static_graph_wrapper_class = resolve_obj_by_qualname( +# get_static_graph_wrapper_cls()) + +# # Always assign PIECEWISE runtime mode to the +# # CUDAGraphWrapper for piecewise_backend, to distinguish +# # it from the FULL cudagraph runtime mode, no matter it +# # is wrapped on a full or piecewise fx graph. +# self.module.__dict__[target] = static_graph_wrapper_class( +# runnable=piecewise_backend, +# vllm_config=self.vllm_config, +# runtime_mode=CUDAGraphMode.PIECEWISE, +# cudagraph_options=CUDAGraphOptions( +# debug_log_enable=piecewise_backend.is_first_graph, +# gc_disable=not piecewise_backend.is_first_graph, +# weak_ref_output=piecewise_backend.is_last_graph)) +# else: +# self.module.__dict__[target] = piecewise_backend + +# compilation_counter.num_piecewise_capturable_graphs_seen += 1 + +# return output + + +# the tag for the part of model being compiled, +# e.g. backbone/eagle_head +model_tag: str = "backbone" + + +@contextmanager +def set_model_tag(tag: str): + """Context manager to set the model tag.""" + global model_tag + assert tag != model_tag, \ + f"Model tag {tag} is the same as the current tag {model_tag}." + old_tag = model_tag + model_tag = tag + try: + yield + finally: + model_tag = old_tag + +VLLM_CACHE_ROOT = os.path.expanduser("~/.cache/atom") + +class VllmBackend: + """The compilation backend for `torch.compile` with vLLM. + It is used for compilation level of `CompilationLevel.PIECEWISE`, + where we customize the compilation. + + The major work of this backend is to split the graph into + piecewise graphs, and pass them to the piecewise backend. + + This backend also adds the PostGradPassManager to Inductor config, + which handles the post-grad passes. + """ + + vllm_config: Config + compilation_config: CompilationConfig + _called: bool = False + # the graph we compiled + graph: fx.GraphModule + # the stiching graph module for all the piecewise graphs + split_gm: fx.GraphModule + piecewise_graphs: list[SplitItem] + returned_callable: Callable + # Inductor passes to run on the graph pre-defunctionalization + post_grad_passes: Sequence[Callable] + sym_tensor_indices: list[int] + input_buffers: list[torch.Tensor] + compiler_manager: CompilerManager + + def __init__( + self, + vllm_config: Config, + prefix: str = "", + ): + + # if the model is initialized with a non-empty prefix, + # then usually it's enough to use that prefix, + # e.g. language_model, vision_model, etc. + # when multiple parts are initialized as independent + # models, we need to use the model_tag to distinguish + # them, e.g. backbone (default), eagle_head, etc. + self.prefix = prefix or model_tag + + # Passes to run on the graph post-grad. + # self.post_grad_pass_manager = PostGradPassManager() + + self.sym_tensor_indices = [] + self.input_buffers = [] + + self.vllm_config = vllm_config + self.compilation_config = vllm_config.compilation_config + + self.compiler_manager: CompilerManager = CompilerManager( + self.compilation_config) + + # `torch.compile` is JIT compiled, so we don't need to + # do anything here + + # def configure_post_pass(self): + # config = self.compilation_config + # self.post_grad_pass_manager.configure(self.vllm_config) + + # # Post-grad custom passes are run using the post_grad_custom_post_pass + # # hook. If a pass for that hook exists, add it to the pass manager. + # inductor_config = config.inductor_compile_config + # PASS_KEY = "post_grad_custom_post_pass" + # if PASS_KEY in inductor_config: + # # Config should automatically wrap all inductor passes + # if isinstance(inductor_config[PASS_KEY], PostGradPassManager): + # assert (inductor_config[PASS_KEY].uuid() == + # self.post_grad_pass_manager.uuid()) + # else: + # assert isinstance(inductor_config[PASS_KEY], InductorPass) + # self.post_grad_pass_manager.add(inductor_config[PASS_KEY]) + # inductor_config[PASS_KEY] = self.post_grad_pass_manager + + def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: + + vllm_config = self.vllm_config + if not self.compilation_config.cache_dir: + # no provided cache dir, generate one based on the known factors + # that affects the compilation. if none of the factors change, + # the cache dir will be the same so that we can reuse the compiled + # graph. + + factors = [] + # 0. factors come from the env, for example, The values of + # VLLM_PP_LAYER_PARTITION will affect the computation graph. + # env_hash = envs.compute_hash() + # factors.append(env_hash) + + # 1. factors come from the vllm_config (it mainly summarizes how the + # model is created) + config_hash = vllm_config.compute_hash() + factors.append(config_hash) + + # 2. factors come from the code files that are traced by Dynamo ( + # it mainly summarizes how the model is used in forward pass) + forward_code_files = list( + sorted(self.compilation_config.traced_files)) + self.compilation_config.traced_files.clear() + logger.debug( + "Traced files (to be considered for compilation cache):\n%s", + "\n".join(forward_code_files)) + hash_content = [] + for filepath in forward_code_files: + hash_content.append(filepath) + if filepath == "": + # This means the function was dynamically generated, with + # e.g. exec(). We can't actually check these. + continue + with open(filepath) as f: + hash_content.append(f.read()) + import hashlib + code_hash = hashlib.md5("\n".join(hash_content).encode(), + usedforsecurity=False).hexdigest() + factors.append(code_hash) + + # 3. compiler hash + compiler_hash = self.compiler_manager.compute_hash(vllm_config) + factors.append(compiler_hash) + + # combine all factors to generate the cache dir + hash_key = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] + + cache_dir = os.path.join( + VLLM_CACHE_ROOT, + "torch_compile_cache", + hash_key, + ) + self.compilation_config.cache_dir = cache_dir + + cache_dir = self.compilation_config.cache_dir + os.makedirs(cache_dir, exist_ok=True) + self.compilation_config.cache_dir = cache_dir + # rank = vllm_config.parallel_config.rank + rank = 0 + # dp_rank = vllm_config.parallel_config.data_parallel_rank + local_cache_dir = os.path.join(cache_dir, f"rank_{rank}", + self.prefix) + os.makedirs(local_cache_dir, exist_ok=True) + self.compilation_config.local_cache_dir = local_cache_dir + + # disable_cache = envs.VLLM_DISABLE_COMPILE_CACHE + disable_cache = False + + if disable_cache: + logger.info("vLLM's torch.compile cache is disabled.") + else: + logger.info("Using cache directory: %s for vLLM's torch.compile", + local_cache_dir) + + self.compiler_manager.initialize_cache(local_cache_dir, disable_cache, + self.prefix) + + # when dynamo calls the backend, it means the bytecode + # transform and analysis are done + compilation_counter.num_graphs_seen += 1 + from atom.utils import torch_compile_start_time + dynamo_time = time.time() - torch_compile_start_time + logger.info("Dynamo bytecode transform time: %.2f s", dynamo_time) + self.compilation_config.compilation_time += dynamo_time + + # we control the compilation process, each instance can only be + # called once + assert not self._called, "VllmBackend can only be called once" + + self.graph = graph + # self.configure_post_pass() + + self.split_gm, self.piecewise_graphs = split_graph( + graph, self.compilation_config.splitting_ops) + + from torch._dynamo.utils import lazy_format_graph_code + + # depyf will hook lazy_format_graph_code and dump the graph + # for debugging, no need to print the graph here + lazy_format_graph_code("before split", self.graph) + lazy_format_graph_code("after split", self.split_gm) + + compilation_counter.num_piecewise_graphs_seen += len( + self.piecewise_graphs) + submod_names_to_compile = [ + item.submod_name for item in self.piecewise_graphs + if not item.is_splitting_graph + ] + + # propagate the split graph to the piecewise backend, + # compile submodules with symbolic shapes + # PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile, + # self.vllm_config, + # self).run(*example_inputs) + + graph_path = os.path.join(local_cache_dir, "computation_graph.py") + if not os.path.exists(graph_path): + # code adapted from https://github.com/thuml/depyf/blob/dab831108a752d1facc00acdd6d4243891845c37/depyf/explain/patched_lazy_format_graph_code.py#L30 # noqa + # use `print_readable` because it can include submodules + src = "from __future__ import annotations\nimport torch\n" + \ + self.split_gm.print_readable(print_output=False) + src = src.replace("", "GraphModule") + with open(graph_path, "w") as f: + f.write(src) + + logger.debug("Computation graph saved to %s", graph_path) + + self._called = True + print('This is cudagraph_mode', self.compilation_config.cudagraph_mode) + + if self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE or \ + not self.compilation_config.cudagraph_copy_inputs: + return self.split_gm + + # if we need to copy input buffers for cudagraph + from torch._guards import detect_fake_mode + fake_mode = detect_fake_mode() + fake_args = [ + fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t + for t in example_inputs + ] + + # index of tensors that have symbolic shapes (batch size) + # for weights and static buffers, they will have concrete shapes. + # symbolic shape only happens for input tensors. + from torch.fx.experimental.symbolic_shapes import is_symbolic + self.sym_tensor_indices = [ + i for i, x in enumerate(fake_args) + if isinstance(x, torch._subclasses.fake_tensor.FakeTensor) and \ + any(is_symbolic(d) for d in x.size()) + ] + + # compiler managed cudagraph input buffers + # we assume the first run with symbolic shapes + # has the maximum size among all the tensors + self.input_buffers = [ + example_inputs[x].clone() for x in self.sym_tensor_indices + ] + + # this is the callable we return to Dynamo to run + def copy_and_call(*args): + list_args = list(args) + for i, index in enumerate(self.sym_tensor_indices): + runtime_tensor = list_args[index] + runtime_shape = runtime_tensor.shape[0] + static_tensor = self.input_buffers[i][:runtime_shape] + + # copy the tensor to the static buffer + static_tensor.copy_(runtime_tensor) + + # replace the tensor in the list_args to the static buffer + list_args[index] = static_tensor + return self.split_gm(*list_args) + + return copy_and_call diff --git a/atom/utils/compiler_inferface.py b/atom/utils/compiler_inferface.py new file mode 100644 index 0000000000..ca9e256def --- /dev/null +++ b/atom/utils/compiler_inferface.py @@ -0,0 +1,464 @@ +import contextlib +import copy +import hashlib +import os +from contextlib import ExitStack +from typing import Any, Callable, Optional +from unittest.mock import patch +import copy +from atom.utils import compilation_counter +import torch +import torch._inductor.compile_fx +import torch.fx as fx + +from atom.config import Config +from atom.utils import is_torch_equal_or_newer + + +class CompilerInterface: + """ + The interface for a compiler that can be used by vLLM. + """ + # The name of the compiler, e.g. inductor. + # This is a class-level attribute. + name: str + + def initialize_cache(self, + cache_dir: str, + disable_cache: bool = False, + prefix: str = ""): + """ + when the vLLM process uses `cache_dir` as the cache directory, + the compiler should initialize itself with the cache directory, + e.g. by re-directing its own cache directory to a sub-directory. + + prefix can be used in combination with cache_dir to figure out the base + cache directory, e.g. there're multiple parts of model being compiled, + but we want to share the same cache directory for all of them. + + e.g. + cache_dir = "/path/to/dir/backbone", prefix = "backbone" + cache_dir = "/path/to/dir/eagle_head", prefix = "eagle_head" + """ + pass + + def compute_hash(self, vllm_config: Config) -> str: + """ + Gather all the relevant information from the vLLM config, + to compute a hash so that we can cache the compiled model. + + See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash] + to check what information + is already considered by default. This function should only + consider the information that is specific to the compiler. + """ + return "" + + def compile( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + compiler_config: dict[str, Any], + runtime_shape: Optional[int] = None, + key: Optional[str] = None, + ) -> tuple[Optional[Callable], Optional[Any]]: + """ + Compile the graph with the given example inputs and compiler config, + with a runtime shape. If the `runtime_shape` is None, it means + the `example_inputs` have a dynamic shape. Otherwise, the + `runtime_shape` specifies the shape of the inputs. Right now we only + support one variable shape for all inputs, which is the batchsize + (number of tokens) during inference. + + Dynamo will make sure `graph(*example_inputs)` is valid. + + The function should return a compiled callable function, as well as + a handle that can be used to directly load the compiled function. + + The handle should be a plain Python object, preferably a string or a + file path for readability. + + If the compiler doesn't support caching, it should return None for the + handle. If the compiler fails to compile the graph, it should return + None for the compiled function as well. + + `key` is required for StandaloneInductorAdapter, it specifies where to + save the compiled artifact. The compiled artifact gets saved to + `cache_dir/key`. + """ + return None, None + + def load(self, + handle: Any, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None) -> Callable: + """ + Load the compiled function from the handle. + Raises an error if the handle is invalid. + + The handle is the second return value of the `compile` function. + """ + raise NotImplementedError("caching is not supported") + +class AlwaysHitShapeEnv: + """ + Why do we need this class: + + For normal `torch.compile` usage, every compilation will have + one Dynamo bytecode compilation and one Inductor compilation. + The Inductor compilation happens under the context of the + Dynamo bytecode compilation, and that context is used to + determine the dynamic shape information, etc. + + For our use case, we only run Dynamo bytecode compilation once, + and run Inductor compilation multiple times with different shapes + plus a general shape. The compilation for specific shapes happens + outside of the context of the Dynamo bytecode compilation. At that + time, we don't have shape environment to provide to Inductor, and + it will fail the Inductor code cache lookup. + + By providing a dummy shape environment that always hits, we can + make the Inductor code cache lookup always hit, and we can + compile the graph for different shapes as needed. + + The following dummy methods are obtained by trial-and-error + until it works. + """ + + def __init__(self) -> None: + self.guards: list[Any] = [] + + def evaluate_guards_expression(self, *args, **kwargs): + return True + + def get_pruned_guards(self, *args, **kwargs): + return [] + + def produce_guards_expression(self, *args, **kwargs): + return "" + + +def get_inductor_factors() -> list[Any]: + factors: list[Any] = [] + # summarize system state + from torch._inductor.codecache import CacheBase + system_factors = CacheBase.get_system() + factors.append(system_factors) + + # summarize pytorch state + from torch._inductor.codecache import torch_key + torch_factors = torch_key() + factors.append(torch_factors) + return factors + + +def set_inductor_config(config, runtime_shape): + if isinstance(runtime_shape, int): + # for a specific batchsize, tuning triton kernel parameters + # can be beneficial + config["max_autotune"] = True + config["coordinate_descent_tuning"] = True + + +class InductorAdaptor(CompilerInterface): + """ + The adaptor for the Inductor compiler, version 2.5, 2.6, 2.7. + """ + name = "inductor" + + def compute_hash(self, vllm_config: Config) -> str: + factors = get_inductor_factors() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] + return hash_str + + def initialize_cache(self, + cache_dir: str, + disable_cache: bool = False, + prefix: str = ""): + self.cache_dir = cache_dir + self.prefix = prefix + self.base_cache_dir = cache_dir[:-len(prefix)] if prefix else cache_dir + if disable_cache: + return + # redirect the cache directory to a sub-directory + # set flags so that Inductor and Triton store their cache + # in the cache_dir, then users only need to copy the cache_dir + # to another machine to reuse the cache. + inductor_cache = os.path.join(self.base_cache_dir, "inductor_cache") + os.makedirs(inductor_cache, exist_ok=True) + os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache + triton_cache = os.path.join(self.base_cache_dir, "triton_cache") + os.makedirs(triton_cache, exist_ok=True) + os.environ["TRITON_CACHE_DIR"] = triton_cache + + def compile( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + compiler_config: dict[str, Any], + runtime_shape: Optional[int] = None, + key: Optional[str] = None, + ) -> tuple[Optional[Callable], Optional[Any]]: + compilation_counter.num_inductor_compiles += 1 + from torch._inductor.compile_fx import compile_fx + current_config = {} + if compiler_config is not None: + current_config.update(compiler_config) + + # disable remote cache + current_config["fx_graph_cache"] = True + current_config["fx_graph_remote_cache"] = False + + set_inductor_config(current_config, runtime_shape) + + # inductor can inplace modify the graph, so we need to copy it + # see https://github.com/pytorch/pytorch/issues/138980 + graph = copy.deepcopy(graph) + + # it's the first time we compile this graph + # the assumption is that we don't have nested Inductor compilation. + # compiled_fx_graph_hash will only be called once, and we can hook + # it to get the hash of the compiled graph directly. + + hash_str, file_path = None, None + from torch._inductor.codecache import (FxGraphCache, + compiled_fx_graph_hash) + if torch.__version__.startswith("2.5"): + original_load = FxGraphCache.load + original_load_name = "torch._inductor.codecache.FxGraphCache.load" + + def hijack_load(*args, **kwargs): + inductor_compiled_graph = original_load(*args, **kwargs) + nonlocal file_path + compiled_fn = inductor_compiled_graph.current_callable + file_path = compiled_fn.__code__.co_filename # noqa + if not file_path.startswith(self.base_cache_dir): + # hooked in the align_inputs_from_check_idxs function + # in torch/_inductor/utils.py + for cell in compiled_fn.__closure__: + if not callable(cell.cell_contents): + continue + if cell.cell_contents.__code__.co_filename.startswith( + self.base_cache_dir): + # this is the real file path compiled from Inductor + file_path = cell.cell_contents.__code__.co_filename + break + return inductor_compiled_graph + + hijacked_compile_fx_inner = torch._inductor.compile_fx.compile_fx_inner # noqa + elif torch.__version__ >= "2.6": + # function renamed in 2.6 + original_load_name = None + + def hijacked_compile_fx_inner(*args, **kwargs): + output = torch._inductor.compile_fx.compile_fx_inner( + *args, **kwargs) + nonlocal hash_str + inductor_compiled_graph = output + if inductor_compiled_graph is not None: + nonlocal file_path + compiled_fn = inductor_compiled_graph.current_callable + file_path = compiled_fn.__code__.co_filename # noqa + if not file_path.startswith(self.base_cache_dir): + # hooked in the align_inputs_from_check_idxs function + # in torch/_inductor/utils.py + for cell in compiled_fn.__closure__: + if not callable(cell.cell_contents): + continue + code = cell.cell_contents.__code__ + if code.co_filename.startswith( + self.base_cache_dir): + # this is the real file path + # compiled from Inductor + file_path = code.co_filename + break + hash_str = inductor_compiled_graph._fx_graph_cache_key + return output + + def hijack_compiled_fx_graph_hash(*args, **kwargs): + out = compiled_fx_graph_hash(*args, **kwargs) + nonlocal hash_str + hash_str = out[0] + return out + + def _check_can_cache(*args, **kwargs): + # no error means it can be cached. + # Inductor refuses to cache the graph outside of Dynamo + # tracing context, and also disables caching for graphs + # with high-order ops. + # For vLLM, in either case, we want to cache the graph. + # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa + return + + def _get_shape_env() -> AlwaysHitShapeEnv: + return AlwaysHitShapeEnv() + + with ExitStack() as stack: + # hijack to get the compiled graph itself + if original_load_name is not None: + stack.enter_context(patch(original_load_name, hijack_load)) + + # for hijacking the hash of the compiled graph + stack.enter_context( + patch("torch._inductor.codecache.compiled_fx_graph_hash", + hijack_compiled_fx_graph_hash)) + + # for providing a dummy shape environment + stack.enter_context( + patch("torch._inductor.codecache.FxGraphCache._get_shape_env", + _get_shape_env)) + + from torch._functorch._aot_autograd.autograd_cache import ( + AOTAutogradCache) + + # torch 2.8+ on main uses _get_shape_env in AOTAutogradCache + if hasattr(AOTAutogradCache, "_get_shape_env"): + stack.enter_context( + patch( + "torch._functorch._aot_autograd.autograd_cache.AOTAutogradCache._get_shape_env", + _get_shape_env)) + + # for forcing the graph to be cached + stack.enter_context( + patch( + "torch._inductor.codecache.FxGraphCache._check_can_cache", + _check_can_cache)) + + # Dynamo metrics context, see method for more details. + stack.enter_context(self.metrics_context()) + + # Disable remote caching. When these are on, on remote cache-hit, + # the monkey-patched functions never actually get called. + # vLLM today assumes and requires the monkey-patched functions to + # get hit. + # TODO(zou3519): we're going to replace this all with + # standalone_compile sometime. + if is_torch_equal_or_newer("2.6"): + stack.enter_context( + torch._inductor.config.patch(fx_graph_remote_cache=False)) + # InductorAdaptor (unfortunately) requires AOTAutogradCache + # to be turned off to run. It will fail to acquire the hash_str + # and error if not. + # StandaloneInductorAdaptor (PyTorch 2.8+) fixes this problem. + stack.enter_context( + torch._functorch.config.patch(enable_autograd_cache=False)) + stack.enter_context( + torch._functorch.config.patch( + enable_remote_autograd_cache=False)) + + # with pass_context(runtime_shape): + # ======= TODO lirong ======= + compiled_graph = compile_fx( + graph, + example_inputs, + inner_compile=hijacked_compile_fx_inner, + config_patches=current_config) + + # We treat VLLM_DISABLE_COMPILE_CACHE as the overall switch for torch + # compilation cache. So turn off the checks if we disable the + # compilation cache. + # if not envs.VLLM_DISABLE_COMPILE_CACHE: + if hash_str is None: + raise RuntimeError( + "vLLM failed to compile the model. The most " + "likely reason for this is that a previous compilation " + "failed, leading to a corrupted compilation artifact. " + "We recommend trying to " + "remove ~/.cache/vllm/torch_compile_cache and try again " + "to see the real issue. ") + assert file_path is not None, ( + "failed to get the file path of the compiled graph") + return compiled_graph, (hash_str, file_path) + + def load(self, + handle: Any, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None) -> Callable: + assert isinstance(handle, tuple) + assert isinstance(handle[0], str) + assert isinstance(handle[1], str) + hash_str = handle[0] + + from torch._functorch._aot_autograd.autograd_cache import ( + AOTAutogradCache) + from torch._inductor.codecache import FxGraphCache + with ExitStack() as exit_stack: + exit_stack.enter_context( + patch("torch._inductor.codecache.FxGraphCache._get_shape_env", + lambda *args, **kwargs: AlwaysHitShapeEnv())) + # torch 2.8+ on main uses _get_shape_env in AOTAutogradCache + if hasattr(AOTAutogradCache, "_get_shape_env"): + exit_stack.enter_context( + patch( + "torch._functorch._aot_autograd.autograd_cache.AOTAutogradCache._get_shape_env", + lambda *args, **kwargs: AlwaysHitShapeEnv())) + + # Dynamo metrics context, see method for more details. + exit_stack.enter_context(self.metrics_context()) + + if torch.__version__.startswith("2.5"): + inductor_compiled_graph = FxGraphCache._lookup_graph( + hash_str, example_inputs, True, False) + assert inductor_compiled_graph is not None, ( + "Inductor cache lookup failed. Please remove" + f"the cache directory and try again." # noqa + ) + elif torch.__version__ >= "2.6": + from torch._inductor.output_code import ( + CompiledFxGraphConstantsWithGm) + constants = CompiledFxGraphConstantsWithGm(graph) + inductor_compiled_graph, _ = FxGraphCache._lookup_graph( + hash_str, example_inputs, True, None, constants) + assert inductor_compiled_graph is not None, ( + "Inductor cache lookup failed. Please remove" + f"the cache directory and try again." # noqa + ) + + # Inductor calling convention (function signature): + # f(list) -> tuple + # Dynamo calling convention (function signature): + # f(*args) -> Any + + # need to know if the graph returns a tuple + from torch._inductor.compile_fx import graph_returns_tuple + returns_tuple = graph_returns_tuple(graph) + + # this is the callable we return to Dynamo to run + def compiled_graph(*args): + # convert args to list + list_args = list(args) + graph_output = inductor_compiled_graph(list_args) + # unpack the tuple if needed + if returns_tuple: + return graph_output + else: + return graph_output[0] + + return compiled_graph + + def metrics_context(self) -> contextlib.AbstractContextManager: + """ + This method returns the Dynamo metrics context (if it exists, + otherwise a null context). It is used by various compile components. + Present in torch>=2.6, it's used inside FxGraphCache in + torch==2.6 (but not after). It might also be used in various other + torch.compile internal functions. + + Because it is re-entrant, we always set it (even if entering via Dynamo + and the context was already entered). We might want to revisit if it + should be set at a different level of compilation. + + This is likely a bug in PyTorch: public APIs should not rely on + manually setting up internal contexts. But we also rely on non-public + APIs which might not provide these guarantees. + """ + if is_torch_equal_or_newer("2.6"): + import torch._dynamo.utils + return torch._dynamo.utils.get_metrics_context() + else: + return contextlib.nullcontext() diff --git a/atom/utils/cuda_graph.py b/atom/utils/cuda_graph.py new file mode 100644 index 0000000000..4fb2cf8477 --- /dev/null +++ b/atom/utils/cuda_graph.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import dataclasses +from contextlib import ExitStack +from typing import Any, Callable, Optional,NamedTuple +from unittest.mock import patch + +import torch + +from atom.utils import compilation_counter +from atom.utils import weak_ref_tensors +from aiter import logger +from atom.config import Config, CUDAGraphMode + +# from vllm.compilation.monitor import validate_cudagraph_capturing_enabled +# from vllm.config import CUDAGraphMode, VllmConfig +# from vllm.forward_context import BatchDescriptor, get_forward_context +# from vllm.logger import init_logger +# from vllm.platforms import current_platform + +# logger = init_logger(__name__) + + +class BatchDescriptor(NamedTuple): + """ + Batch descriptor for cudagraph dispatching. We should keep the num of + items as minimal as possible to properly and uniquely describe the padded + batch for cudagraph. + """ + num_tokens: int + uniform_decode: bool = False + """ + False can also be used for an uniform decode batch to dispatch to the + cudagraph supporting non-uniform batches. + """ + + @property + def non_uniform(self) -> "BatchDescriptor": + """ + Return a non-uniform version of current batch descriptor. + """ + return BatchDescriptor(self.num_tokens, uniform_decode=False) + + +@dataclasses.dataclass +class CUDAGraphEntry: + batch_descriptor: BatchDescriptor + cudagraph: Optional[torch.cuda.CUDAGraph] = None + output: Optional[Any] = None + + # for cudagraph debugging, track the input addresses + # during capture, and check if they are the same during replay + input_addresses: Optional[list[int]] = None + + +@dataclasses.dataclass +class CUDAGraphOptions: + debug_log_enable: bool = True + gc_disable: bool = False + weak_ref_output: bool = True + + +class CUDAGraphWrapper: + """Wraps a runnable to add CUDA graph capturing and replaying ability. And + provide attribute access to the underlying `runnable` via `__getattr__`. + + The workflow of this wrapper in the cudagraph dispatching is as follows: + 1. At initialization, a runtime mode is assigned to the wrapper (FULL or + PIECEWISE). + 2. At runtime, the wrapper receives a runtime_mode and a + batch_descriptor(key) from the forward context and blindly trust them + for cudagraph dispatching. + 3. If runtime_mode is NONE or runtime_mode does not match the mode of the + wrapper, just call the runnable directly. + 4. Otherwise, i.e., the runtime_mode matches the mode of the wrapper, + the wrapper will perform cudagraph capture(if key does not exist, create + a new entry and cache it) or replay (if key exists in the cache). + + Note: CUDAGraphWrapper does not store persistent buffers or copy any + runtime inputs into that buffers for replay. We assume implementing them + is done outside of the wrapper. That is because we do not make any + assumption on the dynamic shape (batch size) of the runtime inputs, as a + trade-off for staying orthogonal to compilation logic. Nevertheless, + tracing and checking the input addresses to be consistent during replay is + guaranteed when VLLM_LOGGING_LEVEL == "DEBUG". + """ + + def __init__(self, + runnable: Callable, + vllm_config: Config, + runtime_mode: CUDAGraphMode, + cudagraph_options: Optional[CUDAGraphOptions] = None): + self.runnable = runnable + self.vllm_config = vllm_config + self.runtime_mode = runtime_mode + self.compilation_config = vllm_config.compilation_config + + self.first_run_finished = False + self.is_debugging_mode = True + + # assert runtime_mode is not NONE(no cudagraph), otherwise, we don't + # need to initialize a CUDAGraphWrapper. + assert self.runtime_mode != CUDAGraphMode.NONE + # TODO: in the future, if we want to use multiple + # streams, it might not be safe to share a global pool. + # only investigate this when we use multiple streams + + # self.graph_pool = current_platform.get_global_graph_pool() + + if cudagraph_options is None: + cudagraph_options = CUDAGraphOptions() + self.cudagraph_options = cudagraph_options + # the entries for different batch descriptors that we need to capture + # cudagraphs for. + self.concrete_cudagraph_entries: dict[BatchDescriptor, CUDAGraphEntry]\ + = {} + + def __getattr__(self, key: str): + # allow accessing the attributes of the runnable. + if hasattr(self.runnable, key): + return getattr(self.runnable, key) + raise AttributeError(f"Attribute {key} not exists in the runnable of " + f"cudagraph wrapper: {self.runnable}") + + def unwrap(self) -> Callable: + # in case we need to access the original runnable. + return self.runnable + + def __call__(self, *args, **kwargs): + forward_context = get_forward_context() + batch_descriptor = forward_context.batch_descriptor + cudagraph_runtime_mode = forward_context.cudagraph_runtime_mode + + if cudagraph_runtime_mode == CUDAGraphMode.NONE or \ + cudagraph_runtime_mode != self.runtime_mode: + # CUDAGraphMode.NONE could mean the profile run, a warmup run, or + # running without cudagraphs. + # We do not trigger capture/replay if the runtime mode is not + # matches. This enables properly dispatching to the correct + # CUDAGraphWrapper when nesting multiple instances with different + # runtime modes. + return self.runnable(*args, **kwargs) + + if batch_descriptor not in self.concrete_cudagraph_entries: + # create a new entry for this batch descriptor + self.concrete_cudagraph_entries[batch_descriptor] = \ + CUDAGraphEntry(batch_descriptor=batch_descriptor) + + entry = self.concrete_cudagraph_entries[batch_descriptor] + + if entry.cudagraph is None: + if self.cudagraph_options.debug_log_enable: + # Since we capture cudagraph for many different shapes and + # capturing is fast, we don't need to log it for every + # shape. E.g. we only log it for the first subgraph in + # piecewise mode. + logger.debug("Capturing a cudagraph on (%s,%s)", + self.runtime_mode.name, entry.batch_descriptor) + # validate that cudagraph capturing is legal at this point. + # ================= TODO lirong ======================== + # validate_cudagraph_capturing_enabled() + + input_addresses = [ + x.data_ptr() for x in args if isinstance(x, torch.Tensor) + ] + entry.input_addresses = input_addresses + cudagraph = torch.cuda.CUDAGraph() + + with ExitStack() as stack: + if self.cudagraph_options.gc_disable: + # during every model forward for piecewise cudagraph + # mode, we will capture many pieces of cudagraphs + # (roughly one per layer). running gc again and again + # across layers will make the cudagraph capture very slow. + # therefore, we only run gc for the first graph, + # and disable gc for the rest of the graphs. + stack.enter_context(patch("gc.collect", lambda: None)) + stack.enter_context( + patch("torch.cuda.empty_cache", lambda: None)) + + # mind-exploding: carefully manage the reference and memory. + with torch.cuda.graph(cudagraph, pool=self.graph_pool): + # `output` is managed by pytorch's cudagraph pool + output = self.runnable(*args, **kwargs) + if self.cudagraph_options.weak_ref_output: + # by converting it to weak ref, + # the original `output` will immediately be released + # to save memory. It is only safe to do this for + # the last graph in piecewise cuadgraph mode, because + # the output of the last graph will not be used by + # any other cuda graph. + output = weak_ref_tensors(output) + + # here we always use weak ref for the output + # to save memory + entry.output = weak_ref_tensors(output) + entry.cudagraph = cudagraph + + compilation_counter.num_cudagraph_captured += 1 + + # important: we need to return the output, rather than + # the weak ref of the output, so that pytorch can correctly + # manage the memory during cuda graph capture + return output + + if self.is_debugging_mode: + # check if the input addresses are the same + new_input_addresses = [ + x.data_ptr() for x in args if isinstance(x, torch.Tensor) + ] + assert new_input_addresses == entry.input_addresses, ( + f"Input addresses for cudagraphs are different " + f"during replay. Expected {entry.input_addresses}, " + f"got {new_input_addresses}") + + entry.cudagraph.replay() + return entry.output diff --git a/atom/utils/decorators.py b/atom/utils/decorators.py new file mode 100644 index 0000000000..7069dddbde --- /dev/null +++ b/atom/utils/decorators.py @@ -0,0 +1,312 @@ +from typing import Callable, Optional, TypeVar, Union, overload +import inspect +import os,sys +from types import CodeType +from abc import abstractmethod +from contextlib import contextmanager +from unittest.mock import patch + +from torch._dynamo.symbolic_convert import InliningInstructionTranslator +import torch +import torch.nn as nn + +from atom.config import get_capture_status, CompilationConfig, Config +from atom.utils import start_monitoring_torch_compile +from atom.config import CompilationLevel + +_T = TypeVar("_T", bound=type[nn.Module]) + +# _current_vllm_config: Optional[Config] = None + +# def get_current_vllm_config() -> Config: +# if _current_vllm_config is None: +# # in ci, usually when we test custom ops/modules directly, +# # we don't set the vllm config. In that case, we set a default +# # config. +# from atom.config import Config +# return Config() +# return _current_vllm_config + + +def init_backend(config: Config): + from .backends import VllmBackend + return VllmBackend(config) + + +class TorchCompileWrapperWithCustomDispatcher: + """ + A wrapper class for torch.compile, with a custom dispatch logic. + Subclasses should: + 1. Implement the forward method + 2. Implement the dispatch logic in the __call__ method + It can use `self.compiled_codes` to access the compiled bytecode, + and `with self.dispatch_to_code(index):` to dispatch to + the compiled code. + 3. Implement the `__init__` method to determine how to call + `torch.compile` over the forward method. + """ + + def __init__(self, + vllm_config: Config, + compiled_callable: Optional[Callable] = None, + compilation_level: int = 0): + self.vllm_config = vllm_config + + if compiled_callable is None: + # default compilation settings + # compiling the forward method + options = None + # print("Torch compile 000000000000000000000000000") + backend = init_backend(vllm_config) + compiled_callable = torch.compile( + self.forward, + # fullgraph=True, + backend=backend, + # dynamic=True, + options=options) + + self.compiled_callable = compiled_callable + self.original_code_object = self.__class__.forward.__code__ + self.compiled_codes: list[CodeType] = [] + torch._dynamo.convert_frame.register_bytecode_hook(self.bytecode_hook) + + # read the env var to determine whether to use the custom dispatcher + # subclasses can use this to switch between the custom dispatcher + # and the default Dynamo guard mechanism. + self.use_custom_dispatcher: bool = \ + compilation_level >= CompilationLevel.DYNAMO_ONCE + + def __call__(self, *args, **kwargs): + """Implement the dispatch logic here, beyond the torch.compile level. + NOTE: this function can have additional arguments beyond the forward + method, for directly dispatching to the compiled code. + """ + # print('compiled_callable=====================') + return self.compiled_callable(*args, **kwargs) + + @abstractmethod + def forward(self, *args, **kwargs): + ... + + def bytecode_hook(self, old_code: CodeType, new_code: CodeType): + """Hook to save the compiled bytecode for direct execution.""" + if old_code is not self.original_code_object: + return + # code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25 + frame = sys._getframe() + while frame and frame.f_back: + frame = frame.f_back + code_name = frame.f_code.co_name + file_name = frame.f_code.co_filename.split(os.path.sep)[-1] + if code_name == "_compile" and file_name == "convert_frame.py": + break + frame = frame.f_locals["frame"] + assert frame.f_code == old_code + + if frame.f_locals["self"] is not self: + return + # print("new_code", new_code) + self.compiled_codes.append(new_code) + debug_dump_dir = self.vllm_config.compilation_config.debug_dump_path + if isinstance(debug_dump_dir, str) and debug_dump_dir != "": + # rank = self.vllm_config.parallel_config.rank + rank = 0 + decompiled_file = os.path.join(debug_dump_dir, f"rank_{rank}", + "transformed_code.py") + if not os.path.exists(decompiled_file): + try: + # usually the decompilation will succeed for most models, + # as we guarantee a full-graph compilation in Dynamo. + # but there's no 100% guarantee, since decompliation is + # not a reversible process. + import depyf + src = depyf.decompile(new_code) + + with open(decompiled_file, "w") as f: + f.write(src) + except Exception: + pass + + if self.vllm_config.compilation_config.use_cudagraph and \ + "update" in new_code.co_names: + import depyf + src = depyf.decompile(new_code) + msg = "Assigning / modifying buffers of nn.Module during forward pass is not allowed when using cudagraph inside the compiler because it will cause silent errors. Please use eager mode or fix the code. The following code contains clues about which buffer is being modified (please search for the usage of the function `update`):\n" + src # noqa + raise RuntimeError(msg) + + @contextmanager + def dispatch_to_code(self, index: int): + """Context manager to dispatch to the compiled code. + Why does this work? Because Dynamo guarantees that the compiled + bytecode has exactly the same arguments, cell variables, and free + variables as the original code. Therefore we can directly switch + the code object in the function and call it. + + See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7 for more details. + """ # noqa + self.__class__.forward.__code__ = self.compiled_codes[index] + yield + self.__class__.forward.__code__ = self.original_code_object + + + +def support_torch_compile( + cls: Optional[_T] = None, + *, + dynamic_arg_dims: Optional[dict[str, Union[int, list[int]]]] = None, +) -> Union[Callable[[_T], _T], _T]: + def cls_decorator_helper(cls: _T) -> _T: + # helper to pass `dynamic_arg_dims`` to `_support_torch_compile`` + # to avoid too much indentation for `_support_torch_compile`` + if not hasattr(cls, 'forward'): + raise TypeError("decorated class should have a forward method.") + sig = inspect.signature(cls.forward) + inferred_dynamic_arg_dims = dynamic_arg_dims + if inferred_dynamic_arg_dims is None: + inferred_dynamic_arg_dims = {} + for k, v in sig.parameters.items(): + if v.annotation in [ + torch.Tensor, Optional[torch.Tensor], + ]: + inferred_dynamic_arg_dims[k] = 0 + + + if len(inferred_dynamic_arg_dims) == 0: + raise ValueError( + "No dynamic dimensions found in the forward method of " + f"{cls}. Please provide dynamic_arg_dims explicitly.") + + for k in inferred_dynamic_arg_dims: + if k not in sig.parameters: + raise ValueError( + f"Argument {k} not found in the forward method of {cls}") + return _support_torch_compile(cls, inferred_dynamic_arg_dims) + + if cls is not None: + # use `support_torch_compile` as a decorator without arguments + assert isinstance(cls, type) + return cls_decorator_helper(cls) + + return cls_decorator_helper + +def _support_torch_compile( + cls: _T, + dynamic_arg_dims: dict[str, Union[int, list[int]]], +) -> _T: + """ + A decorator to add support for compiling the forward method of a class. + """ + if TorchCompileWrapperWithCustomDispatcher in cls.__bases__: + # support decorating multiple times + return cls + # print("_support_torch_compile") + # take care of method resolution order + # make sure super().__init__ is called on the base class + # other than TorchCompileWrapperWithCustomDispatcher + cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, ) + + old_init = cls.__init__ + + + def __init__(self, atom_config: Config, **kwargs): + old_init(self, atom_config=atom_config, **kwargs) + self.atom_config = atom_config + # for CompilationLevel.DYNAMO_AS_IS , the upper level model runner + # will handle the compilation, so we don't need to do anything here. + self.do_not_compile = \ + atom_config.compilation_config.level in [ + CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS + ] + # or (get_capture_status()) + # print("self.do_not_compile",self.do_not_compile) + if self.do_not_compile: + return + + TorchCompileWrapperWithCustomDispatcher.__init__( + self, vllm_config=atom_config, compilation_level=atom_config.compilation_config.level) + + cls.__init__ = __init__ + + def __call__(self, *args, **kwargs): + # torch.compiler.is_compiling() means we are inside the compilation + # e.g. TPU has the compilation logic in model runner, so we don't + # need to compile the model inside. + if self.do_not_compile or torch.compiler.is_compiling(): + return self.forward(*args, **kwargs) + + # print("self.compiled_codes", self.compiled_codes) + # the first compilation needs to have dynamic shapes marked + if len(self.compiled_codes) < 1: + sig = inspect.signature(self.__class__.forward) + bound_args = sig.bind(self, *args, **kwargs) + bound_args.apply_defaults() + for k, dims in dynamic_arg_dims.items(): + arg = bound_args.arguments.get(k) + if arg is not None: + dims = [dims] if isinstance(dims, int) else dims + if isinstance(arg, torch.Tensor): + # In case dims is specified with negative indexing + dims = [ + arg.ndim + dim if dim < 0 else dim for dim in dims + ] + # print(arg.shape) + # print(f"torch._dynamo.mark_dynamic({arg, dims})") + torch._dynamo.mark_dynamic(arg, dims) + else: + raise ValueError( + "Unsupported dynamic dimensions" + f" {dims} for argument {k} with type {type(arg)}.") + # here, it is the starting point of the `torch.compile` process + start_monitoring_torch_compile(self.atom_config) + # print("Start compiling function %s", + # self.original_code_object) + + # if we don't use custom dispatcher, we can directly call the + # compiled function and let torch.compile handle the dispatching, + # with the overhead of guard evaluation and recompilation. + if len(self.compiled_codes) < 1 or not self.use_custom_dispatcher: + # it seems Dynamo reuse the compilation across instances, + # while we need to make sure the compiled code is not reused. + # we need to control all the compilation of the model. + torch._dynamo.eval_frame.remove_from_cache( + self.original_code_object) + + # collect all relevant files traced by Dynamo, + # so that the compilation cache can trigger re-compilation + # properly when any of these files change. + + # 1. the file containing the top-level forward function + self.vllm_config.compilation_config.traced_files.add( + self.original_code_object.co_filename) + + # 2. every time Dynamo sees a function call, it will inline + # the function by calling InliningInstructionTranslator.inline_call + # we hijack this function to know all the functions called + # during Dynamo tracing, and their corresponding files + inline_call = InliningInstructionTranslator.inline_call + + def patched_inline_call(parent, func, args, kwargs): + code = func.get_code() + self.vllm_config.compilation_config.traced_files.add( + code.co_filename) + return inline_call(parent, func, args, kwargs) + + with patch.object(InliningInstructionTranslator, 'inline_call', + patched_inline_call): + # print("self.compiled_callable to call torch compile") + output = self.compiled_callable(*args, **kwargs) + from aiter import logger + logger.info("torch.compiler.is_compiling()") + logger.info(torch.compiler.is_compiling()) + return output + + # usually, capturing the model once is enough, and then we can + # dispatch to the compiled code directly, without going through + # the Dynamo guard mechanism. + with self.dispatch_to_code(0): + model_output = self.forward(*args, **kwargs) + return model_output + + cls.__call__ = __call__ + return cls + diff --git a/bench.py b/bench.py index b7e60dc8a8..381008b60c 100644 --- a/bench.py +++ b/bench.py @@ -2,6 +2,7 @@ import time from random import randint, seed from atom import LLMEngine, SamplingParams +from atom.config import CompilationConfig # from vllm import LLM, SamplingParams @@ -13,7 +14,10 @@ def main(): max_ouput_len = 1024 path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") - llm = LLMEngine(path, enforce_eager=False, max_model_len=4096) + llm = LLMEngine(path, enforce_eager=False, max_model_len=4096, compilation_config=CompilationConfig( + level = 3, + # cudagraph_capture_sizes=[1, 2, 4, 8] + list(range(16, 512 + 1, 16)) + )) prompt_token_ids = [ [randint(0, 10000) for _ in range(randint(100, max_input_len))] From b633ad36719d0afb26e006b26bbc3c50ed6382c6 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 26 Sep 2025 14:06:13 +0800 Subject: [PATCH 040/119] support torch compile and fix acc by impl context --- atom/config.py | 52 +++++-- atom/model_engine/model_runner.py | 8 + atom/model_ops/attention.py | 181 +++++++++++++--------- atom/models/qwen3.py | 12 +- atom/utils/__init__.py | 2 +- atom/utils/backends.py | 219 ++++++++++++++------------- atom/utils/compiler_inferface.py | 90 +++++++++++ atom/utils/cuda_graph.py | 2 +- atom/utils/cuda_piecewise_backend.py | 116 ++++++++++++++ atom/utils/custom_register.py | 34 +++++ atom/utils/decorators.py | 4 +- atom/utils/forward_context.py | 56 +++++++ 12 files changed, 575 insertions(+), 201 deletions(-) create mode 100644 atom/utils/cuda_piecewise_backend.py create mode 100644 atom/utils/custom_register.py create mode 100644 atom/utils/forward_context.py diff --git a/atom/config.py b/atom/config.py index 0710381ba0..1b2ef4b369 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,9 +1,6 @@ import os -import re -from dataclasses import dataclass, field -from typing import Optional - import torch +import re from dataclasses import dataclass from transformers import AutoConfig from typing import Optional, Any, ClassVar @@ -76,7 +73,10 @@ class CompilationConfig: - 1: dynamo as is. - 2: dynamo once. - 3: piecewise compilation.""" - use_cudagraph: bool = field(default_factory=lambda: 0) + # use_cudagraph: bool = field(default_factory=lambda: 0) + + use_cudagraph: bool = True + local_cache_dir: str = field(default=None, init=False) # type: ignore # cudagraph_capture_sizes: Optional[list[int]] = [1,2,4,8] cudagraph_capture_sizes: Optional[list[int]] = None @@ -96,6 +96,8 @@ class CompilationConfig: cache_dir: str = "" + use_inductor: bool = True + # CudaGraph compilation cudagraph_mode: Optional[CUDAGraphMode] = None """ @@ -154,7 +156,9 @@ class CompilationConfig: """ _attention_ops: ClassVar[list[str]] = [ - "aiter.wrapper_fmha_v3_varlen_fwd", + "aiter.unified_attention_with_output" + + # "aiter.wrapper_fmha_v3_varlen_fwd", # "vllm.unified_attention", # "vllm.unified_attention_with_output", # "vllm.mamba_mixer2", @@ -163,6 +167,33 @@ class CompilationConfig: # "vllm.linear_attention", ] + inductor_compile_config: dict = field(default_factory=dict) + """Additional configurations for inductor. + - None: use default configurations.""" + + compile_sizes: Optional[list[Union[int, str]]] = None + """Sizes to compile for inductor. In addition + to integers, it also supports "cudagraph_capture_sizes" to + specify the sizes for cudagraph capture.""" + + def init_with_cudagraph_sizes(self) -> None: + """To complete the initialization of config, + we need to know the cudagraph sizes.""" + computed_compile_sizes = [] + if self.compile_sizes is not None: + # de-duplicate the sizes provided by the config + self.compile_sizes = list(set(self.compile_sizes)) + for x in self.compile_sizes: + if isinstance(x, str): + assert x == "cudagraph_capture_sizes", \ + "Unrecognized size type in compile_sizes, " \ + f"expect 'cudagraph_capture_sizes', got {x}" + computed_compile_sizes.extend(self.cudagraph_capture_sizes) + else: + assert isinstance(x, int) + computed_compile_sizes.append(x) + self.compile_sizes = computed_compile_sizes # type: ignore + def __post_init__(self): if self.level not in {0, 1, 2, 3}: raise ValueError("level must in 0-3") @@ -187,9 +218,6 @@ def compute_hash(self) -> str: factors.append(self.local_cache_dir) factors.append(self.cudagraph_capture_sizes) factors.append(self.cuda_graph_sizes) - factors.append(self.debug_dump_path) - factors.append(self.traced_files) - factors.append(self.cache_dir) return hashlib.sha256(str(factors).encode()).hexdigest() @@ -323,8 +351,6 @@ class Config: torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) compilation_config: CompilationConfig = field(default_factory=CompilationConfig) quant_config: QuantizationConfig = field(default_factory=lambda: QuantizationConfig()) - asyncio_mode: bool = False - master_addr: str = "127.0.0.1" def __post_init__(self): # assert os.path.isdir(self.model) @@ -341,6 +367,10 @@ def __post_init__(self): ), f"torch_profiler_dir {self.torch_profiler_dir} is not a valid directory" if self.compilation_config.level == CompilationLevel.PIECEWISE: self.compilation_config.set_splitting_ops_for_v1() + self._set_cudagraph_sizes() + self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE + self.compilation_config.init_with_cudagraph_sizes() + def compute_hash(self) -> str: """ diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 264b57a339..02e1ebddcb 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -24,6 +24,7 @@ from atom.utils.context import get_context, reset_context, set_context logger = logging.getLogger("atom") +from atom.utils.forward_context import AttentionMetadata, set_forward_context suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -256,6 +257,13 @@ def allocate_kv_cache(self): if config.kv_cache_dtype == "fp8": module.k_scale = self.kv_scale[0, layer_id] module.v_scale = self.kv_scale[1, layer_id] + attention_metadata = AttentionMetadata( + k_cache=module.k_cache, + v_cache=module.v_cache, + k_scale=module.k_scale, + v_scale=module.v_scale + ) + set_forward_context(module.layer_num, attention_metadata) layer_id += 1 diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 005121f127..8f798fbd54 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -8,6 +8,108 @@ from atom.utils.context import get_context # from flash_attn import flash_attn_with_kvcache +from dataclasses import dataclass + +from atom.utils.context import get_context +from atom.utils.custom_register import direct_register_custom_op +from atom.utils.forward_context import ForwardContext, get_forward_context, AttentionMetadata, set_forward_context + +# Dynamo will not try to inspect any of the internal operations for prefill or decode +# @torch.library.custom_op("aiter::unified_attention_with_output", mutates_args=["q", "k", "v", "k_cache", "v_cache", "k_scale", "v_scale"]) +def unified_attention_with_output( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, + scale: float, kv_cache_dtype: str, layer_num: int +)-> torch.Tensor: + context = get_context() + if context.slot_mapping.numel(): + # not dummy run + forward_context: ForwardContext = get_forward_context() + attn_metadata_ = forward_context.no_compile_layers[layer_num] + k_cache = attn_metadata_.k_cache + v_cache = attn_metadata_.v_cache + k_scale = attn_metadata_.k_scale + v_scale = attn_metadata_.v_scale + else: + # dummy run before allocate kv_cache, thus we create manually + k_cache = v_cache = torch.tensor([]) + k_scale = v_scale = None + + if k_cache.numel() and v_cache.numel(): + if kv_cache_dtype == "fp8": + aiter.reshape_and_cache_with_pertoken_quant( + k, + v, + k_cache, + v_cache, + k_scale, + v_scale, + context.slot_mapping, + asm_layout=True, + ) + else: + aiter.reshape_and_cache( + k, + v, + k_cache, + v_cache, + context.slot_mapping, + kv_cache_dtype="auto", + k_scale=None, + v_scale=None, + asm_layout=True, + ) + + + if context.is_prefill: + # if context.block_tables is not None: # prefix cache + # k, v = k_cache, v_cache + o = aiter.flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=context.cu_seqlens_q, + cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q=context.min_seqlen_q, + dropout_p=context.dropout_p, + softmax_scale=scale, + causal=True, + ) + else: # decode + o = aiter.pa_fwd_asm( + q, + k_cache, + v_cache, + context.block_tables, + context.context_lens, + context.block_tables.stride(0), + K_QScale=k_scale, + V_QScale=v_scale, + out_=None, + high_precision=0, + ) + + return o + +# @unified_attention_with_output.register_fake +def _( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, + scale: float, kv_cache_dtype: str, layer_num: int +)-> torch.Tensor: + output_shape = q.shape + output = torch.zeros(output_shape, + dtype=q.dtype, + device=q.device) + return output + + +direct_register_custom_op( + op_name="unified_attention_with_output", + op_func=unified_attention_with_output, + mutates_args=[], + fake_impl=_, +) class Attention(nn.Module): @@ -19,6 +121,7 @@ def __init__( scale, num_kv_heads, kv_cache_dtype="bf16", + layer_num=0, **kwargs, ): super().__init__() @@ -30,86 +133,16 @@ def __init__( self.kv_cache_dtype = kv_cache_dtype self.max_model_len = 0 self.k_scale = self.v_scale = None + self.layer_num = layer_num def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): o: torch.Tensor q = q.view(-1, self.num_heads, self.head_dim) k = k.view(-1, self.num_kv_heads, self.head_dim) v = v.view(-1, self.num_kv_heads, self.head_dim) - context = get_context() - k_cache, v_cache = self.k_cache, self.v_cache - if k_cache.numel() and v_cache.numel() and context.slot_mapping is not None: - if self.kv_cache_dtype == "fp8": - aiter.reshape_and_cache_with_pertoken_quant( - k, - v, - k_cache, - v_cache, - self.k_scale, - self.v_scale, - context.slot_mapping, - asm_layout=True, - ) - else: - aiter.reshape_and_cache( - k, - v, - k_cache, - v_cache, - context.slot_mapping, - kv_cache_dtype="auto", - k_scale=None, - v_scale=None, - asm_layout=True, - ) - - if context.is_prefill: - # if context.block_tables is not None: # prefix cache - # k, v = k_cache, v_cache - o = aiter.flash_attn_varlen_func( - q, - k, - v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q=context.min_seqlen_q, - dropout_p=context.dropout_p, - softmax_scale=self.scale, - causal=True, - ) - else: # decode - o = aiter.pa_fwd_asm( - q, - k_cache, - v_cache, - context.block_tables, - context.context_lens, - context.block_tables.stride(0), - K_QScale=self.k_scale, - V_QScale=self.v_scale, - out_=None, - high_precision=0, - ) + - # o = PagedAttention.forward_decode( - # q, - # k_cache, - # v_cache, - # context.block_tables, - # context.context_lens, - # max_seq_len=self.max_model_len, - # kv_cache_dtype="auto", - # num_kv_heads=self.num_kv_heads, - # scale=self.scale, - # alibi_slopes=None, - # k_scale=self.k_scale, - # v_scale=self.v_scale, - # ) - - # o = flash_attn_with_kvcache(q.unsqueeze(1), k_cache, v_cache, - # cache_seqlens=context.context_lens, block_table=context.block_tables, - # softmax_scale=self.scale, causal=True) + o = torch.ops.aiter.unified_attention_with_output(q, k, v, + self.scale, self.kv_cache_dtype, self.layer_num) o = o.view(-1, self.num_heads * self.head_dim) return o diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 2515c6e5a7..7ee16b56b6 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -36,10 +36,12 @@ def __init__( rope_theta: float = 10000, rope_scaling: tuple | None = None, kv_cache_dtype: str = "fp16", + layer_num: int = 0, quant_config: Optional[QuantizationConfig] = None, ) -> None: super().__init__() tp_size = get_tp_group().world_size + self.layer_num = layer_num self.total_num_heads = num_heads assert self.total_num_heads % tp_size == 0 self.num_heads = self.total_num_heads // tp_size @@ -78,6 +80,7 @@ def __init__( self.scaling, self.num_kv_heads, kv_cache_dtype=kv_cache_dtype, + layer_num=layer_num, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) @@ -138,8 +141,10 @@ def __init__( config: Qwen3Config, cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, + layer_num: int = 0, ) -> None: super().__init__() + self.layer_num = layer_num self.self_attn = Qwen3Attention( hidden_size=config.hidden_size, num_heads=config.num_attention_heads, @@ -151,6 +156,7 @@ def __init__( rope_theta=getattr(config, "rope_theta", 1000000), rope_scaling=getattr(config, "rope_scaling", None), kv_cache_dtype=cache_config, + layer_num=layer_num # quant_config=quant_config, ) self.mlp = Qwen3MLP( @@ -175,7 +181,7 @@ def forward( hidden_states = self.input_layernorm(hidden_states) else: hidden_states, residual = self.input_layernorm(hidden_states, residual) - # hidden_states = self.self_attn(positions, hidden_states) + hidden_states = self.self_attn(positions, hidden_states) hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) hidden_states = self.mlp(hidden_states) return hidden_states, residual @@ -195,9 +201,9 @@ def __init__(self, atom_config: Config) -> None: self.layers = nn.ModuleList( [ Qwen3DecoderLayer( - config, cache_config=cache_config, quant_config=quant_config + config, cache_config=cache_config, quant_config=quant_config, layer_num = layer_num ) - for _ in range(config.num_hidden_layers) + for layer_num in range(config.num_hidden_layers) ] ) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index e15bfc5bcf..256c02c2b7 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -303,7 +303,7 @@ def zmq_socket_ctx( import dataclasses from contextlib import contextmanager from typing import Union, Any -from atom.config import get_capture_status, CompilationConfig, Config +from atom.config import CompilationConfig, Config from atom.config import CompilationLevel import time diff --git a/atom/utils/backends.py b/atom/utils/backends.py index 8afa6f4b35..1d5349f63c 100644 --- a/atom/utils/backends.py +++ b/atom/utils/backends.py @@ -15,7 +15,7 @@ import torch.fx as fx from torch._dispatch.python import enable_python_dispatcher -from .compiler_inferface import InductorAdaptor, CompilerInterface +from .compiler_inferface import InductorAdaptor, CompilerInterface, InductorStandaloneAdaptor from atom.utils import is_torch_equal_or_newer from aiter import logger from atom.config import CompilationConfig, Config, CUDAGraphMode @@ -43,17 +43,17 @@ def get_static_graph_wrapper_cls(cls) -> str: def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface: - return InductorAdaptor() - # if compilation_config.use_inductor: - # if is_torch_equal_or_newer( - # "2.8.0.dev"): - # logger.debug("Using InductorStandaloneAdaptor") - # return InductorStandaloneAdaptor() - # else: - # logger.debug("Using InductorAdaptor") - # return InductorAdaptor() + # return InductorAdaptor() + if compilation_config.use_inductor: + if is_torch_equal_or_newer( + "2.8.0.dev"): + print("Using InductorStandaloneAdaptor") + return InductorStandaloneAdaptor() + else: + print("Using InductorAdaptor") + return InductorAdaptor() # else: - # logger.debug("Using EagerAdaptor") + # logger.info("Using EagerAdaptor") # return EagerAdaptor() @@ -135,11 +135,11 @@ def load(self, compiled_graph = self.compiler.load(handle, graph, example_inputs, graph_index, runtime_shape) if runtime_shape is None: - logger.debug( + logger.info( "Directly load the %s-th graph for dynamic shape from %s via " "handle %s", graph_index, self.compiler.name, handle) else: - logger.debug( + logger.info( "Directly load the %s-th graph for shape %s from %s via " "handle %s", graph_index, str(runtime_shape), self.compiler.name, handle) @@ -213,11 +213,11 @@ def compile(self, logger.info("Cache the graph of shape %s for later use", str(runtime_shape)) if runtime_shape is None: - logger.debug( + logger.info( "Store the %s-th graph for dynamic shape from %s via " "handle %s", graph_index, self.compiler.name, handle) else: - logger.debug( + logger.info( "Store the %s-th graph for shape %s from %s via handle %s", graph_index, str(runtime_shape), self.compiler.name, handle) @@ -297,94 +297,94 @@ def split_graph(graph: fx.GraphModule, compilation_start_time = 0.0 -# class PiecewiseCompileInterpreter(torch.fx.Interpreter): -# """Code adapted from `torch.fx.passes.shape_prop.ShapeProp`. -# It runs the given graph with fake inputs, and compile some -# submodules specified by `compile_submod_names` with the given -# compilation configs. - -# NOTE: the order in `compile_submod_names` matters, because -# it will be used to determine the order of the compiled piecewise -# graphs. The first graph will handle logging, and the last graph -# has some special cudagraph output handling. -# """ - -# def __init__(self, module: torch.fx.GraphModule, -# compile_submod_names: list[str], vllm_config: Config, -# vllm_backend: "VllmBackend"): -# super().__init__(module) -# from torch._guards import detect_fake_mode -# self.fake_mode = detect_fake_mode() -# self.compile_submod_names = compile_submod_names -# self.compilation_config = vllm_config.compilation_config -# self.vllm_config = vllm_config -# self.vllm_backend = vllm_backend -# # When True, it annoyingly dumps the torch.fx.Graph on errors. -# self.extra_traceback = False - -# def run(self, *args): -# fake_args = [ -# self.fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t -# for t in args -# ] -# with self.fake_mode, enable_python_dispatcher(): -# return super().run(*fake_args) - -# def call_module(self, target: torch.fx.node.Target, -# args: tuple[torch.fx.node.Argument, -# ...], kwargs: dict[str, Any]) -> Any: -# assert isinstance(target, str) -# output = super().call_module(target, args, kwargs) - -# if target in self.compile_submod_names: -# index = self.compile_submod_names.index(target) -# submod = self.fetch_attr(target) -# sym_shape_indices = [ -# i for i, x in enumerate(args) if isinstance(x, torch.SymInt) -# ] -# global compilation_start_time -# compiled_graph_for_dynamic_shape = self.vllm_backend.\ -# compiler_manager.compile( -# submod, -# args, -# self.compilation_config.inductor_compile_config, -# self.compilation_config, -# graph_index=index, -# num_graphs=len(self.compile_submod_names), -# runtime_shape=None) -# # Lazy import here to avoid circular import -# from .cuda_graph import CUDAGraphOptions -# from .cuda_piecewise_backend import PiecewiseBackend - -# piecewise_backend = PiecewiseBackend( -# submod, self.vllm_config, index, -# len(self.compile_submod_names), sym_shape_indices, -# compiled_graph_for_dynamic_shape, self.vllm_backend) - -# if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: -# # resolve the static graph wrapper class (e.g. CUDAGraphWrapper -# # class) as platform dependent. -# static_graph_wrapper_class = resolve_obj_by_qualname( -# get_static_graph_wrapper_cls()) - -# # Always assign PIECEWISE runtime mode to the -# # CUDAGraphWrapper for piecewise_backend, to distinguish -# # it from the FULL cudagraph runtime mode, no matter it -# # is wrapped on a full or piecewise fx graph. -# self.module.__dict__[target] = static_graph_wrapper_class( -# runnable=piecewise_backend, -# vllm_config=self.vllm_config, -# runtime_mode=CUDAGraphMode.PIECEWISE, -# cudagraph_options=CUDAGraphOptions( -# debug_log_enable=piecewise_backend.is_first_graph, -# gc_disable=not piecewise_backend.is_first_graph, -# weak_ref_output=piecewise_backend.is_last_graph)) -# else: -# self.module.__dict__[target] = piecewise_backend - -# compilation_counter.num_piecewise_capturable_graphs_seen += 1 - -# return output +class PiecewiseCompileInterpreter(torch.fx.Interpreter): + """Code adapted from `torch.fx.passes.shape_prop.ShapeProp`. + It runs the given graph with fake inputs, and compile some + submodules specified by `compile_submod_names` with the given + compilation configs. + + NOTE: the order in `compile_submod_names` matters, because + it will be used to determine the order of the compiled piecewise + graphs. The first graph will handle logging, and the last graph + has some special cudagraph output handling. + """ + + def __init__(self, module: torch.fx.GraphModule, + compile_submod_names: list[str], vllm_config: Config, + vllm_backend: "VllmBackend"): + super().__init__(module) + from torch._guards import detect_fake_mode + self.fake_mode = detect_fake_mode() + self.compile_submod_names = compile_submod_names + self.compilation_config = vllm_config.compilation_config + self.vllm_config = vllm_config + self.vllm_backend = vllm_backend + # When True, it annoyingly dumps the torch.fx.Graph on errors. + self.extra_traceback = False + + def run(self, *args): + fake_args = [ + self.fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t + for t in args + ] + with self.fake_mode, enable_python_dispatcher(): + return super().run(*fake_args) + + def call_module(self, target: torch.fx.node.Target, + args: tuple[torch.fx.node.Argument, + ...], kwargs: dict[str, Any]) -> Any: + assert isinstance(target, str) + output = super().call_module(target, args, kwargs) + + if target in self.compile_submod_names: + index = self.compile_submod_names.index(target) + submod = self.fetch_attr(target) + sym_shape_indices = [ + i for i, x in enumerate(args) if isinstance(x, torch.SymInt) + ] + global compilation_start_time + compiled_graph_for_dynamic_shape = self.vllm_backend.\ + compiler_manager.compile( + submod, + args, + self.compilation_config.inductor_compile_config, + self.compilation_config, + graph_index=index, + num_graphs=len(self.compile_submod_names), + runtime_shape=None) + # Lazy import here to avoid circular import + from .cuda_graph import CUDAGraphOptions + from .cuda_piecewise_backend import PiecewiseBackend + + piecewise_backend = PiecewiseBackend( + submod, self.vllm_config, index, + len(self.compile_submod_names), sym_shape_indices, + compiled_graph_for_dynamic_shape, self.vllm_backend) + + # if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: + # # resolve the static graph wrapper class (e.g. CUDAGraphWrapper + # # class) as platform dependent. + # static_graph_wrapper_class = resolve_obj_by_qualname( + # get_static_graph_wrapper_cls()) + + # # Always assign PIECEWISE runtime mode to the + # # CUDAGraphWrapper for piecewise_backend, to distinguish + # # it from the FULL cudagraph runtime mode, no matter it + # # is wrapped on a full or piecewise fx graph. + # self.module.__dict__[target] = static_graph_wrapper_class( + # runnable=piecewise_backend, + # vllm_config=self.vllm_config, + # runtime_mode=CUDAGraphMode.PIECEWISE, + # cudagraph_options=CUDAGraphOptions( + # debug_log_enable=piecewise_backend.is_first_graph, + # gc_disable=not piecewise_backend.is_first_graph, + # weak_ref_output=piecewise_backend.is_last_graph)) + # else: + self.module.__dict__[target] = piecewise_backend + + compilation_counter.num_piecewise_capturable_graphs_seen += 1 + + return output # the tag for the part of model being compiled, @@ -506,7 +506,7 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: forward_code_files = list( sorted(self.compilation_config.traced_files)) self.compilation_config.traced_files.clear() - logger.debug( + logger.info( "Traced files (to be considered for compilation cache):\n%s", "\n".join(forward_code_files)) hash_content = [] @@ -535,6 +535,8 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: VLLM_CACHE_ROOT, "torch_compile_cache", hash_key, + # "996ff366ca", + # "e6292aa343", ) self.compilation_config.cache_dir = cache_dir @@ -592,12 +594,13 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: item.submod_name for item in self.piecewise_graphs if not item.is_splitting_graph ] + print("submod_names_to_compile", submod_names_to_compile) # propagate the split graph to the piecewise backend, # compile submodules with symbolic shapes - # PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile, - # self.vllm_config, - # self).run(*example_inputs) + PiecewiseCompileInterpreter(self.split_gm, submod_names_to_compile, + self.vllm_config, + self).run(*example_inputs) graph_path = os.path.join(local_cache_dir, "computation_graph.py") if not os.path.exists(graph_path): @@ -609,7 +612,7 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: with open(graph_path, "w") as f: f.write(src) - logger.debug("Computation graph saved to %s", graph_path) + logger.info("Computation graph saved to %s", graph_path) self._called = True print('This is cudagraph_mode', self.compilation_config.cudagraph_mode) diff --git a/atom/utils/compiler_inferface.py b/atom/utils/compiler_inferface.py index ca9e256def..388af5bec6 100644 --- a/atom/utils/compiler_inferface.py +++ b/atom/utils/compiler_inferface.py @@ -462,3 +462,93 @@ def metrics_context(self) -> contextlib.AbstractContextManager: return torch._dynamo.utils.get_metrics_context() else: return contextlib.nullcontext() + + + +class InductorStandaloneAdaptor(CompilerInterface): + """ + The adaptor for the Inductor compiler. + Requires PyTorch 2.8+. + This is not on by default yet, but we plan to turn it on by default for + PyTorch 2.8. + + Use VLLM_USE_STANDALONE_COMPILE to toggle this on or off. + """ + name = "inductor_standalone" + + def compute_hash(self, vllm_config: Config) -> str: + factors = get_inductor_factors() + hash_str = hashlib.md5(str(factors).encode(), + usedforsecurity=False).hexdigest()[:10] + return hash_str + + def initialize_cache(self, + cache_dir: str, + disable_cache: bool = False, + prefix: str = ""): + self.cache_dir = cache_dir + + def compile( + self, + graph: fx.GraphModule, + example_inputs: list[Any], + compiler_config: dict[str, Any], + runtime_shape: Optional[int] = None, + key: Optional[str] = None, + ) -> tuple[Optional[Callable], Optional[Any]]: + compilation_counter.num_inductor_compiles += 1 + current_config = {} + if compiler_config is not None: + current_config.update(compiler_config) + set_inductor_config(current_config, runtime_shape) + + if isinstance(runtime_shape, int): + dynamic_shapes = "from_example_inputs" + else: + dynamic_shapes = "from_tracing_context" + + from torch._inductor import standalone_compile + + # TODO Lirong + # with pass_context(runtime_shape): + compiled_graph = standalone_compile( + graph, + example_inputs, + dynamic_shapes=dynamic_shapes, + options={"config_patches": current_config}) + + # Save the compiled artifact to disk in the specified path + assert key is not None + path = os.path.join(self.cache_dir, key) + if True: + # if not envs.VLLM_DISABLE_COMPILE_CACHE: + compiled_graph.save(path=path, format="unpacked") + compilation_counter.num_compiled_artifacts_saved += 1 + return compiled_graph, (key, path) + + def load(self, + handle: Any, + graph: fx.GraphModule, + example_inputs: list[Any], + graph_index: int, + runtime_shape: Optional[int] = None) -> Callable: + assert isinstance(handle, tuple) + assert isinstance(handle[0], str) + assert isinstance(handle[1], str) + path = handle[1] + inductor_compiled_graph = torch._inductor.CompiledArtifact.load( + path=path, format="unpacked") + from torch._inductor.compile_fx import graph_returns_tuple + returns_tuple = graph_returns_tuple(graph) + + def compiled_graph_wrapper(*args): + graph_output = inductor_compiled_graph(*args) + # unpack the tuple if needed + # TODO(rzou): the implication is that we're not + # reading the python bytecode correctly in vLLM? + if returns_tuple: + return graph_output + else: + return graph_output[0] + + return compiled_graph_wrapper diff --git a/atom/utils/cuda_graph.py b/atom/utils/cuda_graph.py index 4fb2cf8477..bb13aea7d2 100644 --- a/atom/utils/cuda_graph.py +++ b/atom/utils/cuda_graph.py @@ -155,7 +155,7 @@ def __call__(self, *args, **kwargs): # capturing is fast, we don't need to log it for every # shape. E.g. we only log it for the first subgraph in # piecewise mode. - logger.debug("Capturing a cudagraph on (%s,%s)", + logger.info("Capturing a cudagraph on (%s,%s)", self.runtime_mode.name, entry.batch_descriptor) # validate that cudagraph capturing is legal at this point. # ================= TODO lirong ======================== diff --git a/atom/utils/cuda_piecewise_backend.py b/atom/utils/cuda_piecewise_backend.py new file mode 100644 index 0000000000..0887443ea6 --- /dev/null +++ b/atom/utils/cuda_piecewise_backend.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import dataclasses +from typing import Any, Callable + +import torch.fx as fx + +# import vllm.envs as envs +from atom.utils.backends import VllmBackend +from atom.utils import end_monitoring_torch_compile +from atom.config import Config +from aiter import logger + + + +@dataclasses.dataclass +class ConcreteSizeEntry: + runtime_shape: int + compiled: bool = False + runnable: Callable = None # type: ignore + + +class PiecewiseBackend: + + def __init__(self, graph: fx.GraphModule, vllm_config: Config, + piecewise_compile_index: int, total_piecewise_compiles: int, + sym_shape_indices: list[int], + compiled_graph_for_general_shape: Callable, + vllm_backend: VllmBackend): + """ + The backend for piecewise compilation. + It mainly handles the compilation of static shapes and + dispatching based on runtime shape. + + We will compile `self.graph` once for the general shape, + and then compile for different shapes specified in + `compilation_config.compile_sizes`. + """ + self.graph = graph + self.vllm_config = vllm_config + self.compilation_config = vllm_config.compilation_config + self.piecewise_compile_index = piecewise_compile_index + self.total_piecewise_compiles = total_piecewise_compiles + self.vllm_backend = vllm_backend + + self.is_first_graph = piecewise_compile_index == 0 + self.is_last_graph = ( + piecewise_compile_index == total_piecewise_compiles - 1) + + self.is_full_graph = total_piecewise_compiles == 1 + + self.compile_sizes: set[int] = set( + self.compilation_config.compile_sizes) + + self.first_run_finished = False + + self.compiled_graph_for_general_shape = compiled_graph_for_general_shape # noqa + + self.sym_shape_indices = sym_shape_indices + + # self.is_debugging_mode = envs.VLLM_LOGGING_LEVEL == "DEBUG" + + # the entries for different shapes that we need to compile + self.concrete_size_entries: dict[int, ConcreteSizeEntry] = {} + + # to_be_compiled_sizes tracks the remaining sizes to compile, + # and updates during the compilation process, so we need to copy it + self.to_be_compiled_sizes: set[int] = self.compile_sizes.copy() + + # We only keep compilation management inside this class directly. + for shape in self.compile_sizes: + self.concrete_size_entries[shape] = ConcreteSizeEntry( + runtime_shape=shape, + runnable=self.compiled_graph_for_general_shape, + ) + + def check_for_ending_compilation(self): + if self.is_last_graph and not self.to_be_compiled_sizes: + # no specific sizes to compile + # save the hash of the inductor graph for the next run + self.vllm_backend.compiler_manager.save_to_file() + end_monitoring_torch_compile(self.vllm_config) + + def __call__(self, *args) -> Any: + if not self.first_run_finished: + self.first_run_finished = True + self.check_for_ending_compilation() + return self.compiled_graph_for_general_shape(*args) + + runtime_shape = args[self.sym_shape_indices[0]] + + if runtime_shape not in self.concrete_size_entries: + # we don't need to do anything for this shape + return self.compiled_graph_for_general_shape(*args) + + entry = self.concrete_size_entries[runtime_shape] + + if not entry.compiled: + entry.compiled = True + self.to_be_compiled_sizes.remove(runtime_shape) + # args are real arguments + entry.runnable = self.vllm_backend.compiler_manager.compile( + self.graph, + args, + self.compilation_config.inductor_compile_config, + self.compilation_config, + graph_index=self.piecewise_compile_index, + num_graphs=self.total_piecewise_compiles, + runtime_shape=runtime_shape) + + # finished compilations for all required shapes + if self.is_last_graph and not self.to_be_compiled_sizes: + self.check_for_ending_compilation() + + return entry.runnable(*args) diff --git a/atom/utils/custom_register.py b/atom/utils/custom_register.py new file mode 100644 index 0000000000..1b5bde177f --- /dev/null +++ b/atom/utils/custom_register.py @@ -0,0 +1,34 @@ +import torch +from torch.library import Library +from typing import Callable, Optional +from aiter import logger +aiter_lib = Library("aiter", "FRAGMENT") + + +# from flash_attn import flash_attn_with_kvcache + +def direct_register_custom_op( + op_name: str, + op_func: Callable, + mutates_args: list[str], + fake_impl: Optional[Callable] = None, + target_lib: Optional[Library] = None, + dispatch_key: str = "CUDA", + tags: tuple[torch.Tag, ...] = (), +): + + import torch.library + if hasattr(torch.library, "infer_schema"): + schema_str = torch.library.infer_schema(op_func, + mutates_args=mutates_args) + else: + # for pytorch 2.4 + import torch._custom_op.impl + schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args) + my_lib = target_lib or aiter_lib + my_lib.define(op_name + schema_str, tags=tags) + my_lib.impl(op_name, op_func, dispatch_key=dispatch_key) + if fake_impl is not None: + my_lib._register_fake(op_name, fake_impl) + + diff --git a/atom/utils/decorators.py b/atom/utils/decorators.py index 7069dddbde..7b1dde261e 100644 --- a/atom/utils/decorators.py +++ b/atom/utils/decorators.py @@ -10,7 +10,7 @@ import torch import torch.nn as nn -from atom.config import get_capture_status, CompilationConfig, Config +from atom.config import CompilationConfig, Config from atom.utils import start_monitoring_torch_compile from atom.config import CompilationLevel @@ -56,7 +56,6 @@ def __init__(self, # default compilation settings # compiling the forward method options = None - # print("Torch compile 000000000000000000000000000") backend = init_backend(vllm_config) compiled_callable = torch.compile( self.forward, @@ -217,7 +216,6 @@ def __init__(self, atom_config: Config, **kwargs): atom_config.compilation_config.level in [ CompilationLevel.NO_COMPILATION, CompilationLevel.DYNAMO_AS_IS ] - # or (get_capture_status()) # print("self.do_not_compile",self.do_not_compile) if self.do_not_compile: return diff --git a/atom/utils/forward_context.py b/atom/utils/forward_context.py new file mode 100644 index 0000000000..580cd121dc --- /dev/null +++ b/atom/utils/forward_context.py @@ -0,0 +1,56 @@ +from collections import defaultdict +from contextlib import contextmanager +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union +import torch + + +from aiter.jit.utils.torch_guard import torch_compile_guard + +@dataclass +class AttentionMetadata: + k_cache: torch.Tensor + v_cache: torch.Tensor + k_scale: torch.Tensor + v_scale: torch.Tensor + + +# @torch_compile_guard() +def set_attn_meta(k_cache: torch.Tensor, v_cache: torch.Tensor, + k_scale: torch.Tensor, v_scale: torch.Tensor) -> None: + attn_metadata = AttentionMetadata() + attn_metadata.k_cache = k_cache + attn_metadata.v_cache = v_cache + attn_metadata.k_scale = k_scale + attn_metadata.v_scale = v_scale + + +@dataclass +class ForwardContext: + # copy from vllm_config.compilation_config.static_forward_context + no_compile_layers: dict[int, Any] = field(default_factory=dict) + """ + Type AttentionMetadata for v0, + Type Dict[str, AttentionMetadata] for v1, map from layer_name of each + attention layer to its attention metadata + set dynamically for each forward pass + """ + def __post_init__(self): + if not hasattr(self, 'no_compile_layers') or self.no_compile_layers is None: + self.no_compile_layers = {} + + +_forward_context: Optional[ForwardContext] = ForwardContext() + +def get_forward_context() -> ForwardContext: + """Get the current forward context.""" + assert _forward_context is not None, ( + "Forward context is not set. " + "Please use `set_forward_context` to set the forward context.") + return _forward_context + + +def set_forward_context(layer_id: int, attention_metadata: AttentionMetadata) -> None: + + global _forward_context + _forward_context.no_compile_layers[layer_id] = attention_metadata From e18fb81d45a3395020dc51408b68e7248580b739 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 26 Sep 2025 14:13:29 +0800 Subject: [PATCH 041/119] fix some bug --- atom/config.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/atom/config.py b/atom/config.py index 1b2ef4b369..1f15cc0cec 100644 --- a/atom/config.py +++ b/atom/config.py @@ -3,7 +3,7 @@ import re from dataclasses import dataclass from transformers import AutoConfig -from typing import Optional, Any, ClassVar +from typing import Optional, Any, ClassVar, Union from dataclasses import field from aiter import QuantType from aiter.utility.dtypes import d_dtypes @@ -351,6 +351,22 @@ class Config: torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) compilation_config: CompilationConfig = field(default_factory=CompilationConfig) quant_config: QuantizationConfig = field(default_factory=lambda: QuantizationConfig()) + asyncio_mode: bool = False + master_addr: str = "127.0.0.1" + graph_bs: Optional[list[int]] = None + + + def _set_cudagraph_sizes(self): + if self.compilation_config.cudagraph_capture_sizes: + self.graph_bs = self.compilation_config.cudagraph_capture_sizes + else: + cuda_graph_sizes = self.compilation_config.cuda_graph_sizes + if len(cuda_graph_sizes) == 1: + self.graph_bs = [1, 2, 4, 8] + [ + i for i in range(16, cuda_graph_sizes[0] + 1, 16) + ] + elif len(cuda_graph_sizes) > 1: + self.graph_bs = cuda_graph_sizes def __post_init__(self): # assert os.path.isdir(self.model) @@ -398,5 +414,4 @@ def compute_hash(self) -> str: hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()[:10] - return hash_str - + return hash_str \ No newline at end of file From 0671737cfde2e256fca1c0e1f896a43019df59ec Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 26 Sep 2025 14:47:00 +0800 Subject: [PATCH 042/119] add level=1 --- atom/model_engine/model_runner.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 02e1ebddcb..f34849674d 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -72,8 +72,8 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.capture_cudagraph() torch.set_default_device("cpu") torch.set_default_dtype(default_dtype) - # if self.config.compilation_config.level == 1: - # self.model = torch.compile(self.model, fullgraph=True, backend="eager") + if self.config.compilation_config.level == 1: + self.model = torch.compile(self.model, fullgraph=True, backend="eager") if self.world_size > 1: if rank == 0: @@ -479,9 +479,6 @@ def capture_cudagraph(self): capture_range.set_description(f"Capturing {bs=}") graph = torch.cuda.CUDAGraph() - cu_seqlens_q = torch.arange(0, bs + 1, dtype=torch.int32, device='cuda') - cu_seqlens_k = cu_seqlens_q.clone() - set_context( False, batch_size=bs, From 2f356f656ca4efa75aa2edae4aad8c693e3b7524 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sun, 28 Sep 2025 08:38:44 +0800 Subject: [PATCH 043/119] add some copyright and remove some logger info --- atom/utils/backends.py | 27 +++++---------------------- atom/utils/compiler_inferface.py | 3 +++ atom/utils/custom_register.py | 2 -- atom/utils/decorators.py | 14 +++----------- atom/utils/forward_context.py | 11 ----------- 5 files changed, 11 insertions(+), 46 deletions(-) diff --git a/atom/utils/backends.py b/atom/utils/backends.py index 1d5349f63c..3163db6685 100644 --- a/atom/utils/backends.py +++ b/atom/utils/backends.py @@ -26,21 +26,6 @@ def get_static_graph_wrapper_cls(cls) -> str: return "atom.utils.cuda_graph.CUDAGraphWrapper" -# import vllm.envs as envs -# from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig -# from vllm.logger import init_logger -# from vllm.platforms import current_platform -# from atom.utils import is_torch_equal_or_newer, resolve_obj_by_qualname - -# from .compiler_interface import (CompilerInterface, EagerAdaptor, -# InductorAdaptor, InductorStandaloneAdaptor) -# from .counter import compilation_counter -# from .inductor_pass import InductorPass -# from .pass_manager import PostGradPassManager - -# logger = init_logger(__name__) - - def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface: # return InductorAdaptor() @@ -135,11 +120,11 @@ def load(self, compiled_graph = self.compiler.load(handle, graph, example_inputs, graph_index, runtime_shape) if runtime_shape is None: - logger.info( + logger.debug( "Directly load the %s-th graph for dynamic shape from %s via " "handle %s", graph_index, self.compiler.name, handle) else: - logger.info( + logger.debug( "Directly load the %s-th graph for shape %s from %s via " "handle %s", graph_index, str(runtime_shape), self.compiler.name, handle) @@ -213,11 +198,11 @@ def compile(self, logger.info("Cache the graph of shape %s for later use", str(runtime_shape)) if runtime_shape is None: - logger.info( + logger.debug( "Store the %s-th graph for dynamic shape from %s via " "handle %s", graph_index, self.compiler.name, handle) else: - logger.info( + logger.debug( "Store the %s-th graph for shape %s from %s via handle %s", graph_index, str(runtime_shape), self.compiler.name, handle) @@ -506,7 +491,7 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: forward_code_files = list( sorted(self.compilation_config.traced_files)) self.compilation_config.traced_files.clear() - logger.info( + logger.debug( "Traced files (to be considered for compilation cache):\n%s", "\n".join(forward_code_files)) hash_content = [] @@ -594,7 +579,6 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: item.submod_name for item in self.piecewise_graphs if not item.is_splitting_graph ] - print("submod_names_to_compile", submod_names_to_compile) # propagate the split graph to the piecewise backend, # compile submodules with symbolic shapes @@ -615,7 +599,6 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: logger.info("Computation graph saved to %s", graph_path) self._called = True - print('This is cudagraph_mode', self.compilation_config.cudagraph_mode) if self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE or \ not self.compilation_config.cudagraph_copy_inputs: diff --git a/atom/utils/compiler_inferface.py b/atom/utils/compiler_inferface.py index 388af5bec6..a845f52b16 100644 --- a/atom/utils/compiler_inferface.py +++ b/atom/utils/compiler_inferface.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + import contextlib import copy import hashlib diff --git a/atom/utils/custom_register.py b/atom/utils/custom_register.py index 1b5bde177f..9b5c899379 100644 --- a/atom/utils/custom_register.py +++ b/atom/utils/custom_register.py @@ -5,8 +5,6 @@ aiter_lib = Library("aiter", "FRAGMENT") -# from flash_attn import flash_attn_with_kvcache - def direct_register_custom_op( op_name: str, op_func: Callable, diff --git a/atom/utils/decorators.py b/atom/utils/decorators.py index 7b1dde261e..7d97f6e05f 100644 --- a/atom/utils/decorators.py +++ b/atom/utils/decorators.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + from typing import Callable, Optional, TypeVar, Union, overload import inspect import os,sys @@ -16,17 +19,6 @@ _T = TypeVar("_T", bound=type[nn.Module]) -# _current_vllm_config: Optional[Config] = None - -# def get_current_vllm_config() -> Config: -# if _current_vllm_config is None: -# # in ci, usually when we test custom ops/modules directly, -# # we don't set the vllm config. In that case, we set a default -# # config. -# from atom.config import Config -# return Config() -# return _current_vllm_config - def init_backend(config: Config): from .backends import VllmBackend diff --git a/atom/utils/forward_context.py b/atom/utils/forward_context.py index 580cd121dc..7e031fd35d 100644 --- a/atom/utils/forward_context.py +++ b/atom/utils/forward_context.py @@ -5,8 +5,6 @@ import torch -from aiter.jit.utils.torch_guard import torch_compile_guard - @dataclass class AttentionMetadata: k_cache: torch.Tensor @@ -15,15 +13,6 @@ class AttentionMetadata: v_scale: torch.Tensor -# @torch_compile_guard() -def set_attn_meta(k_cache: torch.Tensor, v_cache: torch.Tensor, - k_scale: torch.Tensor, v_scale: torch.Tensor) -> None: - attn_metadata = AttentionMetadata() - attn_metadata.k_cache = k_cache - attn_metadata.v_cache = v_cache - attn_metadata.k_scale = k_scale - attn_metadata.v_scale = v_scale - @dataclass class ForwardContext: From bb8544c19f9b10f6b3108db3bd15f18deda446bf Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sun, 28 Sep 2025 12:33:05 +0800 Subject: [PATCH 044/119] support other model --- atom/model_ops/linear.py | 750 +++++++++++++++------------------------ atom/models/llama.py | 13 +- atom/models/mixtral.py | 12 +- atom/models/qwen3.py | 14 +- atom/models/utils.py | 4 +- 5 files changed, 321 insertions(+), 472 deletions(-) diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 48cdfdcd65..9a4b0ae563 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -26,419 +26,212 @@ def divide(numerator, denominator): return numerator // denominator -# class LinearBase(nn.Module): - -# def __init__( -# self, -# input_size: int, -# output_size: int | list[int], -# tp_dim: int | None = None, -# bias: bool = False, -# quant_config: Optional[QuantizationConfig] = None, -# ): -# if quant_config is None: -# quant_config = QuantizationConfig() -# quant_type = quant_config["quant_type"] -# params_dtype = quant_config["quant_dtype"] -# super().__init__() -# self.input_size = input_size -# self.output_size = ( -# output_size if isinstance(output_size, int) else sum(output_size) -# ) -# self.tp_dim = tp_dim -# self.tp_rank = get_tp_group().rank -# self.tp_size = get_tp_group().world_size -# self.output_partition_sizes = ( -# output_size if isinstance(output_size, list) else [output_size] -# ) -# if tp_dim == 1: -# self.input_size = divide(input_size, self.tp_size) -# elif tp_dim == 0: -# self.output_size = divide(self.output_size, self.tp_size) -# self.output_partition_sizes = [ -# divide(s, self.tp_size) for s in self.output_partition_sizes -# ] -# self.weight = nn.Parameter( -# torch.empty((self.output_size, self.input_size), dtype=params_dtype), -# requires_grad=False, -# ) -# if bias: -# self.bias = nn.Parameter( -# torch.empty(self.output_size, dtype=params_dtype), requires_grad=False -# ) -# self.bias.weight_loader_process = self.weight_loader_process -# else: -# self.register_parameter("bias", None) -# self.quant_type = quant_type -# self.params_dtype = params_dtype -# if quant_type != QuantType.No: -# if quant_type == QuantType.per_Tensor: -# self.weight_scale = nn.Parameter( -# torch.empty(len(self.output_partition_sizes), 1, dtype=dtypes.fp32), -# requires_grad=False, -# ) -# if not quant_config["is_dynamic"]: -# self.input_scale = nn.Parameter( -# torch.empty( -# len(self.output_partition_sizes), 1, dtype=dtypes.fp32 -# ), -# requires_grad=False, -# ) -# self.input_scale.weight_loader_process = self.weight_loader_process -# self.input_scale.weight_loader = self.weight_loader -# elif quant_type == QuantType.per_Token: -# self.weight_scale = nn.Parameter( -# torch.empty(self.output_size, 1, dtype=dtypes.fp32), -# requires_grad=False, -# ) -# elif quant_type == QuantType.per_1x128: -# self.weight_scale = nn.Parameter( -# torch.empty( -# divide(self.output_size, 128), -# (self.input_size + 127) // 128, -# dtype=dtypes.fp32, -# ), -# requires_grad=False, -# ) -# elif quant_type == QuantType.per_1x32: -# self.weight_scale = nn.Parameter( -# torch.empty( -# self.output_size, -# (self.input_size + 31) // 32, -# dtype=dtypes.fp8_e8m0, -# ), -# requires_grad=False, -# ) -# self.weight.weight_loader_process = self.weight_loader_process -# self.weight_scale.weight_loader_process = self.weight_loader_process -# else: -# self.weight.weight_loader_process = self.weight_loader_process -# self.register_parameter("weight_scale", None) -# self.weight.weight_loader = self.weight_loader -# if self.bias is not None: -# self.bias.weight_loader = self.weight_loader -# if self.weight_scale is not None: -# self.weight_scale.weight_loader = self.weight_loader -# self.need_normalize_e4m3fn_to_e4m3fnuz = params_dtype == torch.float8_e4m3fnuz - -# @staticmethod -# def weight_loader_process( -# param: nn.Parameter, -# loaded_weight: torch.Tensor, -# post_process_func: Callable = lambda a: a, -# ): -# if ( -# param.data.dtype != loaded_weight.dtype -# and param.data.element_size() == loaded_weight.element_size() -# ): -# param.data = param.data.view(loaded_weight.dtype) -# param.data.copy_(post_process_func(loaded_weight)) - -# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): -# param_data = param.data -# param.weight_loader_process(param_data, loaded_weight) - -# def process_weights_after_loading(self): -# if ( -# self.quant_type == QuantType.per_Tensor -# and len(self.output_partition_sizes) > 1 -# ): -# weight_scale, weight = requantize_with_max_scale( -# weight=self.weight.data, -# weight_scale=self.weight_scale.data, -# logical_widths=self.output_partition_sizes, -# normalize_e4m3fn_to_e4m3fnuz=self.need_normalize_e4m3fn_to_e4m3fnuz, -# ) -# self.weight.data = weight -# self.weight_scale.data = weight_scale.view(-1) -# if hasattr(self, "input_scale"): -# self.input_scale.data = ( -# self.input_scale.data.max() * 2.0 -# if self.need_normalize_e4m3fn_to_e4m3fnuz -# else self.input_scale.data.max() -# ) -# elif self.need_normalize_e4m3fn_to_e4m3fnuz: -# self.weight.data, self.weight_scale.data, _ = normalize_e4m3fn_to_e4m3fnuz( -# self.weight.data, self.weight_scale.data -# ) -# if ( -# self.quant_type == QuantType.per_Token and self.params_dtype == dtypes.fp8 -# ) or (self.quant_type == QuantType.per_1x32): -# self.weight.data = shuffle_weight(self.weight.data, (16, 16)) - -# def forward( -# self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 -# ) -> torch.Tensor: -# if self.quant_type.value == QuantType.No.value: -# y = tgemm.mm(x, self.weight, self.bias) -# else: -# if x_scale is None: -# quant_func = get_hip_quant(self.quant_type) -# x, x_scale = quant_func( -# x, -# quant_dtype=self.params_dtype, -# scale=getattr(self, "input_scale", None), -# ) -# if self.quant_type.value == QuantType.per_Tensor.value: -# y = tgemm.mm( -# x, -# self.weight, -# self.bias, -# otype=otype, -# scale_a=x_scale, -# scale_b=self.weight_scale, -# ) -# elif self.quant_type.value == QuantType.per_Token.value: -# if self.params_dtype == dtypes.i8: -# y = gemm_a8w8( -# x, -# self.weight, -# x_scale, -# self.weight_scale, -# self.bias, -# dtype=otype, -# ) -# else: -# y = gemm_a8w8_bpreshuffle( -# x, -# self.weight, -# x_scale, -# self.weight_scale, -# self.bias, -# dtype=otype, -# ) -# elif self.quant_type.value == QuantType.per_1x128.value: -# y = gemm_a8w8_blockscale( -# x, self.weight, x_scale, self.weight_scale, dtype=otype -# ) -# if self.bias is not None: -# y += self.bias -# elif self.quant_type.value == QuantType.per_1x32.value: -# m = x.view(-1, x.size(-1)).shape[0] -# y = torch.empty( -# ((m + 31) // 32 * 32, self.output_size), -# dtype=otype, -# device=x.device, -# ) -# y = gemm_a4w4( -# x, -# self.weight, -# x_scale, -# self.weight_scale, -# y, -# self.bias, -# dtype=otype, -# ) -# if self.tp_dim == 1 and self.tp_size > 1: -# y = get_tp_group().all_reduce(y, open_fp8_quant=False) -# return y - - -# class ReplicatedLinear(LinearBase): - -# def __init__( -# self, -# input_size: int, -# output_size: int, -# bias: bool = False, -# quant_config: Optional[QuantizationConfig] = None, -# **kwargs, -# ): -# super().__init__( -# input_size, -# output_size, -# tp_dim=None, -# bias=bias, -# quant_config=quant_config, -# ) - -# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): -# param_data = param.data -# param.weight_loader_process(param_data, loaded_weight) - - -# class ColumnParallelLinear(LinearBase): - -# def __init__( -# self, -# input_size: int, -# output_size: int, -# bias: bool = False, -# quant_config: Optional[QuantizationConfig] = None, -# **kwargs, -# ): -# self.tp_dim = 0 -# super().__init__( -# input_size, -# output_size, -# self.tp_dim, -# bias, -# quant_config=quant_config, -# ) - -# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): -# param_data = param.data -# shard_size = param_data.size(self.tp_dim) -# start_idx = self.tp_rank * shard_size -# loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) -# param.weight_loader_process(param_data, loaded_weight) - - -# class MergedColumnParallelLinear(LinearBase): - -# def __init__( -# self, -# input_size: int, -# output_sizes: list[int], -# bias: bool = False, -# quant_config: Optional[QuantizationConfig] = None, -# **kwargs, -# ): -# self.output_sizes = output_sizes -# super().__init__( -# input_size, -# output_sizes, -# tp_dim=0, -# bias=bias, -# quant_config=quant_config, -# ) - -# def weight_loader( -# self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int -# ): -# param_data = param.data -# shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size -# shard_size = self.output_sizes[loaded_shard_id] // self.tp_size -# if param is getattr(self, "weight_scale", None) or param is getattr( -# self, "input_scale", None -# ): -# if self.quant_type == QuantType.per_1x128: -# shard_offset //= 128 -# shard_size //= 128 -# elif self.quant_type == QuantType.per_Tensor: -# loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) -# shard_offset = loaded_shard_id -# shard_size = 1 - -# param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) -# loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] -# param.weight_loader_process(param_data, loaded_weight) - - -# class QKVParallelLinear(ColumnParallelLinear): - -# def __init__( -# self, -# hidden_size: int, -# head_size: int, -# total_num_heads: int, -# total_num_kv_heads: int | None = None, -# bias: bool = False, -# quant_config: Optional[QuantizationConfig] = None, -# **kwargs, -# ): -# self.head_size = head_size -# self.total_num_heads = total_num_heads -# self.total_num_kv_heads = total_num_kv_heads or total_num_heads -# tp_size = get_tp_group().world_size -# self.num_heads = divide(self.total_num_heads, tp_size) -# self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) -# input_size = hidden_size -# output_sizes = [ -# self.total_num_heads * self.head_size, -# self.total_num_kv_heads * self.head_size, -# self.total_num_kv_heads * self.head_size, -# ] - -# super().__init__( -# input_size, -# output_sizes, -# bias=bias, -# quant_config=quant_config, -# ) - -# def weight_loader( -# self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str -# ): -# param_data = param.data -# assert loaded_shard_id in ["q", "k", "v"] -# if loaded_shard_id == "q": -# shard_size = self.num_heads * self.head_size -# shard_offset = 0 -# elif loaded_shard_id == "k": -# shard_size = self.num_kv_heads * self.head_size -# shard_offset = self.num_heads * self.head_size -# else: -# shard_size = self.num_kv_heads * self.head_size -# shard_offset = ( -# self.num_heads * self.head_size + self.num_kv_heads * self.head_size -# ) -# if param is getattr(self, "weight_scale", None) or param is getattr( -# self, "input_scale", None -# ): -# if self.quant_type == QuantType.per_1x128: -# shard_offset //= 128 -# shard_size //= 128 -# elif self.quant_type == QuantType.per_Tensor: -# loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) -# shard_offset = ["q", "k", "v"].index(loaded_shard_id) -# shard_size = 1 - -# param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) -# loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] -# param.weight_loader_process(param_data, loaded_weight) - - -# class RowParallelLinear(LinearBase): - -# def __init__( -# self, -# input_size: int, -# output_size: int, -# bias: bool = False, -# quant_config: Optional[QuantizationConfig] = None, -# **kwargs, -# ): -# self.tp_rank = get_tp_group().rank -# super().__init__( -# input_size, -# output_size, -# tp_dim=1, -# bias=bias if self.tp_rank == 0 else False, -# quant_config=quant_config, -# ) - -# def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): -# param_data = param.data -# shard_size = param_data.size(self.tp_dim) -# if len(loaded_weight.shape) == 0: -# loaded_weight = loaded_weight.view(1, 1) -# if loaded_weight.size(self.tp_dim) == 1 and self.tp_size > 1: -# loaded_weight = loaded_weight.repeat(1, self.tp_size) -# start_idx = self.tp_rank * shard_size -# loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) -# param.weight_loader_process(param_data, loaded_weight) - - -import torch.distributed as dist - class LinearBase(nn.Module): def __init__( self, input_size: int, - output_size: int, + output_size: int | list[int], tp_dim: int | None = None, + bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, ): + if quant_config is None: + quant_config = QuantizationConfig() + quant_type = quant_config["quant_type"] + params_dtype = quant_config["quant_dtype"] super().__init__() self.input_size = input_size - self.output_size = output_size + self.output_size = ( + output_size if isinstance(output_size, int) else sum(output_size) + ) self.tp_dim = tp_dim - self.tp_rank = dist.get_rank() - self.tp_size = dist.get_world_size() + self.tp_rank = get_tp_group().rank + self.tp_size = get_tp_group().world_size + self.output_partition_sizes = ( + output_size if isinstance(output_size, list) else [output_size] + ) + if tp_dim == 1: + self.input_size = divide(input_size, self.tp_size) + elif tp_dim == 0: + self.output_size = divide(self.output_size, self.tp_size) + self.output_partition_sizes = [ + divide(s, self.tp_size) for s in self.output_partition_sizes + ] + self.weight = nn.Parameter( + torch.empty((self.output_size, self.input_size), dtype=params_dtype), + requires_grad=False, + ) + if bias: + self.bias = nn.Parameter( + torch.empty(self.output_size, dtype=params_dtype), requires_grad=False + ) + self.bias.weight_loader_process = self.weight_loader_process + else: + self.register_parameter("bias", None) + self.quant_type = quant_type + self.params_dtype = params_dtype + if quant_type != QuantType.No: + if quant_type == QuantType.per_Tensor: + self.weight_scale = nn.Parameter( + torch.empty(len(self.output_partition_sizes), 1, dtype=dtypes.fp32), + requires_grad=False, + ) + if not quant_config["is_dynamic"]: + self.input_scale = nn.Parameter( + torch.empty( + len(self.output_partition_sizes), 1, dtype=dtypes.fp32 + ), + requires_grad=False, + ) + self.input_scale.weight_loader_process = self.weight_loader_process + self.input_scale.weight_loader = self.weight_loader + elif quant_type == QuantType.per_Token: + self.weight_scale = nn.Parameter( + torch.empty(self.output_size, 1, dtype=dtypes.fp32), + requires_grad=False, + ) + elif quant_type == QuantType.per_1x128: + self.weight_scale = nn.Parameter( + torch.empty( + divide(self.output_size, 128), + (self.input_size + 127) // 128, + dtype=dtypes.fp32, + ), + requires_grad=False, + ) + elif quant_type == QuantType.per_1x32: + self.weight_scale = nn.Parameter( + torch.empty( + self.output_size, + (self.input_size + 31) // 32, + dtype=dtypes.fp8_e8m0, + ), + requires_grad=False, + ) + self.weight.weight_loader_process = self.weight_loader_process + self.weight_scale.weight_loader_process = self.weight_loader_process + else: + self.weight.weight_loader_process = self.weight_loader_process + self.register_parameter("weight_scale", None) + self.weight.weight_loader = self.weight_loader + if self.bias is not None: + self.bias.weight_loader = self.weight_loader + if self.weight_scale is not None: + self.weight_scale.weight_loader = self.weight_loader + self.need_normalize_e4m3fn_to_e4m3fnuz = params_dtype == torch.float8_e4m3fnuz + + @staticmethod + def weight_loader_process( + param: nn.Parameter, + loaded_weight: torch.Tensor, + post_process_func: Callable = lambda a: a, + ): + if ( + param.data.dtype != loaded_weight.dtype + and param.data.element_size() == loaded_weight.element_size() + ): + param.data = param.data.view(loaded_weight.dtype) + param.data.copy_(post_process_func(loaded_weight)) - def forward(self, x: torch.Tensor) -> torch.Tensor: - raise NotImplementedError + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + param.weight_loader_process(param_data, loaded_weight) + + def process_weights_after_loading(self): + if ( + self.quant_type == QuantType.per_Tensor + and len(self.output_partition_sizes) > 1 + ): + weight_scale, weight = requantize_with_max_scale( + weight=self.weight.data, + weight_scale=self.weight_scale.data, + logical_widths=self.output_partition_sizes, + normalize_e4m3fn_to_e4m3fnuz=self.need_normalize_e4m3fn_to_e4m3fnuz, + ) + self.weight.data = weight + self.weight_scale.data = weight_scale.view(-1) + if hasattr(self, "input_scale"): + self.input_scale.data = ( + self.input_scale.data.max() * 2.0 + if self.need_normalize_e4m3fn_to_e4m3fnuz + else self.input_scale.data.max() + ) + elif self.need_normalize_e4m3fn_to_e4m3fnuz: + self.weight.data, self.weight_scale.data, _ = normalize_e4m3fn_to_e4m3fnuz( + self.weight.data, self.weight_scale.data + ) + if ( + self.quant_type == QuantType.per_Token and self.params_dtype == dtypes.fp8 + ) or (self.quant_type == QuantType.per_1x32): + self.weight.data = shuffle_weight(self.weight.data, (16, 16)) + + def forward( + self, x: torch.Tensor, x_scale: Optional[torch.Tensor] = None, otype=dtypes.bf16 + ) -> torch.Tensor: + if self.quant_type.value == QuantType.No.value: + y = tgemm.mm(x, self.weight, self.bias) + else: + if x_scale is None: + quant_func = get_hip_quant(self.quant_type) + x, x_scale = quant_func( + x, + quant_dtype=self.params_dtype, + scale=getattr(self, "input_scale", None), + ) + if self.quant_type.value == QuantType.per_Tensor.value: + y = tgemm.mm( + x, + self.weight, + self.bias, + otype=otype, + scale_a=x_scale, + scale_b=self.weight_scale, + ) + elif self.quant_type.value == QuantType.per_Token.value: + if self.params_dtype == dtypes.i8: + y = gemm_a8w8( + x, + self.weight, + x_scale, + self.weight_scale, + self.bias, + dtype=otype, + ) + else: + y = gemm_a8w8_bpreshuffle( + x, + self.weight, + x_scale, + self.weight_scale, + self.bias, + dtype=otype, + ) + elif self.quant_type.value == QuantType.per_1x128.value: + y = gemm_a8w8_blockscale( + x, self.weight, x_scale, self.weight_scale, dtype=otype + ) + if self.bias is not None: + y += self.bias + elif self.quant_type.value == QuantType.per_1x32.value: + m = x.view(-1, x.size(-1)).shape[0] + y = torch.empty( + ((m + 31) // 32 * 32, self.output_size), + dtype=otype, + device=x.device, + ) + y = gemm_a4w4( + x, + self.weight, + x_scale, + self.weight_scale, + y, + self.bias, + dtype=otype, + ) + if self.tp_dim == 1 and self.tp_size > 1: + y = get_tp_group().all_reduce(y, open_fp8_quant=False) + return y class ReplicatedLinear(LinearBase): @@ -448,21 +241,20 @@ def __init__( input_size: int, output_size: int, bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): - super().__init__(input_size, output_size) - self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) + super().__init__( + input_size, + output_size, + tp_dim=None, + bias=bias, + quant_config=quant_config, + ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): - param.data.copy_(loaded_weight) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return F.linear(x, self.weight, self.bias) + param_data = param.data + param.weight_loader_process(param_data, loaded_weight) class ColumnParallelLinear(LinearBase): @@ -472,48 +264,65 @@ def __init__( input_size: int, output_size: int, bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): - super().__init__(input_size, output_size, 0) - self.input_size_per_partition = input_size - self.output_size_per_partition = divide(output_size, self.tp_size) - - self.weight = nn.Parameter(torch.empty(self.output_size_per_partition, self.input_size)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size_per_partition)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) + self.tp_dim = 0 + super().__init__( + input_size, + output_size, + self.tp_dim, + bias, + quant_config=quant_config, + ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param_data.copy_(loaded_weight) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - return F.linear(x, self.weight, self.bias) + param.weight_loader_process(param_data, loaded_weight) -class MergedColumnParallelLinear(ColumnParallelLinear): +class MergedColumnParallelLinear(LinearBase): def __init__( self, input_size: int, output_sizes: list[int], bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): self.output_sizes = output_sizes - super().__init__(input_size, sum(output_sizes), bias=bias) - - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int): + super().__init__( + input_size, + output_sizes, + tp_dim=0, + bias=bias, + quant_config=quant_config, + ) + + def weight_loader( + self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: int + ): param_data = param.data shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size shard_size = self.output_sizes[loaded_shard_id] // self.tp_size + if param is getattr(self, "weight_scale", None) or param is getattr( + self, "input_scale", None + ): + if self.quant_type == QuantType.per_1x128: + shard_offset //= 128 + shard_size //= 128 + elif self.quant_type == QuantType.per_Tensor: + loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) + shard_offset = loaded_shard_id + shard_size = 1 + param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] - param_data.copy_(loaded_weight) + param.weight_loader_process(param_data, loaded_weight) class QKVParallelLinear(ColumnParallelLinear): @@ -525,18 +334,32 @@ def __init__( total_num_heads: int, total_num_kv_heads: int | None = None, bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): self.head_size = head_size self.total_num_heads = total_num_heads self.total_num_kv_heads = total_num_kv_heads or total_num_heads - tp_size = dist.get_world_size() + tp_size = get_tp_group().world_size self.num_heads = divide(self.total_num_heads, tp_size) self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) input_size = hidden_size - output_size = (self.total_num_heads + 2 * self.total_num_kv_heads) * self.head_size - super().__init__(input_size, output_size, bias) - - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str): + output_sizes = [ + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ] + + super().__init__( + input_size, + output_sizes, + bias=bias, + quant_config=quant_config, + ) + + def weight_loader( + self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded_shard_id: str + ): param_data = param.data assert loaded_shard_id in ["q", "k", "v"] if loaded_shard_id == "q": @@ -547,10 +370,23 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor, loaded shard_offset = self.num_heads * self.head_size else: shard_size = self.num_kv_heads * self.head_size - shard_offset = self.num_heads * self.head_size + self.num_kv_heads * self.head_size + shard_offset = ( + self.num_heads * self.head_size + self.num_kv_heads * self.head_size + ) + if param is getattr(self, "weight_scale", None) or param is getattr( + self, "input_scale", None + ): + if self.quant_type == QuantType.per_1x128: + shard_offset //= 128 + shard_size //= 128 + elif self.quant_type == QuantType.per_Tensor: + loaded_weight = loaded_weight.view(1, 1).repeat(self.tp_size, 1) + shard_offset = ["q", "k", "v"].index(loaded_shard_id) + shard_size = 1 + param_data = param_data.narrow(self.tp_dim, shard_offset, shard_size) loaded_weight = loaded_weight.chunk(self.tp_size, self.tp_dim)[self.tp_rank] - param_data.copy_(loaded_weight) + param.weight_loader_process(param_data, loaded_weight) class RowParallelLinear(LinearBase): @@ -560,28 +396,26 @@ def __init__( input_size: int, output_size: int, bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, ): - super().__init__(input_size, output_size, 1) - self.input_size_per_partition = divide(input_size, self.tp_size) - self.output_size_per_partition = output_size - - self.weight = nn.Parameter(torch.empty(self.output_size, self.input_size_per_partition)) - self.weight.weight_loader = self.weight_loader - if bias: - self.bias = nn.Parameter(torch.empty(self.output_size)) - self.bias.weight_loader = self.weight_loader - else: - self.register_parameter("bias", None) + self.tp_rank = get_tp_group().rank + super().__init__( + input_size, + output_size, + tp_dim=1, + bias=bias if self.tp_rank == 0 else False, + quant_config=quant_config, + ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): param_data = param.data shard_size = param_data.size(self.tp_dim) + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.view(1, 1) + if loaded_weight.size(self.tp_dim) == 1 and self.tp_size > 1: + loaded_weight = loaded_weight.repeat(1, self.tp_size) start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) - param_data.copy_(loaded_weight) + param.weight_loader_process(param_data, loaded_weight) - def forward(self, x: torch.Tensor) -> torch.Tensor: - y = F.linear(x, self.weight, self.bias if self.tp_rank == 0 else None) - if self.tp_size > 1: - dist.all_reduce(y) - return y diff --git a/atom/models/llama.py b/atom/models/llama.py index 64127bbbe8..7f4014ea2a 100644 --- a/atom/models/llama.py +++ b/atom/models/llama.py @@ -47,6 +47,7 @@ from aiter.rotary_embedding import get_rope from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead from atom.config import QuantizationConfig, Config +from atom.utils.decorators import support_torch_compile from atom.models.utils import ( PPMissingLayer, @@ -116,6 +117,7 @@ def __init__( bias_o_proj: bool = False, cache_config: str = "bf16", prefix: str = "", + layer_num: int = 0, ) -> None: super().__init__() layer_idx = extract_layer_index(prefix) @@ -146,7 +148,7 @@ def __init__( self.scaling = self.head_dim**-0.5 self.rope_theta = rope_theta self.max_position_embeddings = max_position_embeddings - + self.layer_num = layer_num self.qkv_proj = QKVParallelLinear( hidden_size=hidden_size, head_size=self.head_dim, @@ -181,6 +183,7 @@ def __init__( self.scaling, num_kv_heads=self.num_kv_heads, kv_cache_dtype=cache_config, + layer_num=layer_num, per_layer_sliding_window=sliding_window, prefix=f"{prefix}.attn", ) @@ -227,6 +230,7 @@ def __init__( cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + layer_num: int = 0, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -264,6 +268,7 @@ def __init__( bias_o_proj=bias_o_proj, cache_config=cache_config, prefix=f"{prefix}.self_attn", + layer_num=layer_num, ) self.mlp = LlamaMLP( hidden_size=self.hidden_size, @@ -298,7 +303,7 @@ def forward( return hidden_states, residual -# @support_torch_compile +@support_torch_compile class LlamaModel(nn.Module): def __init__( self, @@ -324,13 +329,15 @@ def __init__( self.embed_tokens = PPMissingLayer() self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, - lambda prefix: layer_type( + lambda prefix, layer_num=None: layer_type( config=config, cache_config=cache_config, quant_config=quant_config, prefix=prefix, + layer_num=layer_num ), prefix=f"{prefix}.layers", + layer_num_offset=0 ) if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/atom/models/mixtral.py b/atom/models/mixtral.py index b90c693446..8e16f94e6b 100644 --- a/atom/models/mixtral.py +++ b/atom/models/mixtral.py @@ -44,6 +44,7 @@ from aiter.rotary_embedding import get_rope from atom.model_ops.embed_head import VocabParallelEmbedding, ParallelLMHead from atom.config import QuantizationConfig, Config +from atom.utils.decorators import support_torch_compile from atom.models.utils import ( PPMissingLayer, @@ -126,6 +127,7 @@ def __init__( cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + layer_num: int = 0, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -151,6 +153,7 @@ def __init__( self.kv_size = self.num_kv_heads * self.head_dim self.scaling = self.head_dim**-0.5 self.rope_theta = rope_theta + self.layer_num = layer_num self.qkv_proj = QKVParallelLinear( hidden_size, @@ -181,6 +184,7 @@ def __init__( self.scaling, num_kv_heads=self.num_kv_heads, kv_cache_dtype=cache_config, + layer_num=layer_num, quant_config=quant_config, prefix=f"{prefix}.attn", ) @@ -206,6 +210,7 @@ def __init__( cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + layer_num: int = 0, ) -> None: super().__init__() self.hidden_size = config.hidden_size @@ -259,7 +264,7 @@ def forward( return hidden_states, residual -# @support_torch_compile +@support_torch_compile class MixtralModel(nn.Module): def __init__( @@ -283,10 +288,11 @@ def __init__( self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, - lambda prefix: MixtralDecoderLayer( - config, cache_config, quant_config=quant_config, prefix=prefix + lambda prefix, layer_num=None: MixtralDecoderLayer( + config, cache_config, quant_config=quant_config, prefix=prefix, layer_num=layer_num ), prefix=f"{prefix}.layers", + layer_num_offset=0, ) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 7ee16b56b6..8fb0bf6ccc 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -59,13 +59,13 @@ def __init__( self.total_num_heads, self.total_num_kv_heads, bias=qkv_bias, - # quant_config=quant_config, + quant_config=quant_config, ) self.o_proj = RowParallelLinear( self.total_num_heads * self.head_dim, hidden_size, bias=False, - # quant_config=quant_config, + quant_config=quant_config, ) self.rotary_emb = get_rope( self.head_dim, @@ -116,13 +116,13 @@ def __init__( hidden_size, [intermediate_size] * 2, bias=False, - # quant_config=quant_config, + quant_config=quant_config, ) self.down_proj = RowParallelLinear( intermediate_size, hidden_size, bias=False, - # quant_config=quant_config, + quant_config=quant_config, ) assert hidden_act == "silu" self.act_fn = SiluAndMul() @@ -156,14 +156,14 @@ def __init__( rope_theta=getattr(config, "rope_theta", 1000000), rope_scaling=getattr(config, "rope_scaling", None), kv_cache_dtype=cache_config, - layer_num=layer_num - # quant_config=quant_config, + layer_num=layer_num, + quant_config=quant_config, ) self.mlp = Qwen3MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, - # quant_config=quant_config, + quant_config=quant_config, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( diff --git a/atom/models/utils.py b/atom/models/utils.py index 6f07df0d20..09c2aff8c2 100644 --- a/atom/models/utils.py +++ b/atom/models/utils.py @@ -101,6 +101,7 @@ def make_layers( num_hidden_layers: int, layer_fn: LayerFn, prefix: str, + layer_num_offset: int = 0, ) -> Tuple[int, int, torch.nn.ModuleList]: """Make a list of layers with the given layer function, taking pipeline parallelism into account. @@ -112,7 +113,8 @@ def make_layers( ) modules = torch.nn.ModuleList( [PPMissingLayer() for _ in range(start_layer)] - + [layer_fn(prefix=f"{prefix}.{idx}") for idx in range(start_layer, end_layer)] + + [layer_fn(prefix=f"{prefix}.{idx}", layer_num=layer_num_offset + idx) + for idx in range(start_layer, end_layer)] + [PPMissingLayer() for _ in range(end_layer, num_hidden_layers)] ) return start_layer, end_layer, modules From 580f877a63d56637f727d1cf182b0152980dd398 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sun, 28 Sep 2025 13:23:29 +0800 Subject: [PATCH 045/119] fix mixtral bug --- atom/models/mixtral.py | 1 + 1 file changed, 1 insertion(+) diff --git a/atom/models/mixtral.py b/atom/models/mixtral.py index 8e16f94e6b..2f2a18166c 100644 --- a/atom/models/mixtral.py +++ b/atom/models/mixtral.py @@ -226,6 +226,7 @@ def __init__( cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", + layer_num=layer_num, ) self.block_sparse_moe = MixtralMoE( config=config, From 7457615e1abb3f663a882860399f2a3bed88b8e1 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 28 Sep 2025 16:22:45 +0000 Subject: [PATCH 046/119] merge output intead of ret one by one --- atom/model_engine/engine_core.py | 26 +++++++----- atom/model_engine/engine_core_mgr.py | 17 ++++---- atom/model_engine/llm_engine.py | 60 +++++++++++++--------------- 3 files changed, 52 insertions(+), 51 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 73602bc4d7..30f9c55636 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -5,7 +5,7 @@ import threading import weakref from contextlib import ExitStack -from typing import Any, Optional +from typing import Any, List, Optional import zmq import zmq.asyncio @@ -64,7 +64,7 @@ def __init__(self, config: Config, input_address: str, output_address: str): logger.info("Engine core model runner loaded") self.scheduler = Scheduler(config) self.input_queue = queue.Queue[Sequence]() - self.output_queue = queue.Queue[Sequence]() + self.output_queue = queue.Queue[List[Sequence]]() self.input_address = input_address self.output_address = output_address logger.info(f"Engine core input address: {self.input_address}") @@ -102,7 +102,7 @@ def run_engine(config: Config, input_address: str, output_address: str): def _send_engine_dead(self): logger.info("Send engine core dead signal") - self.output_queue.put_nowait(get_exit_sequence()) + self.output_queue.put_nowait([get_exit_sequence()]) self.output_thread.join(timeout=5.0) def busy_loop(self): @@ -118,9 +118,9 @@ def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() token_ids = self.model_runner.call("run", scheduled_batchs) self.scheduler.postprocess(scheduled_batchs.seqs, token_ids) - for seq in scheduled_batchs.seqs: - if seq.is_finished: - self.output_queue.put_nowait(seq) + self.output_queue.put_nowait( + [seq for seq in scheduled_batchs.seqs if seq.is_finished] + ) def _process_input_queue(self): while not self.input_queue.empty(): @@ -167,14 +167,18 @@ def process_output_sockets(self, output_address: str): logger.info("Engine core output socket connected") while True: - seq = self.output_queue.get() - if seq.status == SequenceStatus.EXIT_ENGINE: + seqs = self.output_queue.get() + valid_seqs = [ + seq for seq in seqs if seq.status != SequenceStatus.EXIT_ENGINE + ] + serialized_obj = pickle.dumps((EngineCoreRequestType.ADD, valid_seqs)) + socket.send(serialized_obj) + num_valid = len(valid_seqs) + logger.info(f"Engine core output: {num_valid} sequences") + if len(valid_seqs) != len(seqs): socket.send(pickle.dumps((EngineCoreRequestType.SHUTDOWN, None))) logger.info("Engine core output thread closed") break - serialized_obj = pickle.dumps((EngineCoreRequestType.ADD, seq)) - socket.send(serialized_obj) - logger.info(f"Engine core output: {seq.id}") def start_profiler(self): if self.profile_enbaled: diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 4c8e655981..e647fefdcc 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -4,6 +4,7 @@ import queue import weakref from threading import Thread +from typing import List import zmq import zmq.asyncio @@ -25,7 +26,7 @@ class CoreManager: def __init__(self, config: Config): sync_ctx = zmq.Context(io_threads=2) self.ctx = zmq.asyncio.Context(sync_ctx) if config.asyncio_mode else sync_ctx - self.outputs_queue = queue.Queue[Sequence]() + self.outputs_queue = queue.Queue[List[Sequence]]() engine_core_process, addresses = launch_engine_core(config) logger.info("Engine core inited") self.engine_core_process = engine_core_process @@ -62,15 +63,15 @@ def process_outputs_socket(): break obj = self.output_socket.recv(copy=False) - request_type, seq = pickle.loads(obj) + request_type, seqs = pickle.loads(obj) if request_type == EngineCoreRequestType.SHUTDOWN: logger.info("Core client output thread receive shutdown signal") self.engine_core_process.join() self.engine_core_process = None break elif request_type == EngineCoreRequestType.ADD: - logger.info(f"Engine core output sequence id: {seq.id}") - self.outputs_queue.put_nowait(seq) + # logger.info(f"Engine core output sequence id: {seq.id}") + self.outputs_queue.put_nowait(seqs) finally: # Close sockets. shutdown_socket.close(linger=0) @@ -120,12 +121,12 @@ def _shut_down_core_engine(self): # if self.engine_core_process is not None and self.engine_core_process.is_alive(): # shutdown_all_processes([self.engine_core_process]) - def get_output(self) -> Sequence: - outputs = self.outputs_queue.get() - logger.info(f"Get output from engine core, sequence id: {outputs.id}") + def get_output(self) -> List[Sequence]: + seqs = self.outputs_queue.get() + # logger.info(f"Get output from engine core, sequence id: {outputs.id}") # if isinstance(outputs, Exception): # raise outputs from None - return outputs + return seqs def is_rest(self): return not self.outputs_queue.empty() diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index bfa36b0f87..e25ef82d14 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -1,6 +1,7 @@ import logging import time from dataclasses import fields +from typing import List import torch.multiprocessing as mp from tqdm.auto import tqdm @@ -39,18 +40,9 @@ def add_request( # 2. add req to scheduler self.core_mgr.add_request(req) - def step(self): - # seqs, is_prefill = self.scheduler.schedule() - # token_ids = self.model_runner.call("run", seqs, is_prefill) - # self.scheduler.postprocess(seqs, token_ids) - # outputs = [ - # (seq.seq_id, seq.completion_token_ids) for seq in seqs if seq.is_finished - # ] - # num_tokens = sum(len(seq) for seq in seqs) if is_prefill else -len(seqs) - # return outputs, num_tokens - - seq = self.core_mgr.get_output() - return seq + def step(self) -> list[Sequence]: + seqs = self.core_mgr.get_output() + return seqs def is_finished(self): return not self.io_processor.has_pending_requests() @@ -73,9 +65,9 @@ def generate( while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() ): - seq = self.step() - out = self.io_processor.postprocess(seq) - outputs[seq.id] = out + seqs = self.step() + outs = self.io_processor.postprocess(seqs) + outputs.update(outs) # # Stop profiling for all ranks if enabled # if enable_profiling: # self.model_runner.call("stop_profiler") @@ -109,26 +101,30 @@ def preprocess( ) return seq - def postprocess(self, req: Sequence): + def postprocess(self, reqs: List[Sequence]): """responsible for: 1) Compute stats for logging 2) Detokenize""" - self.requests.pop(req.id) - output_str = self.tokenizer.decode(req.completion_token_ids) - req.leave_time = time.time() - print( - f"Request {req.id} finished with reason {req.leave_reason}. " - f"Input tokens: {req.num_prompt_tokens}, output tokens: {req.num_completion_tokens}, " - f"latency: {req.leave_time - req.arrive_time:.2f}s" - ) - return { - "text": output_str, - "token_ids": req.completion_token_ids, - "latency": req.leave_time - req.arrive_time, - "finish_reason": req.leave_reason, - "num_tokens_input": req.num_prompt_tokens, - "num_tokens_output": req.num_completion_tokens, - } + outputs = {} + for req in reqs: + self.requests.pop(req.id) + # output_str = self.tokenizer.decode(req.completion_token_ids) + output_str = "self.tokenizer.decode(req.completion_token_ids)" + req.leave_time = time.time() + print( + f"Request {req.id} finished with reason {req.leave_reason}. " + f"Input tokens: {req.num_prompt_tokens}, output tokens: {req.num_completion_tokens}, " + f"latency: {req.leave_time - req.arrive_time:.2f}s" + ) + outputs[req.id] = { + "text": output_str, + "token_ids": req.completion_token_ids, + "latency": req.leave_time - req.arrive_time, + "finish_reason": req.leave_reason, + "num_tokens_input": req.num_prompt_tokens, + "num_tokens_output": req.num_completion_tokens, + } + return outputs def has_pending_requests(self): return len(self.requests) > 0 From e80d47ec7a63ccb7a5a656e3acea07830b06c2be Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Wed, 1 Oct 2025 06:08:07 +0000 Subject: [PATCH 047/119] clean --- atom/model_engine/engine_core.py | 8 ++++++-- atom/model_engine/llm_engine.py | 22 ++++++++++------------ atom/model_engine/scheduler.py | 3 +++ atom/model_ops/sampler.py | 6 ++++++ 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 30f9c55636..0b14d81254 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -123,12 +123,16 @@ def _process_engine_step(self): ) def _process_input_queue(self): + recv_reqs = [] while not self.input_queue.empty(): seq = self.input_queue.get_nowait() - print(f"Engine core processing sequence {seq.id} with status {seq.status}") + # print(f"Engine core processing sequence {seq.id} with status {seq.status}") if seq.status == SequenceStatus.EXIT_ENGINE: return True - self.scheduler.add(seq) + recv_reqs.append(seq) + if len(recv_reqs) > 0: + print(f"put {len(recv_reqs)} reqs to scheduler") + self.scheduler.extend(recv_reqs) return False def process_input_sockets(self, input_address: str): diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index e25ef82d14..c697be8eac 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -1,3 +1,4 @@ +import itertools import logging import time from dataclasses import fields @@ -52,15 +53,15 @@ def generate( # prompts: list[str] | list[list[int]], prompts: list[str], sampling_params: SamplingParams | list[SamplingParams], - enable_profiling: bool = False, ) -> list[str]: - # # Start profiling for all ranks if enabled - # if enable_profiling: - # self.model_runner.call("start_profiler") - if not isinstance(sampling_params, list): - sampling_params = [sampling_params] * len(prompts) - for prompt, sp in zip(prompts, sampling_params): + if isinstance(sampling_params, list): + iter_func = zip + else: + iter_func = itertools.product + sampling_params = [sampling_params] + for prompt, sp in iter_func(prompts, sampling_params): self.add_request(prompt, sp) + outputs = {} while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() @@ -68,9 +69,7 @@ def generate( seqs = self.step() outs = self.io_processor.postprocess(seqs) outputs.update(outs) - # # Stop profiling for all ranks if enabled - # if enable_profiling: - # self.model_runner.call("stop_profiler") + outputs = [outputs[seq_id] for seq_id in sorted(outputs)] return outputs @@ -108,8 +107,7 @@ def postprocess(self, reqs: List[Sequence]): outputs = {} for req in reqs: self.requests.pop(req.id) - # output_str = self.tokenizer.decode(req.completion_token_ids) - output_str = "self.tokenizer.decode(req.completion_token_ids)" + output_str = self.tokenizer.decode(req.completion_token_ids) req.leave_time = time.time() print( f"Request {req.id} finished with reason {req.leave_reason}. " diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 2dfd2e2c78..e368aaf339 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -30,6 +30,9 @@ def is_finished(self): def add(self, seq: Sequence): self.waiting.append(seq) + def extend(self, seqs: list[Sequence]): + self.waiting.extend(seqs) + def schedule(self) -> ScheduledBatchs: # prefill scheduled_seqs = [] diff --git a/atom/model_ops/sampler.py b/atom/model_ops/sampler.py index 2fa501f2f5..d8d21d2414 100644 --- a/atom/model_ops/sampler.py +++ b/atom/model_ops/sampler.py @@ -1,4 +1,5 @@ import torch +from aiter import mixed_sample from aiter.ops.triton.softmax import softmax from aiter.ops.triton.topk import topk from torch import nn @@ -15,6 +16,11 @@ def forward( logits: torch.Tensor, # (token_num, vocab_size) temperatures: torch.Tensor, # (token_num,) ) -> torch.Tensor: # (token_num,) + sampled_tokens = torch.empty( + logits.size(0), dtype=torch.int, device=logits.device + ) + mixed_sample(sampled_tokens, logits, temperatures, lambd=1.0, eps=self.eps) + return sampled_tokens logits = logits.float() return torch.where( temperatures == 0, self.greedy_sample(logits), self.random_sample(logits) From c63812fbf189327f1d8f1494c184ccd5660a8f61 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Mon, 6 Oct 2025 08:56:55 +0000 Subject: [PATCH 048/119] add async_process --- atom/config.py | 82 ++++---- atom/model_engine/async_proc.py | 246 ++++++++++++++++++++++ atom/model_engine/engine_core.py | 124 ++++++----- atom/model_engine/engine_core_mgr.py | 50 ++--- atom/model_engine/llm_engine.py | 1 + atom/model_engine/model_runner.py | 303 +++++++++++++-------------- atom/model_engine/sequence.py | 28 ++- atom/model_ops/attention.py | 16 +- atom/model_ops/embed_head.py | 6 +- atom/model_ops/linear.py | 24 ++- atom/utils/__init__.py | 133 ++++++++++-- atom/utils/backends.py | 22 +- atom/utils/context.py | 11 +- example.py | 10 +- test.py | 79 +++++++ 15 files changed, 796 insertions(+), 339 deletions(-) create mode 100644 atom/model_engine/async_proc.py create mode 100644 test.py diff --git a/atom/config.py b/atom/config.py index 1f15cc0cec..558baaea6d 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,52 +1,53 @@ +import enum +import hashlib import os -import torch import re -from dataclasses import dataclass -from transformers import AutoConfig -from typing import Optional, Any, ClassVar, Union -from dataclasses import field +from dataclasses import dataclass, field +from typing import Any, ClassVar, Optional, Union + +import torch from aiter import QuantType from aiter.utility.dtypes import d_dtypes from transformers import AutoConfig, PretrainedConfig -import enum -import hashlib CUDA_CAPTURE = True + def get_capture_status() -> bool: return CUDA_CAPTURE + def set_capture_status(capture): global CUDA_CAPTURE - CUDA_CAPTURE = capture + CUDA_CAPTURE = capture class CUDAGraphMode(enum.Enum): - """ Constants for the cudagraph mode in CompilationConfig. + """Constants for the cudagraph mode in CompilationConfig. Meanwhile, the subset enum `NONE`, `PIECEWISE` and `FULL` are also treated as concrete runtime mode for cudagraph runtime dispatching. """ + NONE = 0 PIECEWISE = 1 FULL = 2 FULL_DECODE_ONLY = (FULL, NONE) FULL_AND_PIECEWISE = (FULL, PIECEWISE) - def decode_mode(self) -> 'CUDAGraphMode': - return CUDAGraphMode(self.value[0]) if \ - self.separate_routine() else self + def decode_mode(self) -> "CUDAGraphMode": + return CUDAGraphMode(self.value[0]) if self.separate_routine() else self - def mixed_mode(self) -> 'CUDAGraphMode': - return CUDAGraphMode(self.value[1]) if \ - self.separate_routine() else self + def mixed_mode(self) -> "CUDAGraphMode": + return CUDAGraphMode(self.value[1]) if self.separate_routine() else self def requires_piecewise_compilation(self) -> bool: - return (self.decode_mode() == CUDAGraphMode.PIECEWISE - or self.mixed_mode() == CUDAGraphMode.PIECEWISE) + return ( + self.decode_mode() == CUDAGraphMode.PIECEWISE + or self.mixed_mode() == CUDAGraphMode.PIECEWISE + ) - def max_cudagraph_mode(self) -> 'CUDAGraphMode': - return CUDAGraphMode(max( - self.value)) if self.separate_routine() else self + def max_cudagraph_mode(self) -> "CUDAGraphMode": + return CUDAGraphMode(max(self.value)) if self.separate_routine() else self def has_full_cudagraphs(self) -> bool: return self.max_cudagraph_mode() == CUDAGraphMode.FULL @@ -55,7 +56,6 @@ def separate_routine(self) -> bool: return isinstance(self.value, tuple) - class CompilationLevel: # constants for the levels of the compilation process NO_COMPILATION = 0 @@ -97,7 +97,7 @@ class CompilationConfig: cache_dir: str = "" use_inductor: bool = True - + # CudaGraph compilation cudagraph_mode: Optional[CUDAGraphMode] = None """ @@ -145,7 +145,6 @@ class CompilationConfig: # splitting_ops: Optional[list[str]] = field(default_factory=list) - cudagraph_copy_inputs: bool = False """Whether to copy input tensors for cudagraph. If the caller can guarantee that the same input buffers @@ -157,8 +156,7 @@ class CompilationConfig: _attention_ops: ClassVar[list[str]] = [ "aiter.unified_attention_with_output" - - # "aiter.wrapper_fmha_v3_varlen_fwd", + # "aiter.wrapper_fmha_v3_varlen_fwd", # "vllm.unified_attention", # "vllm.unified_attention_with_output", # "vllm.mamba_mixer2", @@ -185,9 +183,10 @@ def init_with_cudagraph_sizes(self) -> None: self.compile_sizes = list(set(self.compile_sizes)) for x in self.compile_sizes: if isinstance(x, str): - assert x == "cudagraph_capture_sizes", \ - "Unrecognized size type in compile_sizes, " \ - f"expect 'cudagraph_capture_sizes', got {x}" + assert x == "cudagraph_capture_sizes", ( + "Unrecognized size type in compile_sizes, " + f"expect 'cudagraph_capture_sizes', got {x}" + ) computed_compile_sizes.extend(self.cudagraph_capture_sizes) else: assert isinstance(x, int) @@ -220,13 +219,13 @@ def compute_hash(self) -> str: factors.append(self.cuda_graph_sizes) return hashlib.sha256(str(factors).encode()).hexdigest() - def set_splitting_ops_for_v1(self): # NOTE: this function needs to be called only when level is # CompilationLevel.PIECEWISE assert self.level == CompilationLevel.PIECEWISE, ( "set_splitting_ops_for_v1 should only be called when " - "level is CompilationLevel.PIECEWISE") + "level is CompilationLevel.PIECEWISE" + ) if self.splitting_ops is None: # NOTE: When using full cudagraph, instead of setting an empty @@ -238,10 +237,13 @@ def set_splitting_ops_for_v1(self): self.splitting_ops = self._attention_ops - class QuantizationConfig(dict): def __init__( - self, quant_type=QuantType.No, quant_dtype=torch.bfloat16, is_dynamic=True, quant_name="" + self, + quant_type=QuantType.No, + quant_dtype=torch.bfloat16, + is_dynamic=True, + quant_name="", ): super().__init__() self["quant_type"] = quant_type if quant_type is not None else QuantType.No @@ -341,7 +343,8 @@ class Config: gpu_memory_utilization: float = 0.9 tensor_parallel_size: int = 1 enforce_eager: bool = False - hf_config: AutoConfig | None = None + hf_config: PretrainedConfig = field(init=False) + bos_token_id: int = -1 eos_token_id: int = -1 kvcache_block_size: int = 16 num_kvcache_blocks: int = -1 @@ -350,12 +353,13 @@ class Config: port: int = 8006 torch_profiler_dir: str | None = os.getenv("ATOM_TORCH_PROFILER_DIR", None) compilation_config: CompilationConfig = field(default_factory=CompilationConfig) - quant_config: QuantizationConfig = field(default_factory=lambda: QuantizationConfig()) + quant_config: QuantizationConfig = field( + default_factory=lambda: QuantizationConfig() + ) asyncio_mode: bool = False master_addr: str = "127.0.0.1" graph_bs: Optional[list[int]] = None - def _set_cudagraph_sizes(self): if self.compilation_config.cudagraph_capture_sizes: self.graph_bs = self.compilation_config.cudagraph_capture_sizes @@ -387,7 +391,6 @@ def __post_init__(self): self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE self.compilation_config.init_with_cudagraph_sizes() - def compute_hash(self) -> str: """ WARNING: Whenever a new field is added to this config, @@ -412,6 +415,7 @@ def compute_hash(self) -> str: factors.append(vllm_factors) - hash_str = hashlib.md5(str(factors).encode(), - usedforsecurity=False).hexdigest()[:10] - return hash_str \ No newline at end of file + hash_str = hashlib.md5( + str(factors).encode(), usedforsecurity=False + ).hexdigest()[:10] + return hash_str diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py new file mode 100644 index 0000000000..f61b34f6cd --- /dev/null +++ b/atom/model_engine/async_proc.py @@ -0,0 +1,246 @@ +import logging +import multiprocessing +import pickle +import queue +import signal +import threading +import weakref +from contextlib import ExitStack +from multiprocessing.shared_memory import SharedMemory +from multiprocessing.synchronize import Event +from threading import Thread +from typing import List + +import zmq +import zmq.asyncio + +from atom.utils import ( + get_mp_context, + get_open_zmq_ipc_path, + init_exit_handler, + make_zmq_socket, + resolve_obj_by_qualname, + shutdown_all_processes, +) + +logger = logging.getLogger("atom") + + +class AsyncIOProc: + + def __init__( + self, + label: str, + label_shm: str, + io_addrs: tuple[str, str], + sync_event: Event, + # runner class and its args + runner_qualname: str, # atom.model_engine.model_runner.ModelRunner + *args, + **kwargs, + ): + self.label = f"AsyncIOProc({label})" + self.sync_event = sync_event + self.io_addrs = io_addrs + self.io_queues = queue.Queue(), queue.Queue() + self.io_threads = [] + self.shm = SharedMemory(name=f"atom_{label_shm}_shm") + init_exit_handler(self) + + # make sure exit handler is set before runner is created + runner_class = resolve_obj_by_qualname(runner_qualname) # type: ignore + self.runners = [] + self.runners.append(runner_class(*args, **kwargs)) + for addr, q, func in zip( + self.io_addrs, + self.io_queues, + [self.recv_input_from_socket, self.send_output_to_socket], + ): + if addr is None: + continue + t = threading.Thread(target=func, args=(addr, q), daemon=True) + t.start() + self.io_threads.append(t) + self.busy_loop() + + def exit(self): + if not self.still_running: + return + self.still_running = False + logger.debug(f"{self.label}: Shutting down runner...") + for el in self.runners: + el.exit() + self.shm.close() + for t in self.io_threads: + t.join(timeout=0.5) + + def recv_input_from_socket(self, addr: str, input_queue: queue.Queue): + with ExitStack() as stack, zmq.Context() as ctx: + socket = stack.enter_context( + make_zmq_socket(ctx, addr, zmq.DEALER, bind=False) + ) + poller = zmq.Poller() + # Send initial message to input socket - this is required + # before the front-end ROUTER socket can send input messages + # back to us. + socket.send(b"") + poller.register(socket, zmq.POLLIN) + logger.debug(f"{self.label}: input socket connected") + + while self.still_running: + for socket, _ in poller.poll(timeout=1000): + serialized_obj = socket.recv(copy=False) + input = pickle.loads(serialized_obj) + input_queue.put_nowait(input) + + def send_output_to_socket(self, addr: str, output_queue: queue.Queue): + with ExitStack() as stack, zmq.Context() as ctx: + socket = stack.enter_context( + make_zmq_socket(ctx, addr, zmq.PUSH, linger=4000) + ) + logger.debug(f"{self.label}: output socket connected") + + while True: + result = output_queue.get() + serialized_obj = pickle.dumps(result) + socket.send(serialized_obj) + + def busy_loop(self): + while True: + func_name, args = self.get_func() + for runner in self.runners: + func = getattr(runner, func_name, None) + if func is None: + continue + out = func(*args) + if out is not None: + self.io_queues[1].put_nowait(out) + if func_name == "exit": + break + logger.debug(f"{self.label}: exit busy_loop...") + + def get_func(self): + self.sync_event.wait() + n = int.from_bytes(self.shm.buf[0:4], "little") + method_name, *args = pickle.loads(self.shm.buf[4 : n + 4]) + self.sync_event.clear() + return method_name, args + + +class AsyncIOProcManager: + + def __init__(self, finalizer, proc_num: int, runner: str, *args): + self.parent_finalizer = finalizer + io_addrs = [get_open_zmq_ipc_path(), get_open_zmq_ipc_path()] + self.procs = [] + self.sync_events = [] + ctx = get_mp_context() + self.mp_ctx = ctx + self.runner_label = runner.split(".")[-1] + self.label = f"AsyncIOProcManager({self.runner_label})" + self.create_shared_memory() + self.still_running = True + init_exit_handler(self) + for i in range(proc_num): + label = f"ModelRunner{i}/{proc_num}" + addrs = ( + io_addrs if i == 0 else [io_addrs[0], None] + ) # only get output from rank 0 + sync_event = ctx.Event() + process = ctx.Process( + target=AsyncIOProc, + name=label, + args=(label, self.runner_label, addrs, sync_event, runner, i, *args), + ) + process.start() + self.procs.append(process) + self.sync_events.append(sync_event) + + self.zmq_ctx = zmq.Context(io_threads=2) + self.outputs_queue = queue.Queue() + self.output_thread = threading.Thread( + target=self.process_output_sockets, + name=f"{self.label}_output_thread", + args=(io_addrs[1],), + daemon=True, + ) + self.output_thread.start() + self.monitor_procs() + + def create_shared_memory(self): + name = f"atom_{self.runner_label}_shm" + try: + self.shm = SharedMemory(name=name, create=True, size=2**20) + except FileExistsError: + existing_shm = SharedMemory(name=name) + existing_shm.close() + existing_shm.unlink() + self.shm = SharedMemory(name=name, create=True, size=2**20) + + def exit(self): + if not self.still_running: + return + self.still_running = False + # 1. kill all runners + logger.info(f"{self.label}: shutdown all runners...") + shutdown_all_processes(self.procs, allowed_seconds=1) + self.shm.unlink() + self.procs = [] + self.output_thread.join() + logger.info(f"{self.label}: All runners are shutdown.") + # 2. put a None to unblock call_func + self.outputs_queue.put_nowait(SystemExit()) + # 3. call parent finalizer + self.parent_finalizer() + + def process_output_sockets(self, output_address: str): + output_socket = make_zmq_socket(self.zmq_ctx, output_address, zmq.PULL) + try: + poller = zmq.Poller() + poller.register(output_socket, zmq.POLLIN) + while self.still_running: + socks = poller.poll(timeout=1000) + if not socks: + continue + + obj = output_socket.recv(copy=False) + obj = pickle.loads(obj) # type: ignore + self.outputs_queue.put_nowait(obj) + finally: + # Close sockets. + output_socket.close(linger=0) + logger.debug(f"{self.label}: output thread exit") + + def call_func(self, func_name: str, *args, wait_out: bool = False): + logger.debug(f"{self.label}: call_func {func_name} {args}") + n = pickle.dumps((func_name, *args)) + self.shm.buf[0:4] = len(n).to_bytes(4, "little") + self.shm.buf[4 : len(n) + 4] = n + for ev in self.sync_events: + ev.set() + if wait_out: + ret = self.outputs_queue.get() + if isinstance(ret, SystemExit): + raise ret + return ret + + def monitor_procs(self): + self_ref = weakref.ref(self) + procs = self.procs + self.keep_monitoring = True + + def monitor_engine_cores(): + sentinels = [proc.sentinel for proc in procs] + died = multiprocessing.connection.wait(sentinels) + _self = self_ref() + if not _self or not _self.keep_monitoring: + return + proc_name = next(proc.name for proc in procs if proc.sentinel == died[0]) + logger.error( + f"{self.label}: [{proc_name}] proc died unexpectedly, shutting down.", + ) + _self.exit() + + Thread( + target=monitor_engine_cores, daemon=True, name=f"{self.runner_label}Monitor" + ).start() diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 0b14d81254..33ec5fc54c 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -2,31 +2,33 @@ import logging import pickle import queue +import signal import threading import weakref from contextlib import ExitStack +from multiprocessing.shared_memory import SharedMemory from typing import Any, List, Optional import zmq import zmq.asyncio from atom.config import Config -from atom.model_engine.sequence import Sequence, SequenceStatus, get_exit_sequence - -logger = logging.getLogger("atom") - - +from atom.model_engine.async_proc import AsyncIOProcManager from atom.model_engine.model_runner import ModelRunner from atom.model_engine.scheduler import Scheduler +from atom.model_engine.sequence import Sequence, SequenceStatus, get_exit_sequence from atom.utils import ( close_sockets, get_engine_client_zmq_addr, get_mp_context, + init_exit_handler, make_zmq_socket, shutdown_all_processes, zmq_socket_ctx, ) +logger = logging.getLogger("atom") + class EngineCoreRequestType(enum.Enum): """ @@ -46,29 +48,11 @@ class EngineCoreRequestType(enum.Enum): class EngineCore: def __init__(self, config: Config, input_address: str, output_address: str): - self.ps = [] - self.events = [] - ctx = get_mp_context() - if config.tensor_parallel_size > 1: - for i in range(1, config.tensor_parallel_size): - event = ctx.Event() - process = ctx.Process( - target=ModelRunner, name=f"ModelRunner{i}", args=(config, i, event) - ) - process.start() - self.ps.append(process) - self.events.append(event) - self.model_runner = ModelRunner( - config, 0, self.events if config.tensor_parallel_size > 1 else [] - ) - logger.info("Engine core model runner loaded") - self.scheduler = Scheduler(config) + self.label = "Engine Core" self.input_queue = queue.Queue[Sequence]() self.output_queue = queue.Queue[List[Sequence]]() self.input_address = input_address self.output_address = output_address - logger.info(f"Engine core input address: {self.input_address}") - logger.info(f"Engine core output address: {self.output_address}") self.input_thread = threading.Thread( target=self.process_input_sockets, args=(self.input_address,), daemon=True ) @@ -79,14 +63,50 @@ def __init__(self, config: Config, input_address: str, output_address: str): self.output_thread.start() self.profile_enbaled = config.torch_profiler_dir is not None + init_exit_handler(self) + + # Initialize model runner processes + try: + good = False + self.runner_mgr = AsyncIOProcManager( + self._finalizer, + config.tensor_parallel_size, + "atom.model_engine.model_runner.ModelRunner", + config, + ) + num_blocks = self.runner_mgr.call_func("get_num_blocks", wait_out=True) + ret = self.runner_mgr.call_func( + "allocate_kv_cache", num_blocks, wait_out=True + ) + assert ret, "Failed to allocate kv cache" + + config.num_kvcache_blocks = num_blocks + if not config.enforce_eager: + cap_cost = self.runner_mgr.call_func("capture_cudagraph", wait_out=True) + logger.info(f"{self.label}: cudagraph capture cost: {cap_cost} seconds") + good = True + finally: + logger.info( + f"{self.label}: load model runner {'success' if good else 'failed'}" + ) + if not good: + self._finalizer() + + self.scheduler = Scheduler(config) def exit(self): + if not self.still_running: + return + self.still_running = False + self.runner_mgr.call_func("exit") + self.runner_mgr.keep_monitoring = False self._send_engine_dead() - self.model_runner.call("exit") - del self.model_runner - for p in self.ps: - p.join() - logger.info("Engine core model runner exit") + logger.debug(f"{self.label}: model runner exit") + + def _send_engine_dead(self): + logger.debug(f"{self.label}: send SHUTDOWN request") + self.output_queue.put_nowait([get_exit_sequence()]) + self.output_thread.join(timeout=5.0) @staticmethod def run_engine(config: Config, input_address: str, output_address: str): @@ -96,19 +116,17 @@ def run_engine(config: Config, input_address: str, output_address: str): engine.start_profiler() engine.busy_loop() engine.stop_profiler() + except Exception as e: + logger.error(f"run_engine: exception: {e}", exc_info=True) + raise e finally: if engine is not None: engine.exit() - def _send_engine_dead(self): - logger.info("Send engine core dead signal") - self.output_queue.put_nowait([get_exit_sequence()]) - self.output_thread.join(timeout=5.0) - def busy_loop(self): shutdown = False while True: - shutdown = shutdown or self._process_input_queue() + shutdown = shutdown or self.pull_and_process_input_queue() if not self.scheduler.is_finished(): self._process_engine_step() elif shutdown: @@ -116,22 +134,22 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() - token_ids = self.model_runner.call("run", scheduled_batchs) - self.scheduler.postprocess(scheduled_batchs.seqs, token_ids) + out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) + self.scheduler.postprocess(scheduled_batchs.seqs, out) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] ) - def _process_input_queue(self): + def pull_and_process_input_queue(self): recv_reqs = [] while not self.input_queue.empty(): seq = self.input_queue.get_nowait() - # print(f"Engine core processing sequence {seq.id} with status {seq.status}") if seq.status == SequenceStatus.EXIT_ENGINE: + logger.debug(f"{self.label}: input_queue get {seq.status}") return True recv_reqs.append(seq) if len(recv_reqs) > 0: - print(f"put {len(recv_reqs)} reqs to scheduler") + logger.info(f"{self.label}: put {len(recv_reqs)} reqs to scheduler") self.scheduler.extend(recv_reqs) return False @@ -147,20 +165,23 @@ def process_input_sockets(self, input_address: str): # back to us. input_socket.send(b"") poller.register(input_socket, zmq.POLLIN) - logger.info("Engine core input socket connected") + logger.debug(f"{self.label}: input socket connected") + alive = True - while True: + while alive: for input_socket, _ in poller.poll(): # (RequestType, RequestData) serialized_obj = input_socket.recv(copy=False) request_type, req = pickle.loads(serialized_obj) if request_type == EngineCoreRequestType.ADD: - logger.info(f"Engine core input: {request_type} {req.id}") + logger.debug(f"{self.label}: input get {request_type} {req.id}") self.input_queue.put_nowait(req) elif request_type == EngineCoreRequestType.SHUTDOWN: - logger.info("Engine core input thread shutdown") + logger.debug(f"{self.label}: input get {request_type}") self.input_queue.put_nowait(get_exit_sequence()) - break + alive = False + reason = request_type + logger.debug(f"{self.label}: input thread exit due to {reason}") def process_output_sockets(self, output_address: str): """Output socket IO thread.""" @@ -168,7 +189,7 @@ def process_output_sockets(self, output_address: str): socket = stack.enter_context( make_zmq_socket(ctx, output_address, zmq.PUSH, linger=4000) ) - logger.info("Engine core output socket connected") + logger.debug(f"{self.label}: output socket connected") while True: seqs = self.output_queue.get() @@ -178,18 +199,21 @@ def process_output_sockets(self, output_address: str): serialized_obj = pickle.dumps((EngineCoreRequestType.ADD, valid_seqs)) socket.send(serialized_obj) num_valid = len(valid_seqs) - logger.info(f"Engine core output: {num_valid} sequences") + if num_valid > 0: + logger.info(f"{self.label}: output send {num_valid} reqs") if len(valid_seqs) != len(seqs): socket.send(pickle.dumps((EngineCoreRequestType.SHUTDOWN, None))) - logger.info("Engine core output thread closed") + logger.debug( + f"{self.label}: output send {EngineCoreRequestType.SHUTDOWN}" + ) break def start_profiler(self): if self.profile_enbaled: - self.model_runner.call("start_profiler") + self.runner_mgr.call_func("start_profiler") def stop_profiler(self): if self.profile_enbaled: print("Stopping profiler...") - self.model_runner.call("stop_profiler") + self.runner_mgr.call_func("stop_profiler") print("Profiler stopped.") diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index e647fefdcc..81e8f5f429 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -2,6 +2,8 @@ import multiprocessing import pickle import queue +import signal +import sys import weakref from threading import Thread from typing import List @@ -24,11 +26,11 @@ class CoreManager: def __init__(self, config: Config): + self.label = "Engine Core Mgr" sync_ctx = zmq.Context(io_threads=2) self.ctx = zmq.asyncio.Context(sync_ctx) if config.asyncio_mode else sync_ctx self.outputs_queue = queue.Queue[List[Sequence]]() engine_core_process, addresses = launch_engine_core(config) - logger.info("Engine core inited") self.engine_core_process = engine_core_process input_address = addresses["input_address"] @@ -37,8 +39,7 @@ def __init__(self, config: Config): self.ctx, input_address, zmq.ROUTER, bind=True ) identity, _ = self.input_socket.recv_multipart() - logger.info(f"Core engine input socket identity: {identity}") - self.core_engine_identity = identity + self.engine_core_identity = identity self.output_socket = make_zmq_socket(self.ctx, output_address, zmq.PULL) shutdown_path = get_open_zmq_inproc_path() @@ -52,22 +53,28 @@ def process_outputs_socket(): poller = zmq.Poller() poller.register(shutdown_socket, zmq.POLLIN) poller.register(self.output_socket, zmq.POLLIN) - logger.info("Core client output thread started") + logger.debug(f"{self.label}: output thread started") while True: socks = poller.poll() if not socks: continue if len(socks) == 2 or socks[0][0] == shutdown_socket: # shutdown signal, exit thread. - logger.info("Core client output thread receive shutdown signal") + logger.debug( + f"{self.label}: output thread receive shutdown signal" + ) break obj = self.output_socket.recv(copy=False) request_type, seqs = pickle.loads(obj) if request_type == EngineCoreRequestType.SHUTDOWN: - logger.info("Core client output thread receive shutdown signal") - self.engine_core_process.join() - self.engine_core_process = None + logger.debug( + f"{self.label}: output thread receive SHUTDOWN request" + ) + self._shutdown_engine_core() + self.outputs_queue.put_nowait( + SystemExit(f"{self.label}: shutdown") + ) break elif request_type == EngineCoreRequestType.ADD: # logger.info(f"Engine core output sequence id: {seq.id}") @@ -86,46 +93,36 @@ def process_outputs_socket(): self._finalizer = weakref.finalize(self, self.close) def close(self): - self._shut_down_core_engine() + self._shutdown_engine_core() self.input_socket.close() - logger.info("Core client input socket closed") if self.shutdown_path and self.output_queue_thread: with self.ctx.socket(zmq.PAIR) as shutdown_sender: shutdown_sender.connect(self.shutdown_path) # Send shutdown signal. shutdown_sender.send(b"") self.output_queue_thread.join() - logger.info("Core client output thread is shut down") def add_request(self, seq: Sequence): - logger.info(f"Add request to engine core, sequence id: {seq.id}") + logger.debug(f"{self.label}: Add request, sequence id: {seq.id}") self.input_socket.send_multipart( - [self.core_engine_identity, pickle.dumps((EngineCoreRequestType.ADD, seq))], + [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.ADD, seq))], copy=False, ) - def _shut_down_core_engine(self): - logger.info("Shutdown core engine") - if self.engine_core_process is not None: + def _shutdown_engine_core(self): + if self.engine_core_process is not None and self.engine_core_process.is_alive(): self.input_socket.send_multipart( [ - self.core_engine_identity, + self.engine_core_identity, pickle.dumps((EngineCoreRequestType.SHUTDOWN, None)), ], copy=False, ) - self.engine_core_process.join() - - # def _shut_down_core_engine(self): - # logger.info("Shutdown core engine") - # if self.engine_core_process is not None and self.engine_core_process.is_alive(): - # shutdown_all_processes([self.engine_core_process]) def get_output(self) -> List[Sequence]: seqs = self.outputs_queue.get() - # logger.info(f"Get output from engine core, sequence id: {outputs.id}") - # if isinstance(outputs, Exception): - # raise outputs from None + if isinstance(seqs, BaseException): + raise seqs return seqs def is_rest(self): @@ -151,5 +148,4 @@ def launch_engine_core(config: Config): }, ) process.start() - logger.info("Engine core process started") return process, {"input_address": input_address, "output_address": output_address} diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index c697be8eac..79cabebde6 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -23,6 +23,7 @@ def __init__(self, model, **kwargs): config_kwargs = {k: v for k, v in kwargs.items() if k in config_fields} config = Config(model, **config_kwargs) self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True) + config.bos_token_id = self.tokenizer.bos_token_id config.eos_token_id = self.tokenizer.eos_token_id self.rquest_ids = set() diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index f34849674d..c58aad9a0f 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -1,13 +1,12 @@ import logging import os -import pickle import signal +import time import weakref -from multiprocessing.shared_memory import SharedMemory -from multiprocessing.synchronize import Event +from typing import Optional +import numpy as np import torch -import torch.distributed as dist import torch.profiler as torch_profiler import tqdm from aiter import destroy_dist_env, dtypes, init_dist_env @@ -21,6 +20,7 @@ from atom.models.llama import LlamaForCausalLM from atom.models.mixtral import MixtralForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM +from atom.utils import CpuGpuBuffer from atom.utils.context import get_context, reset_context, set_context logger = logging.getLogger("atom") @@ -35,14 +35,14 @@ class ModelRunner: - def __init__(self, config: Config, rank: int, event: Event | list[Event]): + def __init__(self, rank: int, config: Config): self.config = config hf_config = config.hf_config self.block_size = config.kvcache_block_size self.enforce_eager = config.enforce_eager self.world_size = config.tensor_parallel_size self.rank = rank - self.event = event + self.label = f"Model Runner{rank}/{self.world_size}" # Initialize profiler for this rank self.profiler = None @@ -52,89 +52,53 @@ def __init__(self, config: Config, rank: int, event: Event | list[Event]): self.profiler_dir = os.path.join(config.torch_profiler_dir, f"rank_{rank}") os.makedirs(self.profiler_dir, exist_ok=True) + self.graph_bs = [0] # for eager fallback + device = torch.device(f"cuda:{rank}") torch.cuda.set_device(device) + self.device = device os.environ["MASTER_ADDR"] = self.config.master_addr os.environ["MASTER_PORT"] = str(self.config.port) init_dist_env(self.world_size, rankID=rank) + self.init_exit_handler() default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") - self.graph_bs = [0] # for eager fallback - + self.sampler = Sampler() self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) load_model(self.model, config.model) - self.sampler = Sampler() + logger.info(f"Model loaded: {config.model}") self.warmup_model() - self.allocate_kv_cache() + logger.info(f"Model warmup done: {config.model}") self.allocate_decode_vars() - if not self.enforce_eager: - self.capture_cudagraph() torch.set_default_device("cpu") torch.set_default_dtype(default_dtype) + if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") - if self.world_size > 1: - if rank == 0: - self.shm = SharedMemory(name="atom", create=True, size=2**20) - dist.barrier() - else: - dist.barrier() - self.shm = SharedMemory(name="atom") - self.loop() + def init_exit_handler(self): + logger.info(f"Initializing ModelRunner for rank {self.rank}") + self.still_running = True + self._finalizer = weakref.finalize(self, self.exit) def signal_handler(signum, frame): - raise SystemExit( - f"Rank{self.rank}/{self.world_size}: received signal {signum}, exiting..." - ) + sig_name = signal.Signals(signum).name + msg = f"{self.label}: received signal {signum} ({sig_name}), exiting..." + logger.warning(msg) + self._finalizer() signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) - logger.info(f"Initializing ModelRunner for rank {rank}") - self._finalizer = weakref.finalize(self, self.exit) - def exit(self, msg=None): - logger.info(f"Exiting ModelRunner for rank {self.rank}/{self.world_size} {msg}") - if dist.is_initialized() and self.world_size > 1: - self.shm.close() - dist.barrier() - if self.rank == 0: - self.shm.unlink() + def exit(self): + if not self.still_running: + return + self.still_running = False if not self.enforce_eager: - self.graphs = self.graph_pool = None - torch.cuda.synchronize() + self.graphs = self.graph_pool = None # type: ignore destroy_dist_env() - - def loop(self): - while True: - method_name, args = self.read_shm() - self.call(method_name, *args) - if method_name == "exit": - break - - def read_shm(self): - assert self.world_size > 1 and self.rank - self.event.wait() - n = int.from_bytes(self.shm.buf[0:4], "little") - method_name, *args = pickle.loads(self.shm.buf[4 : n + 4]) - self.event.clear() - return method_name, args - - def write_shm(self, method_name, *args): - assert self.world_size > 1 and not self.rank - data = pickle.dumps([method_name, *args]) - n = len(data) - self.shm.buf[0:4] = n.to_bytes(4, "little") - self.shm.buf[4 : n + 4] = data - for event in self.event: - event.set() - - def call(self, method_name, *args): - if self.world_size > 1 and self.rank == 0: - self.write_shm(method_name, *args) - method = getattr(self, method_name, None) - return method(*args) + return True def start_profiler(self): """Start profiling for this rank""" @@ -174,7 +138,7 @@ def warmup_model(self): for _ in range(num_seqs) ] dummy_batch = ScheduledBatchs(seqs, True, False) - self.run(dummy_batch) + self.forward(dummy_batch) torch.cuda.empty_cache() def allocate_decode_vars(self): @@ -184,15 +148,18 @@ def allocate_decode_vars(self): max_bs = min(self.config.max_num_seqs, 512) max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size self.decode_vars = { - "input_ids": torch.zeros(max_bs, dtype=torch.int64), - "positions": torch.zeros(max_bs, dtype=torch.int64), - "slot_mapping": torch.zeros(max_bs, dtype=torch.int64), - "context_lens": torch.zeros(max_bs, dtype=torch.int32), - "block_tables": torch.zeros(max_bs, max_num_blocks, dtype=torch.int32), + "input_ids": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), + "positions": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), + "slot_mapping": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), + "context_lens": CpuGpuBuffer(max_bs, dtype=torch.int32, device=self.device), + "block_tables": CpuGpuBuffer( + max_bs, max_num_blocks, dtype=torch.int32, device=self.device + ), "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), } - def allocate_kv_cache(self): + def get_num_blocks(self): + torch.set_default_device("cuda") config = self.config hf_config = config.hf_config if not hasattr(hf_config, "head_dim") or hf_config.head_dim is None: @@ -201,6 +168,7 @@ def allocate_kv_cache(self): used = total - free peak = torch.cuda.memory_stats()["allocated_bytes.all.peak"] current = torch.cuda.memory_stats()["allocated_bytes.all.current"] + torch.set_default_device("cpu") num_kv_heads = hf_config.num_key_value_heads // self.world_size block_bytes = ( 2 @@ -210,18 +178,27 @@ def allocate_kv_cache(self): * hf_config.head_dim * hf_config.torch_dtype.itemsize ) - config.num_kvcache_blocks = ( + num_kvcache_blocks = ( int(total * config.gpu_memory_utilization - used - peak + current) // block_bytes ) - assert config.num_kvcache_blocks > 0, f"need at least {block_bytes} KV cache" + assert num_kvcache_blocks > 0, f"need at least {block_bytes} KV cache" + return num_kvcache_blocks + + def allocate_kv_cache(self, num_kvcache_blocks): + config = self.config + config.num_kvcache_blocks = num_kvcache_blocks + hf_config = config.hf_config + num_kv_heads = hf_config.num_key_value_heads // self.world_size self.kv_cache = torch.zeros( - 2, - hf_config.num_hidden_layers, - config.num_kvcache_blocks, - self.block_size, - num_kv_heads, - hf_config.head_dim, + ( + 2, + hf_config.num_hidden_layers, + config.num_kvcache_blocks, + self.block_size, + num_kv_heads, + hf_config.head_dim, + ), dtype=dtypes.d_dtypes[config.kv_cache_dtype], device="cuda", ) @@ -261,20 +238,18 @@ def allocate_kv_cache(self): k_cache=module.k_cache, v_cache=module.v_cache, k_scale=module.k_scale, - v_scale=module.v_scale + v_scale=module.v_scale, ) set_forward_context(module.layer_num, attention_metadata) layer_id += 1 + return True def prepare_block_tables(self, seqs: list[Sequence]): - max_len = max(len(seq.block_table) for seq in seqs) - block_tables = [ - seq.block_table + [-1] * (max_len - len(seq.block_table)) for seq in seqs - ] - block_tables = torch.tensor( - block_tables, dtype=torch.int32, pin_memory=True - ).cuda(non_blocking=True) + lens = [len(seq.block_table) for seq in seqs] + block_tables = np.empty((len(seqs), max(lens)), dtype=np.int32) + for i, seq in enumerate(seqs): + block_tables[i, : lens[i]] = seq.block_table return block_tables def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): @@ -307,36 +282,27 @@ def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): slot_mapping.extend(list(range(start, end))) if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True).cuda( - non_blocking=True - ) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True).cuda( - non_blocking=True - ) - cu_seqlens_q = torch.tensor( - cu_seqlens_q, dtype=torch.int32, pin_memory=True - ).cuda(non_blocking=True) - cu_seqlens_k = torch.tensor( - cu_seqlens_k, dtype=torch.int32, pin_memory=True - ).cuda(non_blocking=True) - slot_mapping = torch.tensor( - slot_mapping, dtype=torch.int64, pin_memory=True - ).cuda(non_blocking=True) + block_tables = torch.from_numpy(block_tables).cuda(non_blocking=True) + input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) + positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) + cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) min_seqlen_q = 0 dropout_p = 0.0 set_context( True, len(scheduled_batchs.seqs), - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, + cu_seqlens_q=cu_seqlens_q.cuda(non_blocking=True), + cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping, + slot_mapping=slot_mapping.cuda(non_blocking=True), block_tables=block_tables, dropout_p=dropout_p, ) - return input_ids, positions + return input_ids.cuda(non_blocking=True), positions.cuda(non_blocking=True) def prepare_decode(self, scheduled_batchs: ScheduledBatchs): bs = len(scheduled_batchs.seqs) @@ -348,70 +314,85 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): positions = [] slot_mapping = [] context_lens = [] - cu_seqlens_q = [0] - cu_seqlens_k = [0] - max_seqlen_q = 1 - max_seqlen_k = 0 - min_seqlen_q = 0 + # cu_seqlens_q = [0] + # cu_seqlens_k = [0] + # max_seqlen_q = 1 + # max_seqlen_k = 0 + # min_seqlen_q = 0 dropout_p = 0.0 - for seq in scheduled_batchs.seqs: - current_seq_len = len(seq) - cu_seqlens_q.append(cu_seqlens_q[-1] + 1) - - cu_seqlens_k.append(cu_seqlens_k[-1] + current_seq_len) - max_seqlen_k = max(current_seq_len, max_seqlen_k) - - input_ids.append(seq.last_token) - positions.append(len(seq)) - context_lens.append(len(seq)) - slot_mapping.append( - seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 - ) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) - context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True) + context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] + positions = np.array(context_lens, dtype=np.int64) + context_lens = np.array(context_lens, dtype=np.int32) + input_ids = [seq.last_token for seq in scheduled_batchs.seqs] + input_ids = np.array(input_ids, dtype=np.int64) + slot_mapping = [ + seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 + for seq in scheduled_batchs.seqs + ] + # cu_seqlens_k = np.cumsum(context_lens, dtype=np.int32) + # for seq in scheduled_batchs.seqs: + # # current_seq_len = seq.num_tokens + # # cu_seqlens_q.append(cu_seqlens_q[-1] + 1) + + # # cu_seqlens_k.append(cu_seqlens_k[-1] + current_seq_len) + # # max_seqlen_k = max(current_seq_len, max_seqlen_k) + + # # input_ids.append(seq.last_token) + # # positions.append(current_seq_len) + # # context_lens.append(current_seq_len) + # slot_mapping.append( + # seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 + # ) + slot_mapping.extend([-1] * (graph_bs - bs)) + slot_mapping = np.array(slot_mapping, dtype=np.int64) + # max_seqlen_k = max(context_lens) + # input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) + # positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) + # slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) + # context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True) block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) - cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - - decode_vars = self.decode_vars - decode_vars["slot_mapping"][bs : graph_bs + 1] = -1 - decode_vars["slot_mapping"][:bs].copy_(slot_mapping, non_blocking=True) - decode_vars["input_ids"][:bs].copy_(input_ids, non_blocking=True) - decode_vars["positions"][:bs].copy_(positions, non_blocking=True) - decode_vars["context_lens"][:bs].copy_(context_lens, non_blocking=True) - decode_vars["block_tables"][:bs, : block_tables.size(1)].copy_( - block_tables, non_blocking=True - ) + # cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + # cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) + + var = self.decode_vars + var["slot_mapping"].np[:graph_bs] = slot_mapping + var["input_ids"].np[:bs] = input_ids + var["positions"].np[:bs] = positions + var["context_lens"].np[:bs] = context_lens + var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables + for el in [ + "slot_mapping", + "input_ids", + "positions", + "context_lens", + "block_tables", + ]: + var[el].copy_to_gpu() set_context( False, batch_size=bs, graph_bs=graph_bs, - cu_seqlens_q=cu_seqlens_q, - cu_seqlens_k=cu_seqlens_k, - max_seqlen_q=max_seqlen_q, - max_seqlen_k=max_seqlen_k, - min_seqlen_q=min_seqlen_q, - slot_mapping=decode_vars["slot_mapping"][:bs], - context_lens=decode_vars["context_lens"][:bs], - block_tables=decode_vars["block_tables"][:bs, : block_tables.size(1)], + # cu_seqlens_q=cu_seqlens_q, + # cu_seqlens_k=cu_seqlens_k, + # max_seqlen_q=max_seqlen_q, + # max_seqlen_k=max_seqlen_k, + # min_seqlen_q=min_seqlen_q, + slot_mapping=var["slot_mapping"].gpu[:bs], + context_lens=var["context_lens"].gpu[:bs], + block_tables=var["block_tables"].gpu[:bs, : block_tables.shape[1]], dropout_p=dropout_p, ) - return decode_vars["input_ids"][:bs], decode_vars["positions"][:bs] + return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def prepare_sample(self, seqs: list[Sequence]): - temperatures = [] - for seq in seqs: - temperatures.append(seq.temperature) + temperatures = [seq.temperature for seq in seqs] temperatures = torch.tensor( temperatures, dtype=torch.float32, pin_memory=True ).cuda(non_blocking=True) return temperatures - @torch.inference_mode() def prepare_model(self, scheduled_batchs: ScheduledBatchs): seqs = scheduled_batchs.seqs is_prefill = scheduled_batchs.is_prefill @@ -420,7 +401,6 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): temperatures = self.prepare_sample(seqs) if self.rank == 0 else None return input_ids, positions, temperatures - @torch.inference_mode() def run_model( self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool ): @@ -434,7 +414,8 @@ def run_model( self.graphs[graph_bs].replay() return self.model.compute_logits(self.decode_vars["outputs"][:bs]) - def run(self, scheduled_batchs: ScheduledBatchs) -> list[int]: + @torch.inference_mode() + def forward(self, scheduled_batchs: ScheduledBatchs) -> Optional[list[int]]: input_ids, positions, temperatures = self.prepare_model(scheduled_batchs) logits = self.run_model(input_ids, positions, scheduled_batchs.is_prefill) token_ids = ( @@ -445,11 +426,12 @@ def run(self, scheduled_batchs: ScheduledBatchs) -> list[int]: @torch.inference_mode() def capture_cudagraph(self): - input_ids = self.decode_vars["input_ids"] - positions = self.decode_vars["positions"] - slot_mapping = self.decode_vars["slot_mapping"] - context_lens = self.decode_vars["context_lens"] - block_tables = self.decode_vars["block_tables"] + start_time = time.time() + input_ids = self.decode_vars["input_ids"].gpu + positions = self.decode_vars["positions"].gpu + slot_mapping = self.decode_vars["slot_mapping"].gpu + context_lens = self.decode_vars["context_lens"].gpu + block_tables = self.decode_vars["block_tables"].gpu outputs = self.decode_vars["outputs"] # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) @@ -497,3 +479,4 @@ def capture_cudagraph(self): torch.cuda.synchronize() reset_context() self.graph_bs.sort(reverse=False) + return time.time() - start_time diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index b1c48f3da6..3e21394364 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -13,7 +13,7 @@ class SequenceStatus(Enum): def get_exit_sequence(): - exit_seq = Sequence([-1], 0) + exit_seq = Sequence([-1], 1) exit_seq.status = SequenceStatus.EXIT_ENGINE return exit_seq @@ -24,6 +24,7 @@ class Sequence: def __init__( self, token_ids: list[int], block_size: int, sampling_params=SamplingParams() ): + self.block_size = block_size self.id = next(Sequence.counter) self.status = SequenceStatus.WAITING self.token_ids = copy(token_ids) @@ -35,7 +36,6 @@ def __init__( self.temperature = sampling_params.temperature self.max_tokens = sampling_params.max_tokens self.ignore_eos = sampling_params.ignore_eos - self.block_size = block_size # statistics fields self.arrive_time = 0.0 @@ -43,11 +43,21 @@ def __init__( self.leave_reason = "" def __len__(self): - return self.num_tokens + return self._num_tokens def __getitem__(self, key): return self.token_ids[key] + @property + def num_tokens(self): + return self._num_tokens + + @num_tokens.setter + def num_tokens(self, value): + self._num_tokens = value + self.num_blocks = (value + self.block_size - 1) // self.block_size + self.last_block_num_tokens = value % self.block_size + @property def is_finished(self): return self.status == SequenceStatus.FINISHED @@ -68,13 +78,13 @@ def completion_token_ids(self): def num_cached_blocks(self): return self.num_cached_tokens // self.block_size - @property - def num_blocks(self): - return (self.num_tokens + self.block_size - 1) // self.block_size + # @property + # def num_blocks(self): + # return (self.num_tokens + self.block_size - 1) // self.block_size - @property - def last_block_num_tokens(self): - return self.num_tokens - (self.num_blocks - 1) * self.block_size + # @property + # def last_block_num_tokens(self): + # return self.num_tokens - (self.num_blocks - 1) * self.block_size def block(self, i): assert 0 <= i < self.num_blocks diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 8f798fbd54..391d1d9eac 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -1,3 +1,6 @@ +# from flash_attn import flash_attn_with_kvcache +from dataclasses import dataclass + import aiter import torch import triton @@ -5,14 +8,15 @@ from aiter.paged_attn import PagedAttention from torch import nn -from atom.utils.context import get_context - -# from flash_attn import flash_attn_with_kvcache -from dataclasses import dataclass - from atom.utils.context import get_context from atom.utils.custom_register import direct_register_custom_op -from atom.utils.forward_context import ForwardContext, get_forward_context, AttentionMetadata, set_forward_context +from atom.utils.forward_context import ( + AttentionMetadata, + ForwardContext, + get_forward_context, + set_forward_context, +) + # Dynamo will not try to inspect any of the internal operations for prefill or decode # @torch.library.custom_op("aiter::unified_attention_with_output", mutates_args=["q", "k", "v", "k_cache", "v_cache", "k_scale", "v_scale"]) diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 124cf23464..1a6e359d3d 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -1,8 +1,10 @@ import torch import torch.distributed as dist import torch.nn.functional as F +from aiter import logger from aiter.dist.communication_op import tensor_model_parallel_all_gather from aiter.dist.parallel_state import get_tp_group +from aiter.tuned_gemm import tgemm from torch import nn from atom.utils.context import get_context @@ -42,7 +44,7 @@ def forward(self, x: torch.Tensor): x = mask * (x - self.vocab_start_idx) y = F.embedding(x, self.weight) if self.tp_size > 1: - y = mask.unsqueeze(1) * y + y.masked_fill_(~mask.unsqueeze(1), 0) y = get_tp_group().all_reduce(y, ca_fp8_quant=False) return y @@ -68,7 +70,7 @@ def forward(self, x: torch.Tensor): if context.is_prefill: last_indices = context.cu_seqlens_q[1:] - 1 x = x[last_indices].contiguous() - logits = F.linear(x, self.weight, self.bias) + logits = tgemm.mm(x, self.weight, self.bias) if self.tp_size > 1: logits = tensor_model_parallel_all_gather(logits) # all_logits = ( diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 9a4b0ae563..49c1a91959 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -1,22 +1,25 @@ +from functools import partial as functools_partial +from typing import Callable, Optional + import torch -from torch import nn import torch.nn.functional as F - -# import torch.distributed as dist -from aiter.dist.parallel_state import get_tp_group from aiter import ( QuantType, dtypes, + gemm_a4w4, gemm_a8w8, - gemm_a8w8_bpreshuffle, gemm_a8w8_blockscale, - gemm_a4w4, + gemm_a8w8_bpreshuffle, + get_hip_quant, + get_triton_quant, ) + +# import torch.distributed as dist +from aiter.dist.parallel_state import get_tp_group from aiter.ops.shuffle import shuffle_weight from aiter.tuned_gemm import tgemm -from aiter import get_hip_quant, get_triton_quant -from typing import Optional, Callable -from functools import partial as functools_partial +from torch import nn + from atom.config import QuantizationConfig from atom.model_ops.utils import normalize_e4m3fn_to_e4m3fnuz, requantize_with_max_scale @@ -230,7 +233,7 @@ def forward( dtype=otype, ) if self.tp_dim == 1 and self.tp_size > 1: - y = get_tp_group().all_reduce(y, open_fp8_quant=False) + y = get_tp_group().all_reduce(y, ca_fp8_quant=False) return y @@ -418,4 +421,3 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) param.weight_loader_process(param_data, loaded_weight) - diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index 256c02c2b7..5a12da43da 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -14,7 +14,9 @@ from urllib.parse import urlparse from uuid import uuid4 +import numpy as np import psutil +import torch import zmq import zmq.asyncio @@ -26,14 +28,14 @@ def get_mp_context() -> Union[ForkContext, SpawnContext]: return multiprocessing.get_context("spawn") -def shutdown_all_processes(procs: list[BaseProcess]): +def shutdown_all_processes(procs: list[BaseProcess], allowed_seconds: int = 2): # Shutdown the process. for proc in procs: if proc.is_alive(): proc.terminate() # Allow 5 seconds for remaining procs to terminate. - deadline = time.monotonic() + 5 + deadline = time.monotonic() + allowed_seconds for proc in procs: remaining = deadline - time.monotonic() if remaining <= 0: @@ -277,6 +279,22 @@ def make_zmq_socket( return socket +def init_exit_handler(self: Any): + import weakref + + self.still_running = True + self._finalizer = weakref.finalize(self, self.exit) + + def signal_handler(signum, frame): + sig_str = signal.Signals(signum).name + msg = f"{self.label}: received signal {signum} ({sig_str}), exiting..." + logger.info(msg) + self._finalizer() + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + @contextlib.contextmanager def zmq_socket_ctx( path: str, @@ -295,17 +313,58 @@ def zmq_socket_ctx( finally: ctx.destroy(linger=linger) -from packaging import version -from packaging.version import Version -import importlib -import torch + + +class CpuGpuBuffer: + """Buffer to easily copy tensors between CPU and GPU.""" + + def __init__( + self, + *size: Union[int, torch.SymInt], + dtype: torch.dtype, + device: torch.device, + pin_memory: bool = True, + with_numpy: bool = True, + ) -> None: + self.cpu = torch.zeros(*size, dtype=dtype, device="cpu", pin_memory=pin_memory) + self.gpu = torch.zeros_like(self.cpu, device=device) + self.np: np.ndarray + # To keep type hints simple (avoiding generics and subclasses), we + # only conditionally create the numpy array attribute. This can cause + # AttributeError if `self.np` is accessed when `with_numpy=False`. + if with_numpy: + if dtype == torch.bfloat16: + raise ValueError( + "Bfloat16 torch tensors cannot be directly cast to a " + "numpy array, so call CpuGpuBuffer with with_numpy=False" + ) + self.np = self.cpu.numpy() + + def copy_to_gpu(self, n: Optional[int] = None) -> torch.Tensor: + if n is None: + return self.gpu.copy_(self.cpu, non_blocking=True) + return self.gpu[:n].copy_(self.cpu[:n], non_blocking=True) + + def copy_to_cpu(self, n: Optional[int] = None) -> torch.Tensor: + """NOTE: Because this method is non-blocking, explicit synchronization + is needed to ensure the data is copied to CPU.""" + if n is None: + return self.cpu.copy_(self.gpu, non_blocking=True) + return self.cpu[:n].copy_(self.gpu[:n], non_blocking=True) + + import copy import dataclasses -from contextlib import contextmanager -from typing import Union, Any -from atom.config import CompilationConfig, Config -from atom.config import CompilationLevel +import importlib import time +from contextlib import contextmanager +from typing import Any, Union + +import torch +from packaging import version +from packaging.version import Version + +from atom.config import CompilationConfig, CompilationLevel, Config context_manager = None torch_compile_start_time: float = 0.0 @@ -324,7 +383,7 @@ def is_torch_equal_or_newer(target: str) -> bool: return _is_torch_equal_or_newer(str(torch.__version__), target) except Exception: # Fallback to PKG-INFO to load the package info, needed by the doc gen. - return Version(importlib.metadata.version('torch')) >= Version(target) + return Version(importlib.metadata.version("torch")) >= Version(target) # Helper function used in testing. @@ -346,7 +405,7 @@ def weak_ref_tensor(tensor: Any) -> Any: def weak_ref_tensors( - tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]] + tensors: Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor]], ) -> Union[torch.Tensor, list[Any], tuple[Any], Any]: """ Convenience function to create weak references to tensors, @@ -396,13 +455,13 @@ def expect(self, **kwargs): assert getattr(self, k) - getattr(old, k) == v, ( f"{k} not as expected, before it is {getattr(old, k)}" f", after it is {getattr(self, k)}, " - f"expected diff is {v}") + f"expected diff is {v}" + ) compilation_counter = CompilationCounter() - def resolve_obj_by_qualname(qualname: str) -> Any: """ Resolve an object by its fully-qualified class name. @@ -417,11 +476,14 @@ def start_monitoring_torch_compile(vllm_config: Config): torch_compile_start_time = time.time() compilation_config: CompilationConfig = vllm_config.compilation_config - if compilation_config.level == CompilationLevel.PIECEWISE and \ - compilation_config.debug_dump_path: + if ( + compilation_config.level == CompilationLevel.PIECEWISE + and compilation_config.debug_dump_path + ): import depyf - path = os.path.join(compilation_config.debug_dump_path,"rank_0") - # f"rank_{vllm_config.parallel_config.rank}") + + path = os.path.join(compilation_config.debug_dump_path, "rank_0") + # f"rank_{vllm_config.parallel_config.rank}") global context_manager context_manager = depyf.prepare_debug(path) context_manager.__enter__() @@ -434,3 +496,38 @@ def end_monitoring_torch_compile(vllm_config: Config): if context_manager is not None: context_manager.__exit__(None, None, None) context_manager = None + + +def getLogger(): + global logger + if not logger.handlers: + logger.setLevel(logging.DEBUG) + + console_handler = logging.StreamHandler() + if int(os.environ.get("AITER_LOG_MORE", 0)): + formatter = logging.Formatter( + fmt="[%(name)s %(levelname)s] %(asctime)s.%(msecs)03d - %(processName)s:%(process)d - %(pathname)s:%(lineno)d - %(funcName)s\n%(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + else: + formatter = logging.Formatter( + fmt="[%(name)s] %(message)s", + ) + console_handler.setFormatter(formatter) + console_handler.setLevel(logging.INFO) + + logger.addHandler(console_handler) + if hasattr(torch._dynamo.config, "ignore_logger_methods"): + torch._dynamo.config.ignore_logger_methods = ( + logging.Logger.info, + logging.Logger.warning, + logging.Logger.debug, + logger.warning, + logger.info, + logger.debug, + ) + + return logger + + +logger = getLogger() diff --git a/atom/utils/backends.py b/atom/utils/backends.py index 3163db6685..087c1f18f9 100644 --- a/atom/utils/backends.py +++ b/atom/utils/backends.py @@ -3,6 +3,7 @@ import ast import dataclasses +import logging import os import pprint import time @@ -10,23 +11,30 @@ from contextlib import contextmanager from typing import Any, Callable, Optional - import torch import torch.fx as fx from torch._dispatch.python import enable_python_dispatcher -from .compiler_inferface import InductorAdaptor, CompilerInterface, InductorStandaloneAdaptor -from atom.utils import is_torch_equal_or_newer -from aiter import logger from atom.config import CompilationConfig, Config, CUDAGraphMode -from atom.utils import compilation_counter, resolve_obj_by_qualname +from atom.utils import ( + compilation_counter, + is_torch_equal_or_newer, + resolve_obj_by_qualname, +) + +from .compiler_inferface import ( + CompilerInterface, + InductorAdaptor, + InductorStandaloneAdaptor, +) + +logger = logging.getLogger("atom") def get_static_graph_wrapper_cls(cls) -> str: return "atom.utils.cuda_graph.CUDAGraphWrapper" - def make_compiler(compilation_config: CompilationConfig) -> CompilerInterface: # return InductorAdaptor() if compilation_config.use_inductor: @@ -529,7 +537,7 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: os.makedirs(cache_dir, exist_ok=True) self.compilation_config.cache_dir = cache_dir # rank = vllm_config.parallel_config.rank - rank = 0 + rank = torch.cuda.current_device() # dp_rank = vllm_config.parallel_config.data_parallel_rank local_cache_dir = os.path.join(cache_dir, f"rank_{rank}", self.prefix) diff --git a/atom/utils/context.py b/atom/utils/context.py index 72187faa39..8a348532ef 100644 --- a/atom/utils/context.py +++ b/atom/utils/context.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Optional import torch @@ -8,14 +9,14 @@ class Context: is_prefill: bool = False batch_size: int = 0 graph_bs: int = 0 - cu_seqlens_q: torch.Tensor | None = None - cu_seqlens_k: torch.Tensor | None = None + cu_seqlens_q: Optional[torch.Tensor] = None + cu_seqlens_k: Optional[torch.Tensor] = None max_seqlen_q: int = 0 max_seqlen_k: int = 0 min_seqlen_q: int = 0 - slot_mapping: torch.Tensor | None = None - context_lens: torch.Tensor | None = None - block_tables: torch.Tensor | None = None + slot_mapping: Optional[torch.Tensor] = None + context_lens: Optional[torch.Tensor] = None + block_tables: Optional[torch.Tensor] = None dropout_p: float = 0.0 diff --git a/example.py b/example.py index 1a1a224bf4..b31e868638 100644 --- a/example.py +++ b/example.py @@ -60,7 +60,7 @@ def main(): args = parser.parse_args() model_name_or_path = args.model tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) - # If you want to use torch.compile, please --level=3 + # If you want to use torch.compile, please --level=3 llm = LLMEngine( model_name_or_path, enforce_eager=args.enforce_eager, @@ -79,7 +79,7 @@ def main(): "introduce yourself", "list all prime numbers within 100", "1+2+3=?", - # "2+3+4=?", + "ε¦‚δ½•εœ¨δΈ€δΈͺζœˆε†…ε’žθ‚Œ10ε…¬ζ–€", ] prompts = [ tokenizer.apply_chat_template( @@ -91,7 +91,7 @@ def main(): for prompt in prompts ] print("This is prompts:", prompts) - + # # Calculate total token number of prompts # prompt_token_ids = [llm.tokenizer.encode(prompt) for prompt in prompts] # prompt_lens = [len(token_ids) for token_ids in prompt_token_ids] @@ -105,9 +105,9 @@ def main(): # warmup_prompts.append(warmup_prompt) # # Run warmup with the same batch structure as the actual prompts (no profiling) # _ = llm.generate(warmup_prompts, sampling_params) - + # generate (with profiling) - outputs = llm.generate(prompts, sampling_params, enable_profiling=True) + outputs = llm.generate(prompts, sampling_params) for prompt, output in zip(prompts, outputs): print("\n") diff --git a/test.py b/test.py new file mode 100644 index 0000000000..a993d2eb54 --- /dev/null +++ b/test.py @@ -0,0 +1,79 @@ +import cProfile +import os +import pstats +import time +from random import randint, seed + +from transformers import AutoTokenizer + +from atom import LLMEngine, SamplingParams +from atom.config import CompilationConfig + +# A very long prompt, total number of tokens is about 15k. +# LONG_PROMPT = ["You are an expert in large language models, aren't you?"] * 256 +# LONG_PROMPT = " ".join(LONG_PROMPT) +LONG_PROMPT = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", +] * 64 + + +def main(): + seed(0) + num_seqs = 256 + max_input_len = 1024 + max_ouput_len = 1024 + + path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") + llm = LLMEngine( + path, + enforce_eager=False, + max_model_len=4096, + port=12345, + torch_profiler_dir="./log", + compilation_config=CompilationConfig( + level=3, + # cudagraph_capture_sizes=[1, 2, 4, 8] + list(range(16, 512 + 1, 16)) + ), + ) + + # tokenizer = AutoTokenizer.from_pretrained(path) + + prompt_token_ids = [ + [randint(0, 10000) for _ in range(randint(100, max_input_len))] + for _ in range(num_seqs) + ] + + # sampling_params = SamplingParams(temperature=0, max_tokens=1024) + sampling_params = [ + SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=16) + for _ in range(num_seqs) + ] + + print("------start generating------") + # prompts = [ + # tokenizer.apply_chat_template( + # [{"role": "user", "content": prompt}], + # tokenize=False, + # add_generation_prompt=True, + # enable_thinking=True, + # ) + # for prompt in prompts + # ] + + outputs = llm.generate(["Benchmark: "], SamplingParams()) + t = time.time() + outputs = llm.generate(LONG_PROMPT, sampling_params) + t = time.time() - t + total_tokens = sum(sp.max_tokens for sp in sampling_params) + throughput = total_tokens / t + + print( + f"Total: {total_tokens}tok, Time: {t:.2f}s, Throughput: {throughput:.2f}tok/s" + ) + + +if __name__ == "__main__": + main() From 5037ee795671e66eda12356524a84f5511303d86 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Mon, 6 Oct 2025 08:57:32 +0000 Subject: [PATCH 049/119] [draft] add deferred output --- atom/model_engine/model_runner.py | 104 +++++++++++++++++++++++------- atom/model_engine/scheduler.py | 31 +++++---- 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index c58aad9a0f..a1cdee544c 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -20,7 +20,7 @@ from atom.models.llama import LlamaForCausalLM from atom.models.mixtral import MixtralForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM -from atom.utils import CpuGpuBuffer +from atom.utils import CpuGpuBuffer, init_exit_handler from atom.utils.context import get_context, reset_context, set_context logger = logging.getLogger("atom") @@ -33,6 +33,62 @@ } +class OutputProcessor: + + def __init__(self): + """Asynchronously copy the sampled_token_ids tensor to the host.""" + # Event on the copy stream so we can synchronize the non-blocking copy. + self.async_copy_event = torch.cuda.Event() + self.async_copy_stream = torch.cuda.Stream() + self.deferred_request_id: list[list[int]] = [] + self.deferred_token_id: list[list[int]] = [] + self.token_ids_gpu: list[torch.Tensor] = [] + self.token_ids_cpu: list[torch.Tensor] = [] + + def send_to_cpu_async(self, gpu_tensor: torch.Tensor): + default_stream = torch.cuda.current_stream() + with torch.cuda.stream(self.async_copy_stream): + self.async_copy_stream.wait_stream(default_stream) + cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) + self.async_copy_event.record(self.async_copy_stream) + self.token_ids_gpu.append(gpu_tensor) + self.token_ids_cpu.append(cpu_tensor) + + def recv_async_output(self) -> list[int]: + self.async_copy_event.synchronize() + for _ in self.token_ids_gpu: + token_ids = self.token_ids_cpu.pop(0).tolist() + return token_ids + return [] + + def update_and_ret( + self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor + ) -> list[int]: + token_ids = self.recv_async_output() + self.send_to_cpu_async(sampled_token_ids) + deferred_reqIDs = [] + returned_reqIDs = [] + for req in batch.seqs: + if req.id in self.deferred_request_id: + returned_reqIDs.append(req.id) + else: + deferred_reqIDs.append(req.id) + + if returned_reqIDs: + for ids in self.token_ids_cpu[:-1]: + ids_list = ids.tolist() + for req_id in returned_reqIDs: + self.deferred_request_id.remove(req_id) + req_index = batch.unfinished_prev_req.index(req_id) + batch.unfinished_prev_req.remove(req_id) + req = batch.seqs[req_index] + req.append_token(ids_list[req_index]) + + self.async_copy_event.synchronize() + + return [] + + class ModelRunner: def __init__(self, rank: int, config: Config): @@ -60,10 +116,11 @@ def __init__(self, rank: int, config: Config): os.environ["MASTER_ADDR"] = self.config.master_addr os.environ["MASTER_PORT"] = str(self.config.port) init_dist_env(self.world_size, rankID=rank) - self.init_exit_handler() + init_exit_handler(self) default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") + self.out_processor = OutputProcessor() self.sampler = Sampler() self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) load_model(self.model, config.model) @@ -77,19 +134,6 @@ def __init__(self, rank: int, config: Config): if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") - def init_exit_handler(self): - logger.info(f"Initializing ModelRunner for rank {self.rank}") - self.still_running = True - self._finalizer = weakref.finalize(self, self.exit) - - def signal_handler(signum, frame): - sig_name = signal.Signals(signum).name - msg = f"{self.label}: received signal {signum} ({sig_name}), exiting..." - logger.warning(msg) - self._finalizer() - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) def exit(self): if not self.still_running: @@ -386,7 +430,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] - def prepare_sample(self, seqs: list[Sequence]): + def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: temperatures = [seq.temperature for seq in seqs] temperatures = torch.tensor( temperatures, dtype=torch.float32, pin_memory=True @@ -414,15 +458,29 @@ def run_model( self.graphs[graph_bs].replay() return self.model.compute_logits(self.decode_vars["outputs"][:bs]) + def postprocess( + self, + batch: ScheduledBatchs, + logits: torch.Tensor, + temperatures: Optional[torch.Tensor], + ) -> list[int]: + if self.rank == 0: + sampled_tokens = self.sampler(logits, temperatures) + token_ids = sampled_tokens.tolist() + # token_ids = self.prev_sampled.update_and_ret( + # batch, + # sampled_tokens, + # ) + else: + token_ids = [] + return token_ids + @torch.inference_mode() - def forward(self, scheduled_batchs: ScheduledBatchs) -> Optional[list[int]]: - input_ids, positions, temperatures = self.prepare_model(scheduled_batchs) - logits = self.run_model(input_ids, positions, scheduled_batchs.is_prefill) - token_ids = ( - self.sampler(logits, temperatures).tolist() if self.rank == 0 else None - ) + def forward(self, batch: ScheduledBatchs) -> list[int]: + input_ids, positions, temperatures = self.prepare_model(batch) + logits = self.run_model(input_ids, positions, batch.is_prefill) reset_context() - return token_ids + return self.postprocess(batch, logits, temperatures) @torch.inference_mode() def capture_cudagraph(self): diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index e368aaf339..cc0dfdb3ce 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -10,7 +10,7 @@ class ScheduledBatchs: seqs: list[Sequence] is_prefill: bool - has_seq_out: bool + deferred_reqID: list[int] class Scheduler: @@ -18,11 +18,12 @@ class Scheduler: def __init__(self, config: Config): self.max_num_seqs = config.max_num_seqs self.max_num_batched_tokens = config.max_num_batched_tokens + self.bos_token_id = config.bos_token_id self.eos_token_id = config.eos_token_id self.block_manager = BlockManager(config) self.waiting: deque[Sequence] = deque() self.running: deque[Sequence] = deque() - self.has_seq_out = False + self.deferred_reqID = [] def is_finished(self): return not self.waiting and not self.running @@ -41,7 +42,6 @@ def schedule(self) -> ScheduledBatchs: if not self.running and not self.waiting: self.block_manager.reset() - has_seq_out = self.has_seq_out while self.waiting and num_seqs < self.max_num_seqs: seq = self.waiting[0] if num_batched_tokens + len( @@ -57,9 +57,7 @@ def schedule(self) -> ScheduledBatchs: scheduled_seqs.append(seq) if scheduled_seqs: return ScheduledBatchs( - seqs=scheduled_seqs, - is_prefill=True, - has_seq_out=has_seq_out, + seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[] ) # decode @@ -77,11 +75,10 @@ def schedule(self) -> ScheduledBatchs: scheduled_seqs.append(seq) assert scheduled_seqs self.running.extendleft(reversed(scheduled_seqs)) - self.has_seq_out = False return ScheduledBatchs( seqs=scheduled_seqs, is_prefill=False, - has_seq_out=has_seq_out, + deferred_reqID=self.deferred_reqID, ) def preempt(self, seq: Sequence): @@ -89,9 +86,20 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], token_ids: list[int]): - for seq, token_id in zip(seqs, token_ids): + def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): + # # placeholder for the each decode step + # token_ids = [self.eos_token_id] * len(seqs) + # for seq, token_id in zip(seqs, token_ids): + # seq.append_token(token_id) + + # if not prev_token_ids: + # return + # update token_ids with the actual sampled token ids + self.unfinished_req = [] + token_ids = prev_token_ids + for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): seq.append_token(token_id) + # seq.token_ids[-2] = token_id leave_reason = None if not seq.ignore_eos and token_id == self.eos_token_id: leave_reason = "eos" @@ -102,4 +110,5 @@ def postprocess(self, seqs: list[Sequence], token_ids: list[int]): seq.status = SequenceStatus.FINISHED self.block_manager.deallocate(seq) self.running.remove(seq) - self.has_seq_out = True + else: + self.unfinished_req.append(i) From 7b17aa63745d2e70d47d82ae226e327d8acee090 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 9 Oct 2025 14:30:13 +0800 Subject: [PATCH 050/119] add atom example confluence --- bench_test.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++ confluence-page.md | 29 +++++++++++++++++++ example.py | 2 +- 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 bench_test.py create mode 100644 confluence-page.md diff --git a/bench_test.py b/bench_test.py new file mode 100644 index 0000000000..4c7674bc10 --- /dev/null +++ b/bench_test.py @@ -0,0 +1,71 @@ +import argparse +import os +import pstats +import time +from random import randint, seed + +from transformers import AutoTokenizer + +from atom import LLMEngine, SamplingParams +from atom.config import CompilationConfig + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="config of test", +) + +parser.add_argument( + "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." +) + +# A very long prompt, total number of tokens is about 15k. +LONG_PROMPT = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", +] * 64 + + +def main(): + seed(0) + num_seqs = 256 + max_input_len = 1024 + max_ouput_len = 1024 + + args = parser.parse_args() + path = args.model + llm = LLMEngine( + path, + enforce_eager=False, + max_model_len=4096, + port=12345, + torch_profiler_dir="./log", + compilation_config=CompilationConfig( + level=3, + ), + ) + + + sampling_params = [ + SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=16) + for _ in range(num_seqs) + ] + + print("------start generating------") + + #warm up + outputs_ = llm.generate(["Benchmark: "], SamplingParams()) + t = time.time() + outputs = llm.generate(LONG_PROMPT, sampling_params) + t = time.time() - t + total_tokens = sum(sp.max_tokens for sp in sampling_params) + throughput = total_tokens / t + + print( + f"Total: {total_tokens}tok, Time: {t:.2f}s, Throughput: {throughput:.2f}tok/s" + ) + + +if __name__ == "__main__": + main() diff --git a/confluence-page.md b/confluence-page.md new file mode 100644 index 0000000000..2ad2fcb8fb --- /dev/null +++ b/confluence-page.md @@ -0,0 +1,29 @@ +# Atom + +A lightweight vLLM implementation built from scratch. + +## Installation + +```bash +git clone https://github.com/valarLip/atom.git +cd atom +``` + +## Example + +```bash +python example.py --model meta-llama/Meta-Llama-3-8B +``` + +The default level is 3(running with torch compile), the model we support now is Qwen, Llama, Mixtral, you can set the model name or path using --model + +## Perf running + +```bash +python bench_test.py --model Qwen/Qwen3-0.6B +``` + +You can get the performance detail: +```bash +Total: 4096tok, Time: 0.31s, Throughput: 13333.85tok/s +``` \ No newline at end of file diff --git a/example.py b/example.py index b31e868638..4e39e6411b 100644 --- a/example.py +++ b/example.py @@ -43,7 +43,7 @@ "--cudagraph-capture-sizes", type=str, default="[1,2,4,8,16]", help="Sizes to capture cudagraph." ) -parser.add_argument("--level", type=int, default=0, help="The level of compilation") +parser.add_argument("--level", type=int, default=3, help="The level of compilation") parser.add_argument( "--torch-profiler-dir", type=str, default=None, help="Directory to save torch profiler traces" From 16a01fdeefd5719cde8a4ecdbf51df37eaac29dd Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 9 Oct 2025 16:54:52 +0800 Subject: [PATCH 051/119] rename --- confluence-page.md => atom_example.md | 0 test.py | 79 --------------------------- 2 files changed, 79 deletions(-) rename confluence-page.md => atom_example.md (100%) delete mode 100644 test.py diff --git a/confluence-page.md b/atom_example.md similarity index 100% rename from confluence-page.md rename to atom_example.md diff --git a/test.py b/test.py deleted file mode 100644 index a993d2eb54..0000000000 --- a/test.py +++ /dev/null @@ -1,79 +0,0 @@ -import cProfile -import os -import pstats -import time -from random import randint, seed - -from transformers import AutoTokenizer - -from atom import LLMEngine, SamplingParams -from atom.config import CompilationConfig - -# A very long prompt, total number of tokens is about 15k. -# LONG_PROMPT = ["You are an expert in large language models, aren't you?"] * 256 -# LONG_PROMPT = " ".join(LONG_PROMPT) -LONG_PROMPT = [ - "Hello, my name is", - "The president of the United States is", - "The capital of France is", - "The future of AI is", -] * 64 - - -def main(): - seed(0) - num_seqs = 256 - max_input_len = 1024 - max_ouput_len = 1024 - - path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") - llm = LLMEngine( - path, - enforce_eager=False, - max_model_len=4096, - port=12345, - torch_profiler_dir="./log", - compilation_config=CompilationConfig( - level=3, - # cudagraph_capture_sizes=[1, 2, 4, 8] + list(range(16, 512 + 1, 16)) - ), - ) - - # tokenizer = AutoTokenizer.from_pretrained(path) - - prompt_token_ids = [ - [randint(0, 10000) for _ in range(randint(100, max_input_len))] - for _ in range(num_seqs) - ] - - # sampling_params = SamplingParams(temperature=0, max_tokens=1024) - sampling_params = [ - SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=16) - for _ in range(num_seqs) - ] - - print("------start generating------") - # prompts = [ - # tokenizer.apply_chat_template( - # [{"role": "user", "content": prompt}], - # tokenize=False, - # add_generation_prompt=True, - # enable_thinking=True, - # ) - # for prompt in prompts - # ] - - outputs = llm.generate(["Benchmark: "], SamplingParams()) - t = time.time() - outputs = llm.generate(LONG_PROMPT, sampling_params) - t = time.time() - t - total_tokens = sum(sp.max_tokens for sp in sampling_params) - throughput = total_tokens / t - - print( - f"Total: {total_tokens}tok, Time: {t:.2f}s, Throughput: {throughput:.2f}tok/s" - ) - - -if __name__ == "__main__": - main() From a73f7bca1962116ffccc175a5ef6970228e4f7bd Mon Sep 17 00:00:00 2001 From: jiayyu Date: Sat, 11 Oct 2025 16:39:20 +0800 Subject: [PATCH 052/119] support ds --- atom/config.py | 35 +- atom/model_engine/block_manager.py | 12 +- atom/model_engine/llm_engine.py | 2 +- atom/model_engine/model_runner.py | 188 ++- atom/model_loader/loader.py | 23 +- atom/model_ops/attention_mla.py | 315 +++++ atom/model_ops/linear.py | 10 +- atom/model_ops/moe.py | 100 +- atom/model_ops/topK.py | 4 +- atom/model_ops/utils.py | 29 +- atom/models/deepseek_v2.py | 1877 ++++++++-------------------- atom/utils/context.py | 12 + 12 files changed, 1059 insertions(+), 1548 deletions(-) create mode 100644 atom/model_ops/attention_mla.py diff --git a/atom/config.py b/atom/config.py index 558baaea6d..30ed3923d1 100644 --- a/atom/config.py +++ b/atom/config.py @@ -117,23 +117,23 @@ class CompilationConfig: FULL mode: Capture full cudagraph for all batches. Can be good for small models or workloads with small prompts; not supported by many backends. Generally for performance FULL_AND_PIECEWISE is better. - + FULL_DECODE_ONLY mode: Capture full cudagraph for decode batches only. Mixed prefill-decode batches are run without cudagraphs. Can be good for decode instances in a P/D setup where prefill is not as important so we can save some memory. - + FULL_AND_PIECEWISE mode: Capture full cudagraph for decode batches and piecewise cudagraph for prefill and mixed prefill-decode batches. This is like the most performant mode for most models. Currently, the cudagraph mode is only used for the v1 engine. - Note that the cudagraph logic is generally orthogonal to the - compilation logic. While piecewise cudagraphs require piecewise + Note that the cudagraph logic is generally orthogonal to the + compilation logic. While piecewise cudagraphs require piecewise compilation (level=PIECEWISE and non-empty splitting_ops), full cudagraphs are supported with and without compilation. - - Warning: This flag is new and subject to change in addition + + Warning: This flag is new and subject to change in addition more modes may be added. """ @@ -150,7 +150,7 @@ class CompilationConfig: cudagraph. If the caller can guarantee that the same input buffers are always used, it can set this to False. Otherwise, it should set this to True, and the compiler will copy the input to an - internally managed buffer. Default is False. + internally managed buffer. Default is False. Note that this flag is only effective when cudagraph_mode is PIECEWISE. """ @@ -346,7 +346,7 @@ class Config: hf_config: PretrainedConfig = field(init=False) bos_token_id: int = -1 eos_token_id: int = -1 - kvcache_block_size: int = 16 + kv_cache_block_size: int = 16 num_kvcache_blocks: int = -1 kv_cache_dtype: str = "bf16" enable_prefix_caching: bool = False @@ -357,6 +357,8 @@ class Config: default_factory=lambda: QuantizationConfig() ) asyncio_mode: bool = False + load_dummy: bool = False + enable_expert_parallel: bool = False master_addr: str = "127.0.0.1" graph_bs: Optional[list[int]] = None @@ -374,7 +376,9 @@ def _set_cudagraph_sizes(self): def __post_init__(self): # assert os.path.isdir(self.model) - assert self.kvcache_block_size % 16 == 0 + assert (self.kv_cache_block_size % 16 == 0 or self.kv_cache_block_size == 1), ( + f"kv_cache_block_size ({self.kv_cache_block_size}) must be a multiple of 16 or 1" + ) assert 1 <= self.tensor_parallel_size <= 8 self.hf_config = AutoConfig.from_pretrained(self.model) self.quant_config = get_quant_config(self.hf_config) @@ -419,3 +423,16 @@ def compute_hash(self) -> str: str(factors).encode(), usedforsecurity=False ).hexdigest()[:10] return hash_str + + +_current_atom_config: Optional[Config] = None + + +def set_current_atom_config(atom_config: Config): + global _current_atom_config + _current_atom_config = atom_config + + +def get_current_atom_config() -> Config: + assert _current_atom_config is not None, "Current atom config is not set" + return _current_atom_config diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index e123073c5f..6df703bedb 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -28,7 +28,7 @@ def reset(self): class BlockManager: def __init__(self, config: Config): - block_size = config.kvcache_block_size + block_size = config.kv_cache_block_size num_blocks = config.num_kvcache_blocks assert num_blocks > 0 self.block_size = block_size @@ -108,11 +108,17 @@ def can_append(self, seq: Sequence) -> bool: def may_append(self, seq: Sequence): block_table = seq.block_table last_block = self.blocks[block_table[-1]] - if len(seq) % self.block_size == 1: + if len(seq) % self.block_size == 1 or self.block_size == 1: assert last_block.hash != -1 block_id = self.free_block_ids[0] - self._allocate_block(block_id) + block = self._allocate_block(block_id) block_table.append(block_id) + if self.block_size == 1: + token_ids = [seq[-1]] + prefix = self.blocks[block_table[-2]].hash if len(block_table) > 1 else -1 + h = self.compute_hash(token_ids, prefix) + block.update(h, token_ids) + self.hash_to_block_id[h] = block_id elif len(seq) % self.block_size == 0: assert last_block.hash == -1 token_ids = seq.block(seq.num_blocks - 1) diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 79cabebde6..9e5ca73424 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -28,7 +28,7 @@ def __init__(self, model, **kwargs): self.rquest_ids = set() self.io_processor = InputOutputProcessor( - self.tokenizer, config.kvcache_block_size + self.tokenizer, config.kv_cache_block_size ) self.core_mgr = CoreManager(config) logger.info("LLMEngine init") diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index a1cdee544c..2615a29b2b 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -12,7 +12,7 @@ from aiter import destroy_dist_env, dtypes, init_dist_env from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture -from atom.config import Config +from atom.config import Config, set_current_atom_config from atom.model_engine.scheduler import ScheduledBatchs from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model @@ -20,6 +20,7 @@ from atom.models.llama import LlamaForCausalLM from atom.models.mixtral import MixtralForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM +from atom.models.deepseek_v2 import DeepseekV2ForCausalLM from atom.utils import CpuGpuBuffer, init_exit_handler from atom.utils.context import get_context, reset_context, set_context @@ -30,6 +31,7 @@ "Qwen3ForCausalLM": Qwen3ForCausalLM, "LlamaForCausalLM": LlamaForCausalLM, "MixtralForCausalLM": MixtralForCausalLM, + "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } @@ -93,8 +95,9 @@ class ModelRunner: def __init__(self, rank: int, config: Config): self.config = config + set_current_atom_config(config) hf_config = config.hf_config - self.block_size = config.kvcache_block_size + self.block_size = config.kv_cache_block_size self.enforce_eager = config.enforce_eager self.world_size = config.tensor_parallel_size self.rank = rank @@ -123,8 +126,9 @@ def __init__(self, rank: int, config: Config): self.out_processor = OutputProcessor() self.sampler = Sampler() self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) - load_model(self.model, config.model) - logger.info(f"Model loaded: {config.model}") + torch.set_default_device(None) + load_model(self.model, config.model, config.hf_config, config.load_dummy) + torch.set_default_device("cuda") self.warmup_model() logger.info(f"Model warmup done: {config.model}") self.allocate_decode_vars() @@ -178,7 +182,7 @@ def warmup_model(self): max_num_batched_tokens // max_model_len, self.config.max_num_seqs ) seqs = [ - Sequence([0] * max_num_batched_tokens, block_size=self.block_size) + Sequence([0] * max_model_len, block_size=self.block_size) for _ in range(num_seqs) ] dummy_batch = ScheduledBatchs(seqs, True, False) @@ -199,8 +203,17 @@ def allocate_decode_vars(self): "block_tables": CpuGpuBuffer( max_bs, max_num_blocks, dtype=torch.int32, device=self.device ), + "cu_seqlens_q": CpuGpuBuffer(max_bs + 1, dtype=torch.int32, device=self.device), + "kv_indptr": CpuGpuBuffer(max_bs + 1, dtype=torch.int32, device=self.device), + "kv_indices": CpuGpuBuffer(max_bs * max_num_blocks, dtype=torch.int32, device=self.device), + "kv_last_page_lens": CpuGpuBuffer(max_bs, dtype=torch.int32, device=self.device), "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), } + self.decode_vars["cu_seqlens_q"].cpu.copy_(torch.arange(0, max_bs + 1, step=1, dtype=torch.int32)) + self.decode_vars["cu_seqlens_q"].copy_to_gpu() + + self.decode_vars["kv_last_page_lens"].cpu.fill_(1) + self.decode_vars["kv_last_page_lens"].copy_to_gpu() def get_num_blocks(self): torch.set_default_device("cuda") @@ -214,14 +227,23 @@ def get_num_blocks(self): current = torch.cuda.memory_stats()["allocated_bytes.all.current"] torch.set_default_device("cpu") num_kv_heads = hf_config.num_key_value_heads // self.world_size - block_bytes = ( - 2 - * hf_config.num_hidden_layers - * self.block_size - * num_kv_heads - * hf_config.head_dim - * hf_config.torch_dtype.itemsize - ) + isMLA = hf_config.architectures[0].startswith("DeepseekV") + if isMLA: + block_bytes = ( + hf_config.num_hidden_layers + * self.block_size + * 576 + * dtypes.d_dtypes[config.kv_cache_dtype].itemsize + ) + else: + block_bytes = ( + 2 + * hf_config.num_hidden_layers + * self.block_size + * num_kv_heads + * hf_config.head_dim + * dtypes.d_dtypes[config.kv_cache_dtype].itemsize + ) num_kvcache_blocks = ( int(total * config.gpu_memory_utilization - used - peak + current) // block_bytes @@ -233,29 +255,38 @@ def allocate_kv_cache(self, num_kvcache_blocks): config = self.config config.num_kvcache_blocks = num_kvcache_blocks hf_config = config.hf_config + isMLA = hf_config.architectures[0].startswith("DeepseekV") num_kv_heads = hf_config.num_key_value_heads // self.world_size - self.kv_cache = torch.zeros( - ( + if isMLA: + self.kv_cache = torch.zeros( + hf_config.num_hidden_layers, + config.num_kvcache_blocks, + self.block_size, + 576, + dtype=dtypes.d_dtypes[config.kv_cache_dtype], + device="cuda", + ) + else: + self.kv_cache = torch.zeros( 2, hf_config.num_hidden_layers, config.num_kvcache_blocks, self.block_size, num_kv_heads, hf_config.head_dim, - ), - dtype=dtypes.d_dtypes[config.kv_cache_dtype], - device="cuda", - ) + dtype=dtypes.d_dtypes[config.kv_cache_dtype], + device="cuda", + ) - self.kv_scale = torch.zeros( - 2, - hf_config.num_hidden_layers, - config.num_kvcache_blocks, - self.block_size, - num_kv_heads, - dtype=dtypes.fp32, - device="cuda", - ) + self.kv_scale = torch.zeros( + 2, + hf_config.num_hidden_layers, + config.num_kvcache_blocks, + self.block_size, + num_kv_heads, + dtype=dtypes.fp32, + device="cuda", + ) layer_id = 0 x = 16 // self.kv_cache.element_size() @@ -286,6 +317,14 @@ def allocate_kv_cache(self, num_kvcache_blocks): ) set_forward_context(module.layer_num, attention_metadata) + layer_id += 1 + elif hasattr(module, "kv_cache") and isMLA: + module.kv_cache = self.kv_cache[layer_id].view( + config.num_kvcache_blocks * self.block_size, + 1, + 576, + ) + module.max_model_len = self.config.max_model_len layer_id += 1 return True @@ -358,12 +397,9 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): positions = [] slot_mapping = [] context_lens = [] - # cu_seqlens_q = [0] - # cu_seqlens_k = [0] - # max_seqlen_q = 1 - # max_seqlen_k = 0 - # min_seqlen_q = 0 + cu_seqlens_q = [0] dropout_p = 0.0 + self.total_blocks = 0 context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] positions = np.array(context_lens, dtype=np.int64) @@ -374,30 +410,21 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 for seq in scheduled_batchs.seqs ] - # cu_seqlens_k = np.cumsum(context_lens, dtype=np.int32) - # for seq in scheduled_batchs.seqs: - # # current_seq_len = seq.num_tokens - # # cu_seqlens_q.append(cu_seqlens_q[-1] + 1) - - # # cu_seqlens_k.append(cu_seqlens_k[-1] + current_seq_len) - # # max_seqlen_k = max(current_seq_len, max_seqlen_k) - - # # input_ids.append(seq.last_token) - # # positions.append(current_seq_len) - # # context_lens.append(current_seq_len) - # slot_mapping.append( - # seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 - # ) slot_mapping.extend([-1] * (graph_bs - bs)) slot_mapping = np.array(slot_mapping, dtype=np.int64) - # max_seqlen_k = max(context_lens) - # input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) - # positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) - # slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) - # context_lens = torch.tensor(context_lens, dtype=torch.int32, pin_memory=True) + + if isinstance(self.model, DeepseekV2ForCausalLM): + assert self.block_size == 1, "AITER MLA requires only block size 1." + self.kv_indices: list[int] = [] + self.kv_indptr: list[int] = [0] + self.kv_last_page_lens: list[int] = [] + + for seq in scheduled_batchs.seqs: + current_seq_len = len(seq) + cu_seqlens_q.append(cu_seqlens_q[-1] + 1) + self._update_paged_kv_tensors(seq.block_table, current_seq_len) + block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - # cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) - # cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping @@ -405,31 +432,65 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): var["positions"].np[:bs] = positions var["context_lens"].np[:bs] = context_lens var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables + + if isinstance(self.model, DeepseekV2ForCausalLM) and len(self.kv_indptr) > 0: + # extend to the maximum number of blocks as returned by the scheduler + self.kv_indices.extend([0] * (self.total_blocks - len(self.kv_indices))) + var["kv_indices"].np[: self.total_blocks] = np.array(self.kv_indices, dtype=np.int64) + var["kv_indptr"].np[: bs + 1] = np.array(self.kv_indptr, dtype=np.int64) + var["kv_indptr"].np[bs + 1 : graph_bs + 1] = var["kv_indptr"].np[bs] + var["kv_last_page_lens"].np[:bs] = np.array(self.kv_last_page_lens, dtype=np.int64) + var["cu_seqlens_q"].np[:bs + 1] = np.array(cu_seqlens_q, dtype=np.int64) + for el in [ "slot_mapping", "input_ids", "positions", "context_lens", "block_tables", + "cu_seqlens_q", + "kv_indices", + "kv_indptr", + "kv_last_page_lens" ]: var[el].copy_to_gpu() set_context( False, batch_size=bs, graph_bs=graph_bs, - # cu_seqlens_q=cu_seqlens_q, - # cu_seqlens_k=cu_seqlens_k, - # max_seqlen_q=max_seqlen_q, - # max_seqlen_k=max_seqlen_k, - # min_seqlen_q=min_seqlen_q, + cu_seqlens_q=var["cu_seqlens_q"].gpu[:bs + 1], slot_mapping=var["slot_mapping"].gpu[:bs], context_lens=var["context_lens"].gpu[:bs], block_tables=var["block_tables"].gpu[:bs, : block_tables.shape[1]], dropout_p=dropout_p, + max_q_len=1, + kv_indptr=var["kv_indptr"].gpu[: bs + 1], + kv_indices=var["kv_indices"].gpu, + kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] + def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): + # Get the number of valid blocks based on sequence length. + # If seq_len = 16, block_size = 16, + # block_table_bound is 1 with 1 valid block. + # If seq_len = 15, block_size = 16, + # block_table_bound is 0 + 1 with 1 valid block. + self.total_blocks += len(block_table) + block_table_bound = ( + seq_len // self.block_size + 1 + if seq_len % self.block_size != 0 + else seq_len // self.block_size + ) + self.kv_indices.extend(block_table[:block_table_bound]) + self.kv_indptr.append(self.kv_indptr[-1] + block_table_bound) + + last_page_len = seq_len % self.block_size + if last_page_len == 0: + last_page_len = self.block_size + self.kv_last_page_lens.append(last_page_len) + def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: temperatures = [seq.temperature for seq in seqs] temperatures = torch.tensor( @@ -490,6 +551,10 @@ def capture_cudagraph(self): slot_mapping = self.decode_vars["slot_mapping"].gpu context_lens = self.decode_vars["context_lens"].gpu block_tables = self.decode_vars["block_tables"].gpu + cu_seqlens_q = self.decode_vars["cu_seqlens_q"].gpu + kv_indptr = self.decode_vars["kv_indptr"].gpu + kv_indices = self.decode_vars["kv_indices"].gpu + kv_last_page_lens = self.decode_vars["kv_last_page_lens"].gpu outputs = self.decode_vars["outputs"] # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) @@ -525,6 +590,11 @@ def capture_cudagraph(self): slot_mapping=slot_mapping[:bs], context_lens=context_lens[:bs], block_tables=block_tables[:bs], + max_q_len=1, + cu_seqlens_q=cu_seqlens_q[: bs + 1], + kv_indptr=kv_indptr[: bs + 1], + kv_indices=kv_indices[:], + kv_last_page_lens=kv_last_page_lens[:bs], ) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index 43b6d56498..e0ed0ff258 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -1,4 +1,4 @@ -import os +import os, re from glob import glob import torch from torch import nn @@ -6,14 +6,15 @@ from transformers.utils import SAFE_WEIGHTS_INDEX_NAME from atom.model_loader.weight_utils import download_weights_from_hf, filter_duplicate_safetensors_files from atom.model_ops.base_config import QuantizeMethodBase +from transformers import AutoConfig from tqdm import tqdm - +from atom.model_ops.moe import is_rocm_aiter_fusion_shared_expert_enabled def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor): param.data.copy_(loaded_weight) -def load_model(model: nn.Module, model_name_or_path: str): +def load_model(model: nn.Module, model_name_or_path: str, hf_config: AutoConfig, load_dummy:bool = False): packed_modules_mapping = getattr(model, "packed_modules_mapping", {}) is_path = os.path.isdir(model_name_or_path) path = ( @@ -23,8 +24,11 @@ def load_model(model: nn.Module, model_name_or_path: str): model_name_or_path, None, ["*.safetensors"], ignore_patterns=["original/*"] ) ) + params_dict = dict(model.named_parameters()) hf_weights_files = filter_duplicate_safetensors_files(glob(os.path.join(path, "*.safetensors")), path, SAFE_WEIGHTS_INDEX_NAME) for file in tqdm(hf_weights_files): + if load_dummy: + continue print(f"Loading weights from {file}") with safe_open(file, "pt", "cpu") as f: for name in f.keys(): @@ -37,7 +41,20 @@ def load_model(model: nn.Module, model_name_or_path: str): name = name.replace( "weight_scale_inv", "weight_scale" ) + layerId_ = re.search(r'model\.layers\.(\d+)\.', name) + layerId = int(layerId_.group(1)) if layerId_ else 0 + if hf_config.num_hidden_layers and layerId >= hf_config.num_hidden_layers: + # print(f"Skipping loading {name} as layerId {layerId} >= num_hidden_layers {hf_config.num_hidden_layers}") + continue + if (is_rocm_aiter_fusion_shared_expert_enabled() + and "mlp.shared_experts" in name): + name = name.replace( + "mlp.shared_experts", + f"mlp.experts.{hf_config.n_routed_experts}") for k in packed_modules_mapping: + # We handle the experts below in expert_params_mapping + if ("mlp.experts." in name and name not in params_dict): + continue if k in name: v, shard_id = packed_modules_mapping[k] param_name = name.replace(k, v) diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py new file mode 100644 index 0000000000..2e0a1c5cee --- /dev/null +++ b/atom/model_ops/attention_mla.py @@ -0,0 +1,315 @@ +import torch +from torch import nn +from atom.utils.context import get_context, Context +from atom.model_ops.linear import ColumnParallelLinear, RowParallelLinear +from atom.model_ops.utils import get_and_maybe_dequant_weights +from aiter.rotary_embedding import RotaryEmbedding +from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip + batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as aiter_triton_fp8_bmm, +) +from vllm import _custom_ops as ops + +from aiter import flash_attn_varlen_func, dtypes +from aiter.mla import mla_decode_fwd + +from typing import Optional +from tqdm import tqdm +import logging + +from aiter.dist.parallel_state import ( + get_pp_group, + get_tp_group, + get_tensor_model_parallel_world_size, +) + +logger = logging.getLogger("atom") + + +def dynamic_per_batched_tensor_quant( + x: torch.Tensor, dtype: torch.dtype = torch.float8_e4m3fn +): + DTYPE_MAX = torch.finfo(dtype).max + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-10) + scale = DTYPE_MAX / amax + x_scl_sat = (x * scale).clamp(min=-DTYPE_MAX, max=DTYPE_MAX) + return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal() + + +class MLAAttention(nn.Module): + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + kv_cache_dtype: str, + # MLA Specific Arguments + q_lora_rank: Optional[int], + kv_lora_rank: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + qk_head_dim: int, + v_head_dim: int, + rotary_emb: RotaryEmbedding, + q_proj: ColumnParallelLinear, + kv_b_proj: ColumnParallelLinear, + o_proj: RowParallelLinear, + ) -> None: + super().__init__() + self.num_heads = num_heads + self.head_size = head_size + self.scale = float(scale) + self.num_kv_heads = num_kv_heads + self.kv_cache_dtype = kv_cache_dtype if kv_cache_dtype == "fp8" else "auto" + + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_head_dim + self.v_head_dim = v_head_dim + self.rotary_emb = rotary_emb + self.q_proj = q_proj + self.o_proj = o_proj + self.kv_b_proj = kv_b_proj + self.kv_cache = torch.tensor([]) + self._k_scale = torch.tensor(1.0, dtype=torch.float32) + + def process_weights_after_loading(self): + kv_b_proj_weight = get_and_maybe_dequant_weights(self.kv_b_proj).T + assert kv_b_proj_weight.shape == ( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + ), ( + f"{kv_b_proj_weight.shape=}, " + f"{self.kv_lora_rank=}, " + f"{self.num_heads=}, " + f"{self.qk_nope_head_dim=}, " + f"{self.v_head_dim=}" + ) + kv_b_proj_weight = kv_b_proj_weight.view( + self.kv_lora_rank, + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + ) + W_UK, W_UV = kv_b_proj_weight.split( + [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + ) + + if True: # is_rocm_aiter_fp8bmm_enabled(): + W_K = W_UK.transpose(0, 1) # 16 512 128 + W_V = W_UV.permute(1, 2, 0) # 16 128 512 + self.W_K, self.W_K_scale = dynamic_per_batched_tensor_quant( + W_K, dtype=dtypes.fp8 + ) + self.W_V, self.W_V_scale = dynamic_per_batched_tensor_quant( + W_V, dtype=dtypes.fp8 + ) + + # The kernel operates on non-padded inputs. Hence, pre-compiling + # triton kernel to avoid runtime compilation for unseen batch sizes + # Pre-compile for batch sizes 1 to 1024 to cover most use-cases. + # On DS-R1, this step adds roughly 50s to the model loading time. + max_batch_size = 1024 # [ToDo] Find the optimal upper limit + pre_compilation_list = list(range(1, max_batch_size + 1)) + if get_tp_group().is_first_rank: + pre_compilation_list = tqdm( + pre_compilation_list, + desc="[Aiter Triton] Pre-compiling fp8 BMM kernel", + total=max_batch_size, + ) + + for m in pre_compilation_list: + x = torch.empty( + (self.W_K.shape[0], m, self.W_K.shape[2]), + dtype=torch.bfloat16, + device=self.W_K.device, + ) + aiter_triton_fp8_bmm( + x, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True + ) + + x = torch.empty( + (self.W_V.shape[0], m, self.W_V.shape[2]), + dtype=torch.bfloat16, + device=self.W_V.device, + ) + aiter_triton_fp8_bmm( + x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True + ) + else: + # Convert from (L, N, V) to (N, L, V) + self.W_UV = W_UV.transpose(0, 1) + # Convert from (L, N, P) to (N, P, L) + self.W_UK_T = W_UK.permute(1, 2, 0) + + def _aiter_mla_decode_fwd( + self, + q: torch.Tensor, + kv_buffer: torch.Tensor, + o: torch.Tensor, + qo_indptr: torch.Tensor, + max_seqlen_qo: int, + kv_indptr: Optional[torch.Tensor] = None, + kv_indices: Optional[torch.Tensor] = None, + kv_last_page_lens: Optional[torch.Tensor] = None, + sm_scale: float = 1.0, + logit_cap: float = 0.0, + ) -> None: + mla_decode_fwd( + q, + kv_buffer.view(-1, 1, 1, q.shape[-1]), + o, + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_lens, + max_seqlen_qo, + sm_scale=sm_scale, + logit_cap=logit_cap, + ) + + def _v_up_proj_and_o_proj(self, x): + # Convert from (B, N, L) to (N, B, L) + x = x.view(-1, self.num_heads, self.kv_lora_rank).transpose(0, 1) + # Multiply (N, B, L) x (N, L, V) -> (N, B, V), Convert from (N, B, V) to (B, N, V) + # x = torch.bmm(x, self.W_UV).transpose(0, 1) + x = aiter_triton_fp8_bmm( + x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True + ) + # Convert from (B, N, V) to (B, N * V) + x = x.reshape(-1, self.num_heads * self.v_head_dim) + return self.o_proj(x) + + def _q_proj_and_k_up_proj(self, x): + q_nope, q_pe = ( + self.q_proj(x) + .view(-1, self.num_heads, self.qk_head_dim) + .split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + ) + + # Convert from (B, N, P) to (N, B, P) + q_nope = q_nope.transpose(0, 1) + # Multiply (N, B, P) x (N, P, L) -> (N, B, L), Convert from (N, B, L) to (B, N, L) + # ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) + ql_nope = aiter_triton_fp8_bmm( + q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True + ) + return ql_nope, q_pe + + def _forward_prefill( + self, + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + context: Context, + ) -> torch.Tensor: + assert context is not None + + kv_nope = self.kv_b_proj(kv_c_normed).view( + -1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim + ) + k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + + if k_pe.dim() == 2: + k_pe = k_pe.unsqueeze(1) + k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1) + + output = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=context.cu_seqlens_q, + cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q=context.min_seqlen_q, + dropout_p=context.dropout_p, + softmax_scale=self.scale, + causal=True, + ) + + return self.o_proj(output.flatten(start_dim=-2)) + + def _forward_decode( + self, + q_nope: torch.Tensor, + q_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + context: Context, + ) -> torch.Tensor: + assert kv_c_and_k_pe_cache.numel() > 0 + assert context is not None + B = q_nope.shape[0] + + q = torch.cat([q_nope, q_pe], dim=-1) + o = torch.empty( + B, self.num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device + ) + + kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2) + + self._aiter_mla_decode_fwd( + q, + kv_buffer, + o, + context.cu_seqlens_q, + context.max_q_len, + context.kv_indptr, + context.kv_indices, + context.kv_last_page_lens, + self.scale, + ) + + return self._v_up_proj_and_o_proj(o) + + def forward( + self, + q: torch.Tensor, # query in unified attn + k_c_normed: torch.Tensor, # key in unified attn + k_pe: torch.Tensor, # value in unified attn + positions: torch.Tensor, + ) -> torch.Tensor: + # TODO: kv cache and context + kv_cache = self.kv_cache + context = get_context() + + if context.is_prefill: + prefill_q = self.q_proj(q).view(-1, self.num_heads, self.qk_head_dim) + prefill_q_pe = prefill_q[..., self.qk_nope_head_dim :] + self.rotary_emb(positions, prefill_q_pe, k_pe) + + if kv_cache.numel() > 0: + ops.concat_and_cache_mla( + k_c_normed, + k_pe.squeeze(1), + kv_cache, + context.slot_mapping.flatten(), + kv_cache_dtype=self.kv_cache_dtype, + scale=self._k_scale, + ) + + output = self._forward_prefill( + prefill_q, k_c_normed, k_pe, kv_cache, context + ) + else: + decode_ql_nope, decode_q_pe = self._q_proj_and_k_up_proj(q) + self.rotary_emb(positions, decode_q_pe, k_pe) + + if kv_cache.numel() > 0: + ops.concat_and_cache_mla( + k_c_normed, + k_pe.squeeze(1), + kv_cache, + context.slot_mapping.flatten(), + kv_cache_dtype=self.kv_cache_dtype, + scale=self._k_scale, + ) + + output = self._forward_decode( + decode_ql_nope, decode_q_pe, kv_cache, context + ) + + return output diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 49c1a91959..b66b5787ad 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -25,7 +25,7 @@ def divide(numerator, denominator): - assert numerator % denominator == 0 + assert numerator % denominator == 0, f"numerator {numerator} denominator {denominator}" return numerator // denominator @@ -38,12 +38,14 @@ def __init__( tp_dim: int | None = None, bias: bool = False, quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = False, ): if quant_config is None: quant_config = QuantizationConfig() quant_type = quant_config["quant_type"] params_dtype = quant_config["quant_dtype"] super().__init__() + self.reduce_results = reduce_results self.input_size = input_size self.output_size = ( output_size if isinstance(output_size, int) else sum(output_size) @@ -97,7 +99,7 @@ def __init__( elif quant_type == QuantType.per_1x128: self.weight_scale = nn.Parameter( torch.empty( - divide(self.output_size, 128), + (self.output_size + 127) // 128, (self.input_size + 127) // 128, dtype=dtypes.fp32, ), @@ -232,7 +234,7 @@ def forward( self.bias, dtype=otype, ) - if self.tp_dim == 1 and self.tp_size > 1: + if self.tp_dim == 1 and self.tp_size > 1 and self.reduce_results: y = get_tp_group().all_reduce(y, ca_fp8_quant=False) return y @@ -400,6 +402,7 @@ def __init__( output_size: int, bias: bool = False, quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, **kwargs, ): self.tp_rank = get_tp_group().rank @@ -409,6 +412,7 @@ def __init__( tp_dim=1, bias=bias if self.tp_rank == 0 else False, quant_config=quant_config, + reduce_results=reduce_results, ) def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 9a65f58202..8e314cb6df 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -7,19 +7,19 @@ from transformers import PretrainedConfig from aiter import QuantType, dtypes from aiter import get_hip_quant -from atom.config import QuantizationConfig +from atom.config import QuantizationConfig, get_current_atom_config from atom.model_ops.base_config import QuantizeMethodBase from atom.model_ops.topK import rocm_aiter_grouped_topk as grouped_topk from atom.model_ops.topK import rocm_aiter_topk_softmax as fused_topk from atom.model_ops.topK import ( init_aiter_topK_meta_data, is_rocm_aiter_fusion_shared_expert_enabled, + is_rocm_aiter_fuse_routed_scaling_factor, ) from atom.model_ops.utils import ( - shuffle_weights, - normalize_e4m3fn_to_e4m3fnuz, - per_tensor_dequantize, - all_close_1d, + shuffle_weights, + normalize_e4m3fn_to_e4m3fnuz, + per_tensor_dequantize, ) from atom.model_loader.weight_utils import set_weight_attrs from aiter.fused_moe import fused_moe @@ -178,7 +178,7 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: if self.block_quant: if self.quant_type == QuantType.per_1x128: - block_n = 1 + block_n = 128 block_k = 128 elif self.quant_type == QuantType.per_1x32: block_n = 1 @@ -252,8 +252,8 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: ), requires_grad=False, ) - layer.register_parameter("w13_weight_scale_inv", w13_weight_scale) - layer.register_parameter("w2_weight_scale_inv", w2_weight_scale) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) assert self.quant_config["is_dynamic"] == True # Add the quantization method used (per tensor/grouped/channel) @@ -296,28 +296,28 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: # TODO (rob): refactor block quant into separate class. if self.block_quant: - assert self.quant_config.activation_scheme == "dynamic" + assert self.quant_config["is_dynamic"] == True if self.need_normalize_e4m3fn_to_e4m3fnuz: - w13_weight, w13_weight_scale_inv, w13_input_scale = \ + w13_weight, w13_weight_scale, w13_input_scale = \ normalize_e4m3fn_to_e4m3fnuz( - layer.w13_weight, layer.w13_weight_scale_inv, + layer.w13_weight, layer.w13_weight_scale, layer.w13_input_scale) - w2_weight, w2_weight_scale_inv, w2_input_scale = \ + w2_weight, w2_weight_scale, w2_input_scale = \ normalize_e4m3fn_to_e4m3fnuz( - layer.w2_weight, layer.w2_weight_scale_inv, + layer.w2_weight, layer.w2_weight_scale, layer.w2_input_scale) else: w13_weight = layer.w13_weight.data - w13_weight_scale_inv = layer.w13_weight_scale_inv.data + w13_weight_scale = layer.w13_weight_scale.data w2_weight = layer.w2_weight - w2_weight_scale_inv = layer.w2_weight_scale_inv + w2_weight_scale = layer.w2_weight_scale # torch.compile() cannot use Parameter subclasses. layer.w13_weight = nn.Parameter(w13_weight, requires_grad=False) - layer.w13_weight_scale_inv = nn.Parameter(w13_weight_scale_inv, + layer.w13_weight_scale = nn.Parameter(w13_weight_scale, requires_grad=False) layer.w2_weight = nn.Parameter(w2_weight, requires_grad=False) - layer.w2_weight_scale_inv = nn.Parameter(w2_weight_scale_inv, + layer.w2_weight_scale = nn.Parameter(w2_weight_scale, requires_grad=False) shuffled_w13, shuffled_w2 = shuffle_weights( @@ -327,59 +327,7 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: requires_grad=False) layer.w2_weight = torch.nn.Parameter(shuffled_w2, requires_grad=False) - - # # DeepGemm scales need to be transposed and aligned. We try to do - # # it ahead of time for performance reasons. - # if self.allow_deep_gemm: - # # Lazy import to avoid CUDA initialization problems. - # import deep_gemm as dg - # if _is_col_major(layer.w13_weight_scale_inv): - # layer.w13_weight_scale_inv = \ - # dg.get_col_major_tma_aligned_tensor(layer.w13_weight_scale_inv).contiguous() - # if _is_col_major(layer.w2_weight_scale_inv): - # layer.w2_weight_scale_inv = \ - # dg.get_col_major_tma_aligned_tensor(layer.w2_weight_scale_inv).contiguous() return - - # # If checkpoint is fp16, quantize in place. - # if not self.quant_config.is_checkpoint_fp8_serialized: - # fp8_dtype = current_platform.fp8_dtype() - # w13_weight = torch.empty_like(layer.w13_weight.data, - # dtype=fp8_dtype) - # w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype) - - # # Re-initialize w13_scale because we directly quantize - # # merged w13 weights and generate a single scaling factor. - # layer.w13_weight_scale = torch.nn.Parameter(torch.ones( - # layer.local_num_experts, - # dtype=torch.float32, - # device=w13_weight.device), - # requires_grad=False) - # for expert in range(layer.local_num_experts): - # w13_weight[expert, :, :], layer.w13_weight_scale[ - # expert] = ops.scaled_fp8_quant( - # layer.w13_weight.data[expert, :, :]) - # w2_weight[expert, :, :], layer.w2_weight_scale[ - # expert] = ops.scaled_fp8_quant( - # layer.w2_weight.data[expert, :, :]) - # layer.w13_weight = torch.nn.Parameter(w13_weight, - # requires_grad=False) - # layer.w2_weight = torch.nn.Parameter(w2_weight, - # requires_grad=False) - # if self.rocm_aiter_moe_enabled: - - # shuffled_w13, shuffled_w2 = shuffle_weights( - # layer.w13_weight, layer.w2_weight) - - # layer.w13_weight = torch.nn.Parameter(shuffled_w13, - # requires_grad=False) - # layer.w2_weight = torch.nn.Parameter(shuffled_w2, - # requires_grad=False) - # return - - # If checkpoint is fp8, we need to handle that the - # MoE kernels require single activation scale and single weight - # scale for w13 per expert. else: # Fp8 moe kernels require a single activation scale. # We take the max of all the scales in case they differ. @@ -496,9 +444,9 @@ def apply( expert_mask=expert_map, activation=activation, quant_type=self.quant_type, - w1_scale=(layer.w13_weight_scale_inv + w1_scale=(layer.w13_weight_scale if self.block_quant else layer.w13_weight_scale), - w2_scale=(layer.w2_weight_scale_inv + w2_scale=(layer.w2_weight_scale if self.block_quant else layer.w2_weight_scale), a1_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale) @@ -607,7 +555,9 @@ def __init__( tp_rank = 0 if self.tp_size == 1 else get_tp_group().rank_in_group self.global_num_experts = num_experts - # Use expert parallelism instead of tensor parallelism? + # TODO: This slows load + # atom_config = get_current_atom_config() + # use_ep = self.tp_size > 1 and atom_config.enable_expert_parallel use_ep = self.tp_size > 1 if use_ep: @@ -635,7 +585,7 @@ def __init__( self.num_fused_shared_experts = ( config.n_shared_experts - if config is not None and is_rocm_aiter_fusion_shared_expert_enabled() + if config is not None and hasattr(config, 'n_shared_experts') and is_rocm_aiter_fusion_shared_expert_enabled() else 0 ) self.routed_scaling_factor = ( @@ -675,8 +625,8 @@ def __init__( top_k=self.top_k, tp_rank=self.ep_rank if use_ep else tp_rank, tp_size=self.ep_size if use_ep else tp_size, - shared_experts_score=1.0, - max_num_tokens=config.max_num_batched_tokens, # Default max tokens + shared_experts_score=1.0 if is_rocm_aiter_fuse_routed_scaling_factor() else 1/self.routed_scaling_factor, + max_num_tokens=16384, # Default max tokens is_EP=use_ep, ) if is_rocm_aiter_fusion_shared_expert_enabled(): diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index 625f6d5881..2b3420f849 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -4,8 +4,10 @@ def is_rocm_aiter_fusion_shared_expert_enabled(): - return False + return True +def is_rocm_aiter_fuse_routed_scaling_factor(): + return False aiter_topK_meta_data = None diff --git a/atom/model_ops/utils.py b/atom/model_ops/utils.py index 6caf87e756..d0bf292575 100644 --- a/atom/model_ops/utils.py +++ b/atom/model_ops/utils.py @@ -1,10 +1,8 @@ from typing import Tuple, Optional, List, Union import torch -from torch.library import Library -from typing import Callable, Optional, Tuple -from aiter import per_tensor_quant, dtypes +from aiter import per_tensor_quant, dtypes, QuantType from aiter.ops.shuffle import shuffle_weight -from aiter.ops.quant import per_tensor_quant +from torch import nn def per_tensor_dequantize( tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor] @@ -81,15 +79,15 @@ def shuffle_weights( *tensors: torch.Tensor, layout: tuple[int, int] = (16, 16) ) -> tuple[torch.Tensor, ...]: """ - Applies shuffle_weight function from AITER to each + Applies shuffle_weight function from AITER to each input tensor and returns them. - + Rearranges (shuffles) the input tensor/s into a specified block layout for optimized computation. Args: *tensors: Variable number of torch.Tensor objects. - layout: A pair of integers specifying the + layout: A pair of integers specifying the block sizes used to divide the tensors during shuffling. Default is (16, 16). @@ -109,4 +107,19 @@ def per_tensor_dequantize( torch.Tensor]) -> torch.Tensor: fake_qweight = tensor.to(torch.float16) dq_weight = fake_qweight * inv_scale - return dq_weight \ No newline at end of file + return dq_weight + + +def get_and_maybe_dequant_weights(layer: nn.Module) -> torch.Tensor: + if layer.quant_type != QuantType.No: + # NOTE: This should only be used offline, since it's O(N^3) + eye = torch.eye( + layer.input_size, + dtype=torch.bfloat16, + device=layer.weight.device, + ) + dequant_weights = layer(eye) + del eye + # standardize to (output, input) + return dequant_weights.T + return layer.weight diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index 71225dff2c..41d8dcc241 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -1,4 +1,15 @@ -# Copyright 2023-2024 SGLang Team +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py +# Copyright 2023 The vLLM team. +# Copyright 2023 DeepSeek-AI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,110 +21,44 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# ============================================================================== - -# Adapted from: -# https://github.com/vllm-project/vllm/blob/fb6af8bc086328ca6659e72d11ffd4309ce4de22/vllm/model_executor/models/deepseek_v2.py -"""Inference-only DeepseekV2 model.""" - -import logging -import os -from dataclasses import dataclass -from enum import Enum, IntEnum, auto -from typing import Any, Dict, Iterable, Optional, Tuple +"""Inference-only DeepseekV2/DeepseekV3 model.""" +from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union import torch -import torch.nn.functional as F from torch import nn -from tqdm import tqdm from transformers import PretrainedConfig -from sglang.srt.distributed import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, - parallel_state, - tensor_model_parallel_all_reduce, -) -from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.dp_attention import ( - dp_gather_partial, - dp_scatter, - get_attention_dp_size, - get_attention_tp_rank, - get_attention_tp_size, - tp_all_gather, - tp_reduce_scatter, -) -from sglang.srt.layers.layernorm import RMSNorm -from sglang.srt.layers.linear import ( - ColumnParallelLinear, +from atom.model_ops.attention_mla import MLAAttention +from atom.config import QuantizationConfig, Config +from aiter.dist.parallel_state import (get_pp_group, + get_tensor_model_parallel_world_size) +from aiter.dist.communication_op import tensor_model_parallel_all_reduce +from atom.model_ops.activation import SiluAndMul +from atom.model_ops.moe import FusedMoE +from atom.model_ops.layernorm import RMSNorm +from atom.model_ops.linear import (ColumnParallelLinear, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, ) -from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, EPMoE -from sglang.srt.layers.moe.ep_moe.token_dispatcher import DeepEPDispatcher -from sglang.srt.layers.moe.fused_moe_triton import FusedMoE -from sglang.srt.layers.moe.topk import select_experts -from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.layers.quantization.fp8_kernel import ( - _enable_jit_deepgemm_bmm, - per_tensor_quant_mla_deep_gemm_masked_fp8, - per_tensor_quant_mla_fp8, +from aiter.rotary_embedding import get_rope +from atom.model_ops.embed_head import ( + ParallelLMHead, VocabParallelEmbedding) + +from atom.models.utils import (PPMissingLayer, + make_empty_intermediate_tensors_factory, make_layers, + maybe_prefix, + IntermediateTensors, ) -from sglang.srt.layers.quantization.fp8_utils import ( - block_quant_to_tensor_quant, - channel_quant_to_tensor_quant, - normalize_e4m3fn_to_e4m3fnuz, -) -from sglang.srt.layers.quantization.int8_utils import ( - block_dequant as int8_block_dequant, -) -from sglang.srt.layers.radix_attention import RadixAttention -from sglang.srt.layers.rotary_embedding import get_rope, get_rope_wrapper -from sglang.srt.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from sglang.srt.managers.expert_distribution import ExpertDistributionRecorder -from sglang.srt.managers.schedule_batch import global_server_args_dict -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode -from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.utils import BumpAllocator, DeepEPMode, add_prefix, is_cuda, is_hip - -_is_hip = is_hip() -_is_cuda = is_cuda() - -if _is_cuda: - from deep_gemm import m_grouped_gemm_fp8_fp8_bf16_nt_masked - from sgl_kernel import awq_dequantize, bmm_fp8, merge_state_v2 -else: - from vllm._custom_ops import awq_dequantize - -if _is_hip: - from sglang.srt.layers.attention.triton_ops.rocm_mla_decode_rope import ( - decode_attention_fwd_grouped_rope, - ) -expert_distribution_recorder = ExpertDistributionRecorder() - -logger = logging.getLogger(__name__) - - -class AttnForwardMethod(IntEnum): - # Use multi-head attention - MHA = auto() - - # Use absorbed multi-latent attention - MLA = auto() - - # Use multi-head attention, but with KV cache chunked. - # This method can avoid OOM when prefix lengths are long. - MHA_CHUNKED_KV = auto() +from atom.model_ops.topK import ( + is_rocm_aiter_fusion_shared_expert_enabled, + is_rocm_aiter_fuse_routed_scaling_factor +) class DeepseekV2MLP(nn.Module): + def __init__( self, hidden_size: int, @@ -122,65 +67,31 @@ def __init__( quant_config: Optional[QuantizationConfig] = None, reduce_results: bool = True, prefix: str = "", - tp_rank: Optional[int] = None, - tp_size: Optional[int] = None, ) -> None: super().__init__() self.gate_up_proj = MergedColumnParallelLinear( - hidden_size, - [intermediate_size] * 2, - bias=False, - quant_config=quant_config, - prefix=add_prefix("gate_up_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, - ) - self.down_proj = RowParallelLinear( - intermediate_size, - hidden_size, + hidden_size, [intermediate_size] * 2, bias=False, quant_config=quant_config, - reduce_results=reduce_results, - prefix=add_prefix("down_proj", prefix), - tp_rank=tp_rank, - tp_size=tp_size, - ) + prefix=f"{prefix}.gate_up_proj") + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj") if hidden_act != "silu": - raise ValueError( - f"Unsupported activation: {hidden_act}. " - "Only silu is supported for now." - ) + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") self.act_fn = SiluAndMul() - def forward(self, x, forward_mode: Optional[ForwardMode] = None): - gate_up, _ = self.gate_up_proj(x) + def forward(self, x): + gate_up = self.gate_up_proj(x) x = self.act_fn(gate_up) - x, _ = self.down_proj(x) + x = self.down_proj(x) return x -class MoEGate(nn.Module): - def __init__( - self, - config, - prefix: str = "", - ): - super().__init__() - self.weight = nn.Parameter( - torch.empty((config.n_routed_experts, config.hidden_size)) - ) - if config.topk_method == "noaux_tc": - self.e_score_correction_bias = nn.Parameter( - torch.empty((config.n_routed_experts)) - ) - else: - self.e_score_correction_bias = None - - def forward(self, hidden_states): - logits = F.linear(hidden_states, self.weight, None) - return logits - - class DeepseekV2MoE(nn.Module): def __init__( @@ -193,137 +104,92 @@ def __init__( self.tp_size = get_tensor_model_parallel_world_size() self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts - self.n_share_experts_fusion = global_server_args_dict["n_share_experts_fusion"] - - if self.tp_size > config.n_routed_experts: - raise ValueError( - f"Tensor parallel size {self.tp_size} is greater than " - f"the number of experts {config.n_routed_experts}." - ) if config.hidden_act != "silu": - raise ValueError( - f"Unsupported activation: {config.hidden_act}. " - "Only silu is supported for now." - ) - - self.gate = MoEGate(config=config, prefix=add_prefix("gate", prefix)) - - MoEImpl = ( - DeepEPMoE - if global_server_args_dict["enable_deepep_moe"] - else (EPMoE if global_server_args_dict["enable_ep_moe"] else FusedMoE) - ) + raise ValueError(f"Unsupported activation: {config.hidden_act}. " + "Only silu is supported for now.") + + self.gate = ReplicatedLinear(config.hidden_size, + config.n_routed_experts, + bias=False, + quant_config=None, + prefix=f"{prefix}.gate") + if config.topk_method == "noaux_tc": + self.gate.e_score_correction_bias = nn.Parameter( + torch.empty(config.n_routed_experts)) + else: + self.gate.e_score_correction_bias = None - self.experts = MoEImpl( - num_experts=config.n_routed_experts + self.n_share_experts_fusion, - top_k=config.num_experts_per_tok + min(self.n_share_experts_fusion, 1), + self.experts = FusedMoE( + num_experts=config.n_routed_experts, + top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, + reduce_results=False, renormalize=config.norm_topk_prob, quant_config=quant_config, use_grouped_topk=True, num_expert_group=config.n_group, topk_group=config.topk_group, - correction_bias=self.gate.e_score_correction_bias, - routed_scaling_factor=self.routed_scaling_factor, - prefix=add_prefix("experts", prefix), - **( - dict(deepep_mode=DeepEPMode[global_server_args_dict["deepep_mode"]]) - if global_server_args_dict["enable_deepep_moe"] - else {} - ), - ) - - if config.n_shared_experts is not None and self.n_share_experts_fusion == 0: - intermediate_size = config.moe_intermediate_size * config.n_shared_experts - # disable tp for shared experts when enable deepep moe - if not global_server_args_dict["enable_deepep_moe"]: - self.shared_experts = DeepseekV2MLP( - hidden_size=config.hidden_size, - intermediate_size=intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - reduce_results=False, - prefix=add_prefix("shared_experts", prefix), - ) - else: - self.shared_experts = DeepseekV2MLP( - hidden_size=config.hidden_size, - intermediate_size=intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - reduce_results=False, - prefix=add_prefix("shared_experts", prefix), - tp_rank=0, - tp_size=1, - ) - - if global_server_args_dict["enable_deepep_moe"]: - # TODO: we will support tp < ep in the future - self.ep_size = get_tensor_model_parallel_world_size() - self.num_experts = config.n_routed_experts - self.top_k = config.num_experts_per_tok - self.renormalize = config.norm_topk_prob - self.topk_group = config.topk_group - self.num_expert_group = config.n_group - self.correction_bias = ( - self.gate.e_score_correction_bias.data - if self.gate.e_score_correction_bias is not None - else None - ) - - self.deepep_dispatcher = DeepEPDispatcher( - group=parallel_state.get_tp_group().device_group, - router_topk=self.top_k, - permute_fusion=True, - num_experts=config.n_routed_experts, - num_local_experts=config.n_routed_experts // self.tp_size, + prefix=f"{prefix}.experts", + scoring_func=config.scoring_func, + e_score_correction_bias=self.gate.e_score_correction_bias, + config=config) + + if config.n_shared_experts is not None and not is_rocm_aiter_fusion_shared_expert_enabled(): + intermediate_size = (config.moe_intermediate_size * + config.n_shared_experts) + self.shared_experts = DeepseekV2MLP( hidden_size=config.hidden_size, - params_dtype=config.torch_dtype, - deepep_mode=DeepEPMode[global_server_args_dict["deepep_mode"]], - async_finish=True, # TODO - return_recv_hook=True, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=False, + prefix=f"{prefix}.shared_experts", ) - def forward( - self, hidden_states: torch.Tensor, forward_mode: Optional[ForwardMode] = None - ) -> torch.Tensor: - if not global_server_args_dict["enable_deepep_moe"]: - return self.forward_normal(hidden_states) - else: - return self.forward_deepep(hidden_states, forward_mode) - - def forward_normal(self, hidden_states: torch.Tensor) -> torch.Tensor: - shared_output = self._forward_shared_experts(hidden_states) + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + shared_output = None + if self.n_shared_experts is not None and not is_rocm_aiter_fusion_shared_expert_enabled(): + shared_output = self.shared_experts(hidden_states) # router_logits: (num_tokens, n_experts) router_logits = self.gate(hidden_states) - final_hidden_states = ( - self.experts(hidden_states=hidden_states, router_logits=router_logits) - * self.routed_scaling_factor - ) + if hidden_states.dtype != torch.float16: + final_hidden_states = self.experts( + hidden_states=hidden_states, + router_logits=router_logits) + if not is_rocm_aiter_fuse_routed_scaling_factor(): + final_hidden_states = final_hidden_states * self.routed_scaling_factor + else: + # Fix FP16 overflow + # See DeepseekV2DecoderLayer for more details. + final_hidden_states = self.experts(hidden_states=hidden_states, + router_logits=router_logits) if shared_output is not None: - final_hidden_states = final_hidden_states + shared_output + if hidden_states.dtype != torch.float16: + final_hidden_states = final_hidden_states + shared_output + else: + # Fix FP16 overflow + # See DeepseekV2DecoderLayer for more details. + final_hidden_states = final_hidden_states + shared_output \ + * (1. / self.routed_scaling_factor) if self.tp_size > 1: - final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) - return final_hidden_states + final_hidden_states = tensor_model_parallel_all_reduce( + final_hidden_states) - def _forward_shared_experts(self, hidden_states): - if self.n_share_experts_fusion == 0: - return self.shared_experts(hidden_states) - else: - return None + return final_hidden_states.view(num_tokens, hidden_dim) def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: import math - if scale <= 1: return 1.0 return 0.1 * mscale * math.log(scale) + 1.0 -class DeepseekV2AttentionMLA(nn.Module): +class DeepseekV2Attention(nn.Module): def __init__( self, @@ -338,13 +204,11 @@ def __init__( rope_theta: float = 10000, rope_scaling: Optional[Dict[str, Any]] = None, max_position_embeddings: int = 8192, + cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, - reduce_results: bool = True, - layer_id: int = None, prefix: str = "", ) -> None: super().__init__() - self.layer_id = layer_id self.hidden_size = hidden_size self.qk_nope_head_dim = qk_nope_head_dim self.qk_rope_head_dim = qk_rope_head_dim @@ -352,639 +216,270 @@ def __init__( self.v_head_dim = v_head_dim self.q_lora_rank = q_lora_rank self.kv_lora_rank = kv_lora_rank - self.dp_size = get_attention_dp_size() - attn_tp_rank = get_attention_tp_rank() - attn_tp_size = get_attention_tp_size() - self.num_heads = num_heads - assert num_heads % attn_tp_size == 0 - self.num_local_heads = num_heads // attn_tp_size + tp_size = get_tensor_model_parallel_world_size() + assert num_heads % tp_size == 0 + self.num_local_heads = num_heads // tp_size self.scaling = self.qk_head_dim**-0.5 self.rope_theta = rope_theta self.max_position_embeddings = max_position_embeddings - # For tensor parallel attention if self.q_lora_rank is not None: - self.q_a_proj = ReplicatedLinear( - self.hidden_size, - self.q_lora_rank, - bias=False, - quant_config=quant_config, - prefix=add_prefix("q_a_proj", prefix), - ) - self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) - self.q_b_proj = ColumnParallelLinear( - q_lora_rank, - self.num_heads * self.qk_head_dim, - bias=False, - quant_config=quant_config, - prefix=add_prefix("q_b_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, - ) + self.q_a_proj = ReplicatedLinear(self.hidden_size, + self.q_lora_rank, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_a_proj") + self.q_a_layernorm = RMSNorm(self.q_lora_rank, + eps=config.rms_norm_eps) + self.q_b_proj = ColumnParallelLinear(q_lora_rank, + self.num_heads * + self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_b_proj") else: - self.q_proj = ColumnParallelLinear( - self.hidden_size, - self.num_heads * self.qk_head_dim, - bias=False, - quant_config=quant_config, - prefix=add_prefix("q_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, - ) - self.kv_b_proj = ColumnParallelLinear( - self.kv_lora_rank, - self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), - bias=False, - quant_config=quant_config, - prefix=add_prefix("kv_b_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, - ) - # O projection. - self.o_proj = RowParallelLinear( - self.num_heads * self.v_head_dim, - self.hidden_size, - bias=False, - quant_config=quant_config, - reduce_results=reduce_results, - prefix=add_prefix("o_proj", prefix), - tp_rank=attn_tp_rank, - tp_size=attn_tp_size, - ) + self.q_proj = ColumnParallelLinear(self.hidden_size, + self.num_heads * + self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj") self.kv_a_proj_with_mqa = ReplicatedLinear( self.hidden_size, self.kv_lora_rank + self.qk_rope_head_dim, bias=False, quant_config=quant_config, - prefix=add_prefix("kv_a_proj_with_mqa", prefix), - ) - self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps) - + prefix=f"{prefix}.kv_a_proj_with_mqa") + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, + eps=config.rms_norm_eps) + self.kv_b_proj = ColumnParallelLinear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_b_proj") + # O projection. + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj") if rope_scaling: - rope_scaling["rope_type"] = "deepseek_yarn" + rope_scaling["rope_type"] = 'deepseek_yarn' - self.rotary_emb = get_rope( - qk_rope_head_dim, - rotary_dim=qk_rope_head_dim, - max_position=max_position_embeddings, - base=rope_theta, - rope_scaling=rope_scaling, - is_neox_style=False, - ) + self.rotary_emb = get_rope(qk_rope_head_dim, + rotary_dim=qk_rope_head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=False) if rope_scaling: mscale_all_dim = rope_scaling.get("mscale_all_dim", False) scaling_factor = rope_scaling["factor"] mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale - else: - self.rotary_emb.forward = self.rotary_emb.forward_native - - self.attn_mqa = RadixAttention( - self.num_local_heads, - self.kv_lora_rank + self.qk_rope_head_dim, - self.scaling, - num_kv_heads=1, - layer_id=layer_id, - v_head_dim=self.kv_lora_rank, - quant_config=quant_config, - prefix=add_prefix("attn_mqa", prefix), - ) - self.attn_mha = RadixAttention( - self.num_local_heads, - self.qk_nope_head_dim + self.qk_rope_head_dim, - self.scaling, - num_kv_heads=self.num_local_heads, - layer_id=layer_id, - v_head_dim=self.v_head_dim, - quant_config=quant_config, - prefix=add_prefix("attn_mha", prefix), - ) - - self.w_kc = None - self.w_vc = None - self.w_scale = None - - self.w_scale_k = None - self.w_scale_v = None - self.use_deep_gemm_bmm = False - - self.flashinfer_mla_disable_ragged = global_server_args_dict[ - "flashinfer_mla_disable_ragged" - ] - self.disable_chunked_prefix_cache = global_server_args_dict[ - "disable_chunked_prefix_cache" - ] - self.attention_backend = global_server_args_dict["attention_backend"] - self.rocm_fused_decode_mla = os.getenv("SGLANG_ROCM_FUSED_DECODE_MLA") == "1" - - # TODO: Design a finer way to determine the threshold - self.chunked_prefix_cache_threshold = 8192 - - def dispatch_attn_forward_method( - self, forward_batch: ForwardBatch - ) -> AttnForwardMethod: - if self.attention_backend == "flashinfer": - # Flashinfer MLA: Do not absorb when enabling ragged prefill - if ( - not self.flashinfer_mla_disable_ragged - and forward_batch.forward_mode.is_extend() - and not forward_batch.forward_mode.is_target_verify() - and not forward_batch.forward_mode.is_draft_extend() - and sum(forward_batch.extend_prefix_lens_cpu) == 0 - ): - return AttnForwardMethod.MHA - else: - return AttnForwardMethod.MLA - elif self.attention_backend == "fa3": - # Flash Attention: Use MHA with chunked KV cache when prefilling on long sequences. - if ( - forward_batch.forward_mode.is_extend() - and not self.disable_chunked_prefix_cache - and not forward_batch.forward_mode.is_target_verify() - and not forward_batch.forward_mode.is_draft_extend() - and sum(forward_batch.extend_prefix_lens_cpu) - >= self.chunked_prefix_cache_threshold - ): - return AttnForwardMethod.MHA_CHUNKED_KV - else: - return AttnForwardMethod.MLA - else: - # Triton: Use normal computation for prefill and use weight absorption for extend/decode - if ( - forward_batch.forward_mode.is_extend() - and not forward_batch.forward_mode.is_target_verify() - and not forward_batch.forward_mode.is_draft_extend() - and sum(forward_batch.extend_prefix_lens_cpu) == 0 - ): - return AttnForwardMethod.MHA - else: - return AttnForwardMethod.MLA + self.attn = Attention(self.num_local_heads, + self.qk_head_dim, + self.scaling, + num_kv_heads=self.num_local_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn") def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - ) -> torch.Tensor: - if hidden_states.shape[0] == 0: - assert ( - not self.o_proj.reduce_results - ), "short-circuiting allreduce will lead to hangs" - return hidden_states - - attn_forward_method = self.dispatch_attn_forward_method(forward_batch) - - if attn_forward_method == AttnForwardMethod.MHA: - return self.forward_normal(positions, hidden_states, forward_batch) - elif attn_forward_method == AttnForwardMethod.MHA_CHUNKED_KV: - return self.forward_normal_chunked_kv( - positions, hidden_states, forward_batch - ) - else: - return self.forward_absorb( - positions, hidden_states, forward_batch, zero_allocator - ) - - def forward_normal( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, ) -> torch.Tensor: if self.q_lora_rank is not None: q = self.q_a_proj(hidden_states)[0] q = self.q_a_layernorm(q) - q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) + q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, + self.qk_head_dim) else: - q = self.q_proj(hidden_states)[0].view( - -1, self.num_local_heads, self.qk_head_dim - ) - _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads, + self.qk_head_dim) + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], + dim=-1) latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] - kv_a, _ = latent_cache.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + kv_a, _ = latent_cache.split( + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) latent_cache = latent_cache.unsqueeze(1) kv_a = self.kv_a_layernorm(kv_a.contiguous()) kv = self.kv_b_proj(kv_a)[0] - kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim) - k_nope = kv[..., : self.qk_nope_head_dim] - v = kv[..., self.qk_nope_head_dim :] - k_pe = latent_cache[:, :, self.kv_lora_rank :] - q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - q[..., self.qk_nope_head_dim :] = q_pe - k = torch.empty_like(q) - k[..., : self.qk_nope_head_dim] = k_nope - k[..., self.qk_nope_head_dim :] = k_pe + kv = kv.view(-1, self.num_local_heads, + self.qk_nope_head_dim + self.v_head_dim) + k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + k_pe = latent_cache[:, :, self.kv_lora_rank:] - latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1) - latent_cache[:, :, self.kv_lora_rank :] = k_pe + q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - # Save latent cache - forward_batch.token_to_kv_pool.set_kv_buffer( - self.attn_mha, forward_batch.out_cache_loc, latent_cache, None - ) - attn_output = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False) - attn_output = attn_output.reshape(-1, self.num_local_heads * self.v_head_dim) + q[..., self.qk_nope_head_dim:] = q_pe + k = torch.empty_like(q) + k[..., :self.qk_nope_head_dim] = k_nope + k[..., self.qk_nope_head_dim:] = k_pe + # padding value to qk_head_dim for alignment + v = torch.nn.functional.pad( + v, [0, self.qk_head_dim - self.v_head_dim], + value=0).view(-1, self.num_local_heads * self.qk_head_dim) + attn_output = self.attn(q, k, v) + attn_output = attn_output.view( + -1, self.num_local_heads, + self.qk_head_dim)[..., :self.v_head_dim].reshape( + -1, self.num_local_heads * self.v_head_dim) output, _ = self.o_proj(attn_output) return output - def forward_absorb( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - ) -> torch.Tensor: - q_len = hidden_states.shape[0] - q_input = hidden_states.new_empty( - q_len, self.num_local_heads, self.kv_lora_rank + self.qk_rope_head_dim - ) - if self.q_lora_rank is not None: - q = self.q_a_proj(hidden_states)[0] - q = self.q_a_layernorm(q) - q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) - else: - q = self.q_proj(hidden_states)[0].view( - -1, self.num_local_heads, self.qk_head_dim - ) - q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - if self.use_deep_gemm_bmm: - q_nope_val, q_nope_scale, masked_m, expected_m, aligned_m = ( - per_tensor_quant_mla_deep_gemm_masked_fp8( - q_nope.transpose(0, 1), dtype=torch.float8_e4m3fn - ) - ) - q_nope_out = q_nope.new_empty( - (self.num_local_heads, aligned_m, self.kv_lora_rank) - ) - m_grouped_gemm_fp8_fp8_bf16_nt_masked( - (q_nope_val, q_nope_scale), - (self.w_kc, self.w_scale_k), - q_nope_out, - masked_m, - expected_m, - ) - q_nope_out = q_nope_out[:, :expected_m, :] - elif self.w_kc.dtype == torch.float8_e4m3fnuz: - # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz - q_nope_out = torch.bmm( - q_nope.to(torch.bfloat16).transpose(0, 1), - self.w_kc.to(torch.bfloat16) * self.w_scale, - ) - elif self.w_kc.dtype == torch.float8_e4m3fn: - q_nope_val, q_nope_scale = per_tensor_quant_mla_fp8( - q_nope.transpose(0, 1), - zero_allocator.allocate(1), - ) - q_nope_out = bmm_fp8( - q_nope_val, self.w_kc, q_nope_scale, self.w_scale, torch.bfloat16 - ) - else: - q_nope_out = torch.bmm(q_nope.transpose(0, 1), self.w_kc) - q_input[..., : self.kv_lora_rank] = q_nope_out.transpose(0, 1) +class DeepseekV2MLAAttention(nn.Module): + """ + Main reference: DeepseekV2 paper, and FlashInfer Implementation + (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551). - latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] - v_input = latent_cache[..., : self.kv_lora_rank] - v_input = self.kv_a_layernorm(v_input.contiguous()).unsqueeze(1) - k_input = latent_cache.unsqueeze(1) - k_input[..., : self.kv_lora_rank] = v_input - k_pe = k_input[..., self.kv_lora_rank :] + For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py + """ - q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - q_input[..., self.kv_lora_rank :] = q_pe - k_input[..., self.kv_lora_rank :] = k_pe + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + v_head_dim: int, + q_lora_rank: Optional[int], + kv_lora_rank: int, + rope_theta: float = 10000, + rope_scaling: Optional[Dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: str = "bf16", + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim - attn_output = self.attn_mqa(q_input, k_input, v_input, forward_batch) - attn_output = attn_output.view(-1, self.num_local_heads, self.kv_lora_rank) + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank - if self.use_deep_gemm_bmm: - attn_output_val, attn_output_scale, masked_m, expected_m, aligned_m = ( - per_tensor_quant_mla_deep_gemm_masked_fp8( - attn_output.transpose(0, 1), dtype=torch.float8_e4m3fn - ) - ) - attn_bmm_output = attn_output.new_empty( - (self.num_local_heads, aligned_m, self.v_head_dim) - ) - m_grouped_gemm_fp8_fp8_bf16_nt_masked( - (attn_output_val, attn_output_scale), - (self.w_vc, self.w_scale_v), - attn_bmm_output, - masked_m, - expected_m, - ) - attn_bmm_output = attn_bmm_output[:, :expected_m, :] - elif self.w_vc.dtype == torch.float8_e4m3fnuz: - # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz - attn_bmm_output = torch.bmm( - attn_output.to(torch.bfloat16).transpose(0, 1), - self.w_vc.to(torch.bfloat16) * self.w_scale, - ) - elif self.w_vc.dtype == torch.float8_e4m3fn: - attn_output_val, attn_output_scale = per_tensor_quant_mla_fp8( - attn_output.transpose(0, 1), - zero_allocator.allocate(1), - ) - attn_bmm_output = bmm_fp8( - attn_output_val, - self.w_vc, - attn_output_scale, - self.w_scale, - torch.bfloat16, - ) - else: - attn_bmm_output = torch.bmm(attn_output.transpose(0, 1), self.w_vc) - attn_output = attn_bmm_output.transpose(0, 1).flatten(1, 2) - output, _ = self.o_proj(attn_output) + self.num_heads = num_heads + tp_size = get_tensor_model_parallel_world_size() + assert num_heads % tp_size == 0 + self.num_local_heads = num_heads // tp_size - return output + self.scaling = self.qk_head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings - def forward_absorb_fused_mla_rope( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - ) -> torch.Tensor: - enable_rope_fusion = ( - os.getenv("SGLANG_FUSED_MLA_ENABLE_ROPE_FUSION", "1") == "1" - ) - q_len = hidden_states.shape[0] - q_input = hidden_states.new_empty( - q_len, self.num_local_heads, self.kv_lora_rank + self.qk_rope_head_dim - ) if self.q_lora_rank is not None: - q = self.q_a_proj(hidden_states)[0] - q = self.q_a_layernorm(q) - q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) - else: - q = self.q_proj(hidden_states)[0].view( - -1, self.num_local_heads, self.qk_head_dim - ) - q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - - if self.w_kc.dtype == torch.float8_e4m3fnuz: - # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz - q_nope_out = torch.bmm( - q_nope.to(torch.bfloat16).transpose(0, 1), - self.w_kc.to(torch.bfloat16) * self.w_scale, - ) - elif self.w_kc.dtype == torch.float8_e4m3fn: - q_nope_val, q_nope_scale = per_tensor_quant_mla_fp8( - q_nope.transpose(0, 1), - zero_allocator.allocate(1), - dtype=torch.float8_e4m3fn, - ) - q_nope_out = bmm_fp8( - q_nope_val, self.w_kc, q_nope_scale, self.w_scale, torch.bfloat16 - ) - else: - q_nope_out = torch.bmm(q_nope.transpose(0, 1), self.w_kc) - q_input[..., : self.kv_lora_rank] = q_nope_out.transpose(0, 1) - - latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] - v_input = latent_cache[..., : self.kv_lora_rank] - v_input = self.kv_a_layernorm(v_input.contiguous()).unsqueeze(1) - k_input = latent_cache.unsqueeze(1) - k_input[..., : self.kv_lora_rank] = v_input - - if not enable_rope_fusion: - k_pe = k_input[..., self.kv_lora_rank :] - q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - q_input[..., self.kv_lora_rank :] = q_pe - k_input[..., self.kv_lora_rank :] = k_pe - k_pe_output = None + self.q_a_proj = ReplicatedLinear(self.hidden_size, + self.q_lora_rank, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_a_proj") + self.q_a_layernorm = RMSNorm(self.q_lora_rank, + eps=config.rms_norm_eps) + self.q_b_proj = ColumnParallelLinear(q_lora_rank, + self.num_heads * + self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_b_proj") else: - k_pe_output = torch.empty_like(k_input[..., self.kv_lora_rank :]) - - q_input[..., self.kv_lora_rank :] = q_pe + self.q_proj = ColumnParallelLinear(self.hidden_size, + self.num_heads * + self.qk_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.q_proj") - # attn_output = self.attn_mqa(q_input, k_input, v_input, forward_batch) - # Use Fused ROPE with use_rope=OFF. - attn_output = torch.empty( - (q_len, self.num_local_heads, self.kv_lora_rank), - dtype=q.dtype, - device=q.device, - ) - attn_logits, _, kv_indptr, kv_indices, _, _, _ = ( - forward_batch.attn_backend.forward_metadata - ) - cos_sin_cache = self.rotary_emb.cos_sin_cache - num_kv_split = forward_batch.attn_backend.num_kv_splits - sm_scale = self.attn_mqa.scaling - if attn_logits is None: - attn_logits = torch.empty( - ( - forward_batch.batch_size, - self.num_local_heads, - num_kv_split, - self.kv_lora_rank + 1, - ), - dtype=torch.float32, - device=q.device, - ) - - # save current latent cache. - forward_batch.token_to_kv_pool.set_kv_buffer( - self.attn_mqa, forward_batch.out_cache_loc, k_input, None - ) - key_cache_buf = forward_batch.token_to_kv_pool.get_key_buffer( - self.attn_mqa.layer_id - ) - val_cache_buf = key_cache_buf[..., : self.kv_lora_rank] - - decode_attention_fwd_grouped_rope( - q_input, - key_cache_buf, - val_cache_buf, - attn_output, - kv_indptr, - kv_indices, - k_pe_output, + self.kv_a_proj_with_mqa = ReplicatedLinear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_a_proj_with_mqa") + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, + eps=config.rms_norm_eps) + self.kv_b_proj = ColumnParallelLinear( self.kv_lora_rank, - self.rotary_emb.rotary_dim, - cos_sin_cache, - positions, - attn_logits, - num_kv_split, - sm_scale, - logit_cap=self.attn_mqa.logit_cap, - use_rope=enable_rope_fusion, - is_neox_style=self.rotary_emb.is_neox_style, - ) - - if enable_rope_fusion: - k_input[..., self.kv_lora_rank :] = k_pe_output - forward_batch.token_to_kv_pool.set_kv_buffer( - self.attn_mqa, forward_batch.out_cache_loc, k_input, None - ) - - attn_output = attn_output.view(-1, self.num_local_heads, self.kv_lora_rank) - - if self.w_vc.dtype == torch.float8_e4m3fnuz: - # TODO(kernel): add bmm_fp8 for torch.float8_e4m3fnuz - attn_bmm_output = torch.bmm( - attn_output.to(torch.bfloat16).transpose(0, 1), - self.w_vc.to(torch.bfloat16) * self.w_scale, - ) - elif self.w_vc.dtype == torch.float8_e4m3fn: - attn_output_val, attn_output_scale = per_tensor_quant_mla_fp8( - attn_output.transpose(0, 1), - zero_allocator.allocate(1), - dtype=torch.float8_e4m3fn, - ) - attn_bmm_output = bmm_fp8( - attn_output_val, - self.w_vc, - attn_output_scale, - self.w_scale, - torch.bfloat16, - ) - else: - attn_bmm_output = torch.bmm(attn_output.transpose(0, 1), self.w_vc) - attn_output = attn_bmm_output.transpose(0, 1).flatten(1, 2) - output, _ = self.o_proj(attn_output) - - return output - - def _chunked_prefix_attn_mha( - self, - q: torch.Tensor, - accum_output: torch.Tensor, - accum_lse: torch.Tensor, - forward_batch: ForwardBatch, - ) -> torch.Tensor: - - assert forward_batch.num_prefix_chunks is not None - for i in range(forward_batch.num_prefix_chunks): - forward_batch.set_prefix_chunk_idx(i) - - # Fetch latent cache from memory pool with precomputed chunked kv indices - latent_cache_buf = forward_batch.token_to_kv_pool.get_key_buffer( - self.attn_mha.layer_id - ) - latent_cache = latent_cache_buf[ - forward_batch.prefix_chunk_kv_indices[i] - ].contiguous() + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_b_proj") + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim, + self.hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj") - kv_a_normed, k_pe = latent_cache.split( - [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 - ) - kv_a_normed = kv_a_normed.squeeze(1).contiguous() - kv = self.kv_b_proj(kv_a_normed)[0] - kv = kv.view( - -1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim - ) - v = kv[..., self.qk_nope_head_dim :] - k_nope = kv[..., : self.qk_nope_head_dim] - - k = torch.empty( - ( - k_nope.shape[0], - self.num_local_heads, - self.qk_nope_head_dim + self.qk_rope_head_dim, - ), - dtype=v.dtype, - device=v.device, - ) - k[..., : self.qk_nope_head_dim] = k_nope - k[..., self.qk_nope_head_dim :] = k_pe + if rope_scaling: + rope_scaling["rope_type"] = 'deepseek_yarn' + self.rotary_emb = get_rope(qk_rope_head_dim, + rotary_dim=qk_rope_head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + is_neox_style=False) + if rope_scaling: + mscale_all_dim = rope_scaling.get("mscale_all_dim", False) + scaling_factor = rope_scaling["factor"] + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) + self.scaling = self.scaling * mscale * mscale - output, lse = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False) - lse = torch.transpose(lse, 0, 1).contiguous() - tmp_output = torch.empty_like(accum_output) - tmp_lse = torch.empty_like(accum_lse) - merge_state_v2(output, lse, accum_output, accum_lse, tmp_output, tmp_lse) - accum_output, accum_lse = tmp_output, tmp_lse + # In the MLA backend, kv_cache includes both k_c and + # pe (i.e. decoupled position embeddings). In particular, + # the concat_and_cache_mla op requires + # k_c.size(1) + k_pe.size(1) == kv_cache.size(2) + # i.e. + # kv_lora_rank + qk_rope_head_dim == head_size + self.mla_attn = MLAAttention( + num_heads=self.num_local_heads, + head_size=self.kv_lora_rank + self.qk_rope_head_dim, + scale=self.scaling, + num_kv_heads=1, + kv_cache_dtype=cache_config, + # MLA Args + q_lora_rank=self.q_lora_rank, + kv_lora_rank=self.kv_lora_rank, + qk_nope_head_dim=self.qk_nope_head_dim, + qk_rope_head_dim=self.qk_rope_head_dim, + qk_head_dim=self.qk_head_dim, + v_head_dim=self.v_head_dim, + rotary_emb=self.rotary_emb, + q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj, + kv_b_proj=self.kv_b_proj, + o_proj=self.o_proj, + ) - return accum_output + self.prefix = prefix + self.debug_layer_idx = int(self.prefix.split(".")[-2]) - def forward_normal_chunked_kv( + def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, - forward_batch: ForwardBatch, ) -> torch.Tensor: - # In normal mha, the k and v tensors will become overly large when the prefix length is long. - # To avoid this, we split the kv cache into chunks and process them one after another. - # Since mha is compute friendly, the for loop induced here will not introduce significant overhead. - # The top comments in https://github.com/vllm-project/vllm/blob/main/vllm/v1/attention/backends/mla/common.py - # will be helpful for understanding the purpose of this function. - - # First do normal mha forward to get output for extended part if self.q_lora_rank is not None: - q = self.q_a_proj(hidden_states)[0] - q = self.q_a_layernorm(q) - q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, self.qk_head_dim) + ckq = self.q_a_proj(hidden_states) + hidden_states_or_q_c = self.q_a_layernorm(ckq) else: - q = self.q_proj(hidden_states)[0].view( - -1, self.num_local_heads, self.qk_head_dim - ) - _, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] - kv_a, _ = latent_cache.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) - latent_cache = latent_cache.unsqueeze(1) - kv_a = self.kv_a_layernorm(kv_a.contiguous()) - kv = self.kv_b_proj(kv_a)[0] - kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim) - k_nope = kv[..., : self.qk_nope_head_dim] - v = kv[..., self.qk_nope_head_dim :] - k_pe = latent_cache[:, :, self.kv_lora_rank :] - - q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - q[..., self.qk_nope_head_dim :] = q_pe - k = torch.empty_like(q) - k[..., : self.qk_nope_head_dim] = k_nope - k[..., self.qk_nope_head_dim :] = k_pe - - latent_cache[:, :, : self.kv_lora_rank] = kv_a.unsqueeze(1) - latent_cache[:, :, self.kv_lora_rank :] = k_pe - - # Save latent cache - forward_batch.token_to_kv_pool.set_kv_buffer( - self.attn_mha, forward_batch.out_cache_loc, latent_cache, None - ) - - # Do mha for extended part without prefix - forward_batch.set_attn_attend_prefix_cache(False) - attn_output, lse = self.attn_mha(q, k, v, forward_batch, save_kv_cache=False) - lse = torch.transpose(lse, 0, 1).contiguous() - - # Do mha attention with chunked prefix cache if there are any sequence with prefix - if any(forward_batch.extend_prefix_lens_cpu): - # Only initialize the info once - if forward_batch.num_prefix_chunks is None: - forward_batch.prepare_chunked_prefix_cache_info(q.device) - - forward_batch.set_attn_attend_prefix_cache(True) - attn_output = self._chunked_prefix_attn_mha( - q=q, - accum_output=attn_output, - accum_lse=lse, - forward_batch=forward_batch, - ) - - attn_output = attn_output.reshape(-1, self.num_local_heads * self.v_head_dim) - output, _ = self.o_proj(attn_output) - return output - - -class _FFNInputMode(Enum): - # The MLP sublayer requires 1/tp_size tokens as input - SCATTERED = auto() - # The MLP sublayer requires all tokens as input - FULL = auto() - - -@dataclass -class _DecoderLayerInfo: - is_sparse: bool - ffn_input_mode: _FFNInputMode + hidden_states_or_q_c = hidden_states + kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states).split( + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + kv_c_normed = self.kv_a_layernorm(kv_c) + return self.mla_attn(hidden_states_or_q_c, + kv_c_normed, + k_pe, + positions) class DeepseekV2DecoderLayer(nn.Module): @@ -992,659 +487,269 @@ class DeepseekV2DecoderLayer(nn.Module): def __init__( self, config: PretrainedConfig, - layer_id: int, + prefix: str, + cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, - is_nextn: bool = False, - prefix: str = "", + layer_num: int = 0, ) -> None: super().__init__() self.hidden_size = config.hidden_size rope_theta = getattr(config, "rope_theta", 10000) rope_scaling = getattr(config, "rope_scaling", None) - max_position_embeddings = getattr(config, "max_position_embeddings", 8192) - self.enable_dp_attention = global_server_args_dict["enable_dp_attention"] - self.layer_id = layer_id - self.dp_size = get_attention_dp_size() - self.attn_tp_size = get_attention_tp_size() - self.attn_tp_rank = get_attention_tp_rank() - self.self_attn = DeepseekV2AttentionMLA( + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + # DecoderLayers are created with `make_layers` which passes the prefix + # with the layer's index. + layer_idx = int(prefix.split(sep='.')[-1]) + self.layer_idx = layer_idx + + self.self_attn = DeepseekV2MLAAttention( config=config, hidden_size=self.hidden_size, num_heads=config.num_attention_heads, qk_nope_head_dim=config.qk_nope_head_dim, qk_rope_head_dim=config.qk_rope_head_dim, v_head_dim=config.v_head_dim, - q_lora_rank=( - config.q_lora_rank if hasattr(config, "q_lora_rank") else None - ), + q_lora_rank=config.q_lora_rank + if hasattr(config, "q_lora_rank") else None, kv_lora_rank=config.kv_lora_rank, rope_theta=rope_theta, rope_scaling=rope_scaling, max_position_embeddings=max_position_embeddings, + cache_config=cache_config, quant_config=quant_config, - layer_id=layer_id, - reduce_results=False, - prefix=add_prefix("self_attn", prefix), + prefix=f"{prefix}.self_attn", ) - self.info = self._compute_info(config, layer_id=layer_id, is_nextn=is_nextn) - previous_layer_info = self._compute_info( - config, layer_id=layer_id - 1, is_nextn=False - ) - - if self.info.is_sparse: + if (config.n_routed_experts is not None + and layer_idx >= config.first_k_dense_replace + and layer_idx % config.moe_layer_freq == 0): self.mlp = DeepseekV2MoE( config=config, quant_config=quant_config, - prefix=add_prefix("mlp", prefix), + prefix=f"{prefix}.mlp", ) else: - if self._enable_moe_dense_fully_dp(): - mlp_tp_rank, mlp_tp_size = 0, 1 - else: - mlp_tp_rank, mlp_tp_size = None, None self.mlp = DeepseekV2MLP( hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, hidden_act=config.hidden_act, quant_config=quant_config, - prefix=add_prefix("mlp", prefix), - tp_rank=mlp_tp_rank, - tp_size=mlp_tp_size, + prefix=f"{prefix}.mlp", ) - - self.input_is_scattered = ( - previous_layer_info.ffn_input_mode == _FFNInputMode.SCATTERED - ) - self.is_last_layer = self.layer_id == config.num_hidden_layers - 1 - - self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - @staticmethod - def _enable_moe_dense_fully_dp(): - return global_server_args_dict["moe_dense_tp_size"] == 1 - - @staticmethod - def _compute_info(config: PretrainedConfig, layer_id: int, is_nextn: bool): - is_sparse = is_nextn or ( - config.n_routed_experts is not None - and layer_id >= config.first_k_dense_replace - and layer_id % config.moe_layer_freq == 0 - ) - ffn_input_mode = ( - _FFNInputMode.SCATTERED - if (global_server_args_dict["enable_deepep_moe"] and is_sparse) - or (DeepseekV2DecoderLayer._enable_moe_dense_fully_dp() and not is_sparse) - else _FFNInputMode.FULL - ) - return _DecoderLayerInfo(is_sparse=is_sparse, ffn_input_mode=ffn_input_mode) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.routed_scaling_factor = config.routed_scaling_factor def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - zero_allocator: BumpAllocator, - ) -> torch.Tensor: - if self.info.ffn_input_mode == _FFNInputMode.SCATTERED: - return self.forward_ffn_with_scattered_input( - positions, hidden_states, forward_batch, residual, zero_allocator - ) - elif self.info.ffn_input_mode == _FFNInputMode.FULL: - return self.forward_ffn_with_full_input( - positions, hidden_states, forward_batch, residual, zero_allocator - ) - else: - raise NotImplementedError - - def forward_ffn_with_full_input( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, residual: Optional[torch.Tensor], - zero_allocator: BumpAllocator, ) -> torch.Tensor: - - if hidden_states.shape[0] == 0: - residual = hidden_states - else: - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) - - assert not ( - self.attn_tp_size != 1 and self.input_is_scattered - ), "moe_layer_freq > 1 is not supported when attn_tp_size > 1" - - # Self Attention - hidden_states = self.self_attn( - positions=positions, - hidden_states=hidden_states, - forward_batch=forward_batch, - zero_allocator=zero_allocator, - ) - - # Gather - if get_tensor_model_parallel_world_size() > 1: - # all gather and all reduce - if self.dp_size != 1: - if self.attn_tp_rank == 0: - hidden_states += residual - hidden_states, local_hidden_states = ( - forward_batch.gathered_buffer, - hidden_states, - ) - dp_gather_partial(hidden_states, local_hidden_states, forward_batch) - dp_scatter(residual, hidden_states, forward_batch) - hidden_states = self.post_attention_layernorm(hidden_states) - else: - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual - ) - else: - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual - ) - - # Fully Connected - hidden_states = self.mlp(hidden_states) - - # TODO(ch-wan): ues reduce-scatter in MLP to avoid this scatter - # Scatter - if self.dp_size != 1: - # important: forward batch.gathered_buffer is used both after scatter and after gather. - # be careful about this! - hidden_states, global_hidden_states = ( - forward_batch.gathered_buffer[: forward_batch.input_ids.shape[0]], - hidden_states, - ) - dp_scatter(hidden_states, global_hidden_states, forward_batch) - - return hidden_states, residual - - def forward_ffn_with_scattered_input( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - zero_allocator: BumpAllocator, - ) -> torch.Tensor: - - if hidden_states.shape[0] == 0: + # Self Attention + if residual is None: residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) else: - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) - - if self.attn_tp_size != 1 and self.input_is_scattered: - hidden_states, local_hidden_states = ( - forward_batch.gathered_buffer[: forward_batch.input_ids.shape[0]], - hidden_states, - ) - tp_all_gather( - list(hidden_states.tensor_split(self.attn_tp_size)), local_hidden_states - ) - - # Self Attention + hidden_states, residual = self.input_layernorm( + hidden_states, residual) hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, - forward_batch=forward_batch, - zero_allocator=zero_allocator, ) - if self.attn_tp_size != 1: - if self.input_is_scattered: - tensor_list = list(hidden_states.tensor_split(self.attn_tp_size)) - hidden_states = tensor_list[self.attn_tp_rank] - tp_reduce_scatter(hidden_states, tensor_list) - if hidden_states.shape[0] != 0: - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual - ) - else: - if self.attn_tp_rank == 0: - hidden_states += residual - tensor_list = list(hidden_states.tensor_split(self.attn_tp_size)) - hidden_states = tensor_list[self.attn_tp_rank] - tp_reduce_scatter(hidden_states, tensor_list) - residual = hidden_states - if hidden_states.shape[0] != 0: - hidden_states = self.post_attention_layernorm(hidden_states) - else: - if hidden_states.shape[0] != 0: - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual - ) - - if not ( - self._enable_moe_dense_fully_dp() - and (not self.info.is_sparse) - and hidden_states.shape[0] == 0 - ): - hidden_states = self.mlp(hidden_states, forward_batch.forward_mode) - - if self.is_last_layer and self.attn_tp_size != 1: - hidden_states += residual - residual = None - hidden_states, local_hidden_states = ( - forward_batch.gathered_buffer[: forward_batch.input_ids.shape[0]], - hidden_states, - ) - tp_all_gather( - list(hidden_states.tensor_split(self.attn_tp_size)), local_hidden_states - ) + if hidden_states.dtype == torch.float16: + # Fix FP16 overflow + # We scale both hidden_states and residual before + # rmsnorm, and rmsnorm result would not affect by scale. + hidden_states *= 1. / self.routed_scaling_factor + if self.layer_idx == 0: + # The residual is shared by all layers, we only scale it on + # first layer. + residual *= 1. / self.routed_scaling_factor + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + + if isinstance(self.mlp, + DeepseekV2MLP) and hidden_states.dtype == torch.float16: + # Fix FP16 overflow + # Scaling the DeepseekV2MLP output, it is the input of + # input_layernorm of next decoder layer. + # The scaling of DeepseekV2MOE output would be done in the forward + # of DeepseekV2MOE + hidden_states *= 1. / self.routed_scaling_factor return hidden_states, residual class DeepseekV2Model(nn.Module): - fall_back_to_pt_during_load = False - def __init__( self, - config: PretrainedConfig, - quant_config: Optional[QuantizationConfig] = None, + atom_config: Config, prefix: str = "", - ) -> None: + layer_type: type[nn.Module] = DeepseekV2DecoderLayer, + ): super().__init__() - self.padding_id = config.pad_token_id + + config = atom_config.hf_config + cache_config = atom_config.kv_cache_dtype + quant_config = atom_config.quant_config + self.config = config + self.vocab_size = config.vocab_size - self.embed_tokens = VocabParallelEmbedding( - config.vocab_size, - config.hidden_size, - enable_tp=not global_server_args_dict["enable_dp_attention"], - ) - self.layers = nn.ModuleList( - [ - DeepseekV2DecoderLayer( - config, - layer_id, - quant_config=quant_config, - prefix=add_prefix(f"layers.{layer_id}", prefix), - ) - for layer_id in range(config.num_hidden_layers) - ] + if get_pp_group().is_first_rank: + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + else: + self.embed_tokens = PPMissingLayer() + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix, layer_num=None: DeepseekV2DecoderLayer( + config, + prefix, + cache_config=cache_config, + quant_config=quant_config, + layer_num=layer_num, + ), + prefix=f"{prefix}.layers", + layer_num_offset=0 ) - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.dp_size = get_attention_dp_size() + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, - forward_batch: ForwardBatch, - input_embeds: torch.Tensor = None, - ) -> torch.Tensor: - zero_allocator = BumpAllocator( - # TODO for two-batch-overlap, we need a larger buffer size - buffer_size=len(self.layers) * 2, - dtype=torch.float32, - device=( - input_embeds.device if input_embeds is not None else input_ids.device - ), - ) - - if input_embeds is None: - hidden_states = self.embed_tokens(input_ids) - else: - hidden_states = input_embeds - - residual = None - for i in range(len(self.layers)): - expert_distribution_recorder.set_current_layer(i) - layer = self.layers[i] - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual, zero_allocator - ) - if not forward_batch.forward_mode.is_idle(): - if residual is None: - hidden_states = self.norm(hidden_states) + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds else: - hidden_states, _ = self.norm(hidden_states, residual) + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for layer in self.layers[self.start_layer:self.end_layer]: + hidden_states, residual = layer(positions, hidden_states, residual) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) return hidden_states + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + return FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.n_routed_experts + (self.config.n_shared_experts if is_rocm_aiter_fusion_shared_expert_enabled() else 0)) + class DeepseekV2ForCausalLM(nn.Module): + packed_modules_mapping = { + "q_proj": ("qkv_proj", "q"), + "k_proj": ("qkv_proj", "k"), + "v_proj": ("qkv_proj", "v"), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } def __init__( - self, - config: PretrainedConfig, - quant_config: Optional[QuantizationConfig] = None, - prefix: str = "", - ) -> None: + self, + atom_config: Config, + prefix: str = "", + layer_type: type[nn.Module] = DeepseekV2DecoderLayer + ): super().__init__() + config = atom_config.hf_config + quant_config = atom_config.quant_config self.config = config - self.tp_size = get_tensor_model_parallel_world_size() self.quant_config = quant_config - self.n_share_experts_fusion = global_server_args_dict["n_share_experts_fusion"] - if self.n_share_experts_fusion > 0: - # Only Deepseek V3/R1 can use shared experts fusion optimization now. - if ( - self.config.architectures[0] != "DeepseekV3ForCausalLM" - or self.config.n_routed_experts != 256 - ): - self.n_share_experts_fusion = 0 - global_server_args_dict["n_share_experts_fusion"] = 0 - logger.info( - "Only Deepseek V3/R1 can use shared experts fusion optimization. Shared experts fusion optimization is disabled." - ) - else: - assert ( - self.n_share_experts_fusion == self.tp_size - ), f"Shared experts fusion optimization is enabled in DeepSeek V3/R1, set it to {self.tp_size} can get best optimized performace." - elif self.n_share_experts_fusion == 0: - if ( - torch.cuda.get_device_capability("cuda") >= (9, 0) - and self.config.architectures[0] == "DeepseekV3ForCausalLM" - and self.config.n_routed_experts == 256 - and (not global_server_args_dict["enable_deepep_moe"]) - ): - self.n_share_experts_fusion = self.tp_size - global_server_args_dict["n_share_experts_fusion"] = self.tp_size - logger.info( - "Deepseek V3/R1 with fp8 can use shared experts fusion optimization when SM version >=90. Shared experts fusion optimization is enabled." - ) - - self.model = DeepseekV2Model( - config, quant_config, prefix=add_prefix("model", prefix) - ) - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=add_prefix("lm_head", prefix), + self.model = DeepseekV2Model(atom_config=atom_config, + prefix=maybe_prefix(prefix, "model"), + layer_type = layer_type) + if get_pp_group().is_last_rank: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + org_num_embeddings=config.vocab_size, + prefix=maybe_prefix(prefix, "lm_head"), + ) + else: + self.lm_head = PPMissingLayer() + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors ) - self.logits_processor = LogitsProcessor(config) - self.dp_size = get_attention_dp_size() - def get_input_embeddings(self) -> nn.Embedding: - return self.model.embed_tokens + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) - @torch.no_grad() def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, - forward_batch: ForwardBatch, - input_embeds: torch.Tensor = None, - ) -> torch.Tensor: - - hidden_states = self.model(input_ids, positions, forward_batch, input_embeds) - - return self.logits_processor( - input_ids, hidden_states, self.lm_head, forward_batch - ) + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + return hidden_states - def post_load_weights(self): - - # Perform post-processing after loading weights - for layer_id in range(self.config.num_hidden_layers): - self_attn = self.model.layers[layer_id].self_attn - if hasattr(self_attn.kv_b_proj, "qweight"): - # AWQ compatible - if _is_cuda: - w = awq_dequantize( - self_attn.kv_b_proj.qweight, - self_attn.kv_b_proj.scales, - self_attn.kv_b_proj.qzeros, - ).T - else: - w = awq_dequantize( - self_attn.kv_b_proj.qweight, - self_attn.kv_b_proj.scales, - self_attn.kv_b_proj.qzeros, - 0, - 0, - 0, - ).T - else: - w = self_attn.kv_b_proj.weight - # NOTE(HandH1998): Since `bmm_fp8` only supports per-tensor scale, we have to requantize `self_attn.kv_b_proj`. - # This may affect the accuracy of fp8 model. - # Fix deepseek v3 blockwise bmm by using deep_gemm - use_deep_gemm_bmm = False - model_dtype = torch.get_default_dtype() - - if w.dtype in ( - torch.float8_e4m3fn, - torch.float8_e4m3fnuz, - ): - if hasattr(self.quant_config, "weight_block_size"): - weight_block_size = self.quant_config.weight_block_size - if weight_block_size is not None: - assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") - if _is_hip: - weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( - weight=w, - weight_scale=self_attn.kv_b_proj.weight_scale_inv, - input_scale=None, - ) - else: - weight = w - weight_scale = self_attn.kv_b_proj.weight_scale_inv - - if ( - _is_cuda - and _enable_jit_deepgemm_bmm - and weight_block_size[0] == 128 - and weight_block_size[1] == 128 - and model_dtype == torch.bfloat16 - ): - block_scale = weight_scale - use_deep_gemm_bmm = True - else: - w, scale = block_quant_to_tensor_quant( - weight, weight_scale, weight_block_size - ) - self_attn.w_scale = scale - else: - weight = w - weight_scale = self_attn.kv_b_proj.weight_scale - w, scale = channel_quant_to_tensor_quant(weight, weight_scale) - self_attn.w_scale = scale - - if w.dtype == torch.int8: - if hasattr(self.quant_config, "weight_block_size"): - # block-wise int8 need it - weight_block_size = self.quant_config.weight_block_size - if weight_block_size is not None: - assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") - weight = w - weight_scale = self_attn.kv_b_proj.weight_scale_inv - w = int8_block_dequant( - weight, weight_scale, weight_block_size - ).to(torch.bfloat16) - else: - # channel-wise int8 need it - w = w.to(torch.bfloat16) * self_attn.kv_b_proj.weight_scale.to( - torch.bfloat16 - ) - - w_kc, w_vc = w.unflatten( - 0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim) - ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) - if not use_deep_gemm_bmm: - self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2) - self_attn.w_vc = w_vc.contiguous().transpose(1, 2) - if ( - hasattr(self_attn.kv_b_proj, "weight_scale") - and self_attn.w_scale is None - ): - self_attn.w_scale = self_attn.kv_b_proj.weight_scale - if _is_hip: - self_attn.w_scale *= 2.0 - else: - num_tiles_k = self_attn.qk_nope_head_dim // weight_block_size[1] - num_tiles_n = self_attn.v_head_dim // weight_block_size[0] - ws_kc, ws_vc = block_scale.unflatten( - 0, (-1, (num_tiles_k + num_tiles_n)) - ).split([num_tiles_k, num_tiles_n], dim=1) - self_attn.w_scale_k = ws_kc.transpose(1, 2).contiguous() - self_attn.w_scale_v = ws_vc.contiguous() - self_attn.w_kc = w_kc.transpose(1, 2).contiguous() - self_attn.w_vc = w_vc.contiguous() - self_attn.use_deep_gemm_bmm = True - - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - if self.n_share_experts_fusion > 0: - weights_list = list(weights) - weights_dict = dict(weights_list) - if self.quant_config.get_name() == "w8a8_int8": - suffix_list = [ - "down_proj.weight", - "down_proj.weight_scale", - "gate_proj.weight", - "gate_proj.weight_scale", - "up_proj.weight", - "up_proj.weight_scale", - ] - else: - suffix_list = [ - "down_proj.weight", - "down_proj.weight_scale_inv", - "gate_proj.weight", - "gate_proj.weight_scale_inv", - "up_proj.weight", - "up_proj.weight_scale_inv", - ] - names_to_remove = [] - for moe_layer in tqdm( - range( - self.config.first_k_dense_replace, - self.config.num_hidden_layers, - self.config.moe_layer_freq, - ), - desc=f"Cloning {self.n_share_experts_fusion} " - "replicas of the shared expert into MoE", - ): - for num_repeat in range(self.n_share_experts_fusion): - for suffix in suffix_list: - shared_expert_weight_name = ( - f"model.layers.{moe_layer}.mlp.shared_experts.{suffix}" - ) - weights_list.append( - ( - f"model.layers.{moe_layer}." - f"mlp.experts." - f"{self.config.n_routed_experts + num_repeat}" - f".{suffix}", - weights_dict[shared_expert_weight_name], - ) - ) - names_to_remove += [shared_expert_weight_name] - weights = [w for w in weights_list if w[0] not in names_to_remove] + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> Optional[torch.Tensor]: + logits = self.lm_head(hidden_states) + return logits - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - MoEImpl = ( - DeepEPMoE - if global_server_args_dict["enable_deepep_moe"] - else (EPMoE if global_server_args_dict["enable_ep_moe"] else FusedMoE) - ) - expert_params_mapping = MoEImpl.make_expert_params_mapping( - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts + self.n_share_experts_fusion, - ) - params_dict = dict(self.named_parameters()) - for name, loaded_weight in weights: - # TODO(HandH1998): Modify it when nextn is supported. - if hasattr(self.config, "num_nextn_predict_layers"): - num_nextn_layers = self.config.num_nextn_predict_layers - if num_nextn_layers > 0 and name.startswith("model.layers"): - name_list = name.split(".") - if ( - len(name_list) >= 3 - and int(name_list[2]) >= self.config.num_hidden_layers - ): - continue - if "rotary_emb.inv_freq" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - - self.post_load_weights() - - def get_embed_and_head(self): - return self.model.embed_tokens.weight, self.lm_head.weight - - def set_embed_and_head(self, embed, head): - del self.model.embed_tokens.weight - del self.lm_head.weight - self.model.embed_tokens.weight = embed - self.lm_head.weight = head - torch.cuda.empty_cache() - torch.cuda.synchronize() + def make_empty_intermediate_tensors( + self, batch_size: int, dtype: torch.dtype, + device: torch.device) -> IntermediateTensors: + return IntermediateTensors({ + "hidden_states": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + "residual": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + }) + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + return self.model.get_expert_mapping() class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM): pass - - -EntryClass = [DeepseekV2ForCausalLM, DeepseekV3ForCausalLM] diff --git a/atom/utils/context.py b/atom/utils/context.py index 8a348532ef..84e15c3089 100644 --- a/atom/utils/context.py +++ b/atom/utils/context.py @@ -19,6 +19,10 @@ class Context: block_tables: Optional[torch.Tensor] = None dropout_p: float = 0.0 + max_q_len: Optional[int] = None + kv_indptr: Optional[torch.Tensor] = None + kv_indices: Optional[torch.Tensor] = None + kv_last_page_lens: Optional[torch.Tensor] = None _CONTEXT = Context() @@ -40,6 +44,10 @@ def set_context( context_lens=None, block_tables=None, dropout_p=0.0, + max_q_len=None, + kv_indptr=None, + kv_indices=None, + kv_last_page_lens=None, ): global _CONTEXT _CONTEXT = Context( @@ -55,6 +63,10 @@ def set_context( context_lens, block_tables, dropout_p, + max_q_len, + kv_indptr, + kv_indices, + kv_last_page_lens, ) From 1879b15369f87ec71a0fba4f22307b03365f8aae Mon Sep 17 00:00:00 2001 From: jiayyu Date: Sat, 11 Oct 2025 16:52:58 +0800 Subject: [PATCH 053/119] example --- example.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/example.py b/example.py index 4e39e6411b..238d7aa7a2 100644 --- a/example.py +++ b/example.py @@ -49,6 +49,18 @@ "--torch-profiler-dir", type=str, default=None, help="Directory to save torch profiler traces" ) +parser.add_argument( + "--enable-expert-parallel", action="store_true", help="Enable expert parallel(EP MoE)." +) + +parser.add_argument( + "--block-size", type=int, default=16, help="KV cache block size." +) + +parser.add_argument( + "--load_dummy", action="store_true", help="Skip loading model weights." +) + def parse_size_list(size_str: str) -> List[int]: import ast try: @@ -66,7 +78,10 @@ def main(): enforce_eager=args.enforce_eager, tensor_parallel_size=args.tensor_parallel_size, kv_cache_dtype=args.kv_cache_dtype, + kv_cache_block_size=args.block_size, port=args.port, + load_dummy=args.load_dummy, + enable_expert_parallel=args.enable_expert_parallel, torch_profiler_dir=args.torch_profiler_dir, compilation_config=CompilationConfig( level = args.level, From 06e1862e5b0c8ce7336d62c629a1c22d6ff1a415 Mon Sep 17 00:00:00 2001 From: jiayyu Date: Mon, 13 Oct 2025 11:53:53 +0800 Subject: [PATCH 054/119] fix slop mapping --- atom/model_engine/sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 3e21394364..e7cd8e0f50 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -56,7 +56,7 @@ def num_tokens(self): def num_tokens(self, value): self._num_tokens = value self.num_blocks = (value + self.block_size - 1) // self.block_size - self.last_block_num_tokens = value % self.block_size + self.last_block_num_tokens = self._num_tokens - (self.num_blocks - 1) * self.block_size @property def is_finished(self): From 3dd24ba346883b97f3738b354696a74a0f1b5684 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Mon, 13 Oct 2025 15:38:23 +0000 Subject: [PATCH 055/119] mt weight load --- atom/model_engine/model_runner.py | 2 +- atom/model_loader/loader.py | 199 ++++++++++++------ atom/model_ops/moe.py | 339 +++++++++++++++++------------- 3 files changed, 328 insertions(+), 212 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 2615a29b2b..b14835b400 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -17,10 +17,10 @@ from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler +from atom.models.deepseek_v2 import DeepseekV2ForCausalLM from atom.models.llama import LlamaForCausalLM from atom.models.mixtral import MixtralForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM -from atom.models.deepseek_v2 import DeepseekV2ForCausalLM from atom.utils import CpuGpuBuffer, init_exit_handler from atom.utils.context import get_context, reset_context, set_context diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index e0ed0ff258..585d3fd852 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -1,99 +1,160 @@ -import os, re +import concurrent.futures +import os +import re from glob import glob +from typing import Generator, List, Tuple + +import safetensors import torch from torch import nn -from safetensors import safe_open +from tqdm import tqdm +from transformers import AutoConfig from transformers.utils import SAFE_WEIGHTS_INDEX_NAME -from atom.model_loader.weight_utils import download_weights_from_hf, filter_duplicate_safetensors_files + +from atom.model_loader.weight_utils import ( + download_weights_from_hf, + filter_duplicate_safetensors_files, +) from atom.model_ops.base_config import QuantizeMethodBase -from transformers import AutoConfig -from tqdm import tqdm from atom.model_ops.moe import is_rocm_aiter_fusion_shared_expert_enabled + def default_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor): param.data.copy_(loaded_weight) -def load_model(model: nn.Module, model_name_or_path: str, hf_config: AutoConfig, load_dummy:bool = False): - packed_modules_mapping = getattr(model, "packed_modules_mapping", {}) - is_path = os.path.isdir(model_name_or_path) +def safetensors_weights_iterator( + model_name_or_path: str, + disable_mmap: bool = False, +) -> Generator[Tuple[str, torch.Tensor], None, None]: + """Iterate over the weights in the model safetensor files.""" path = ( model_name_or_path - if is_path + if os.path.isdir(model_name_or_path) else download_weights_from_hf( model_name_or_path, None, ["*.safetensors"], ignore_patterns=["original/*"] ) ) + hf_weights_files = filter_duplicate_safetensors_files( + glob(os.path.join(path, "*.safetensors")), path, SAFE_WEIGHTS_INDEX_NAME + ) + enable_tqdm = ( + not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + ) + + iters = tqdm( + hf_weights_files, + desc=f"Loading safetensors shards[{model_name_or_path}]", + disable=not enable_tqdm, + ) + for st_file in iters: + if disable_mmap: + with open(st_file, "rb") as f: + result = safetensors.torch.load(f.read()) + for name, param in result.items(): + yield name, param + else: + with safetensors.safe_open(st_file, framework="pt", device="cpu") as f: + for name in f.keys(): + yield name, f.get_tensor(name) + + +def load_model( + model: nn.Module, + model_name_or_path: str, + hf_config: AutoConfig, + load_dummy: bool = False, +): + packed_modules_mapping = getattr(model, "packed_modules_mapping", {}) params_dict = dict(model.named_parameters()) - hf_weights_files = filter_duplicate_safetensors_files(glob(os.path.join(path, "*.safetensors")), path, SAFE_WEIGHTS_INDEX_NAME) - for file in tqdm(hf_weights_files): - if load_dummy: - continue - print(f"Loading weights from {file}") - with safe_open(file, "pt", "cpu") as f: - for name in f.keys(): - if name.endswith( - "kv_scale" - ): + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [] + for name, weight_tensor in safetensors_weights_iterator(model_name_or_path): + if load_dummy: + continue + if name.endswith("kv_scale"): + continue + if "weight_scale_inv" in name: + name = name.replace("weight_scale_inv", "weight_scale") + layerId_ = re.search(r"model\.layers\.(\d+)\.", name) + layerId = int(layerId_.group(1)) if layerId_ else 0 + if hf_config.num_hidden_layers and layerId >= hf_config.num_hidden_layers: + # print(f"Skipping loading {name} as layerId {layerId} >= num_hidden_layers {hf_config.num_hidden_layers}") + continue + if ( + is_rocm_aiter_fusion_shared_expert_enabled() + and "mlp.shared_experts" in name + ): + name = name.replace( + "mlp.shared_experts", + f"mlp.experts.{hf_config.n_routed_experts}", + ) + for k in packed_modules_mapping: + # We handle the experts below in expert_params_mapping + if "mlp.experts." in name and name not in params_dict: continue - weight_tensor = f.get_tensor(name) - if "weight_scale_inv" in name: - name = name.replace( - "weight_scale_inv", "weight_scale" + if k in name: + v, shard_id = packed_modules_mapping[k] + param_name = name.replace(k, v) + param = model.get_parameter(param_name) + weight_loader = getattr(param, "weight_loader") + # weight_loader(param, weight_tensor, shard_id) + futures.append( + executor.submit(weight_loader, param, weight_tensor, shard_id) ) - layerId_ = re.search(r'model\.layers\.(\d+)\.', name) - layerId = int(layerId_.group(1)) if layerId_ else 0 - if hf_config.num_hidden_layers and layerId >= hf_config.num_hidden_layers: - # print(f"Skipping loading {name} as layerId {layerId} >= num_hidden_layers {hf_config.num_hidden_layers}") - continue - if (is_rocm_aiter_fusion_shared_expert_enabled() - and "mlp.shared_experts" in name): - name = name.replace( - "mlp.shared_experts", - f"mlp.experts.{hf_config.n_routed_experts}") - for k in packed_modules_mapping: - # We handle the experts below in expert_params_mapping - if ("mlp.experts." in name and name not in params_dict): - continue - if k in name: - v, shard_id = packed_modules_mapping[k] - param_name = name.replace(k, v) - param = model.get_parameter(param_name) + break + else: + # Check if model has expert mapping before processing + if hasattr(model, "get_expert_mapping"): + for k in model.get_expert_mapping(): + param_name, weight_name, expert_id, shard_id = k + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if ( + name.endswith(".bias") or name.endswith("_bias") + ) and name not in dict(model.named_parameters()): + continue + param = model.get_parameter(name) weight_loader = getattr(param, "weight_loader") - weight_loader(param, weight_tensor, shard_id) - break - else: - # Check if model has expert mapping before processing - if hasattr(model, "get_expert_mapping"): - for k in model.get_expert_mapping(): - param_name, weight_name, expert_id, shard_id = k - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if ((name.endswith(".bias") or name.endswith("_bias")) - and name not in dict(model.named_parameters())): - continue - param = model.get_parameter(name) - weight_loader = getattr(param, "weight_loader") - weight_loader(param, - weight_tensor, - name, - shard_id=shard_id, - expert_id=expert_id) - break - else: - param = model.get_parameter(name) - weight_loader = getattr( - param, "weight_loader", default_weight_loader + futures.append( + executor.submit( + weight_loader, + param, + weight_tensor, + name, + shard_id, + expert_id, ) - weight_loader(param, weight_tensor) + ) + # weight_loader( + # param, + # weight_tensor, + # name, + # shard_id=shard_id, + # expert_id=expert_id, + # ) + break else: - # Model doesn't have expert mapping, use generic loading param = model.get_parameter(name) weight_loader = getattr( param, "weight_loader", default_weight_loader ) - weight_loader(param, weight_tensor) + futures.append( + executor.submit(weight_loader, param, weight_tensor) + ) + # weight_loader(param, weight_tensor) + else: + # Model doesn't have expert mapping, use generic loading + param = model.get_parameter(name) + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + # weight_loader(param, weight_tensor) + futures.append(executor.submit(weight_loader, param, weight_tensor)) + # Wait for all tasks to complete and raise any exceptions. + for future in concurrent.futures.as_completed(futures): + future.result() for _, module in model.named_modules(): if hasattr(module, "process_weights_after_loading"): module.process_weights_after_loading() diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 8e314cb6df..3559957d62 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -1,36 +1,43 @@ -from functools import lru_cache from abc import abstractmethod -import torch -from torch import nn +from functools import lru_cache from typing import Callable, List, Optional, Tuple + +import torch +from aiter import ActivationType, QuantType, dtypes, get_hip_quant from aiter.dist.parallel_state import get_tp_group +from aiter.fused_moe import fused_moe +from torch import nn from transformers import PretrainedConfig -from aiter import QuantType, dtypes -from aiter import get_hip_quant + from atom.config import QuantizationConfig, get_current_atom_config +from atom.model_loader.weight_utils import set_weight_attrs from atom.model_ops.base_config import QuantizeMethodBase -from atom.model_ops.topK import rocm_aiter_grouped_topk as grouped_topk -from atom.model_ops.topK import rocm_aiter_topk_softmax as fused_topk from atom.model_ops.topK import ( init_aiter_topK_meta_data, - is_rocm_aiter_fusion_shared_expert_enabled, is_rocm_aiter_fuse_routed_scaling_factor, + is_rocm_aiter_fusion_shared_expert_enabled, ) +from atom.model_ops.topK import rocm_aiter_grouped_topk as grouped_topk +from atom.model_ops.topK import rocm_aiter_topk_softmax as fused_topk from atom.model_ops.utils import ( - shuffle_weights, normalize_e4m3fn_to_e4m3fnuz, per_tensor_dequantize, + shuffle_weights, ) -from atom.model_loader.weight_utils import set_weight_attrs -from aiter.fused_moe import fused_moe -from aiter import ActivationType + class FusedMoEMethodBase(QuantizeMethodBase): @abstractmethod - def create_weights(self, layer: torch.nn.Module, num_experts: int, - hidden_size: int, intermediate_size_per_partition: int, - params_dtype: torch.dtype, **extra_weight_attrs): + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): raise NotImplementedError @abstractmethod @@ -58,26 +65,38 @@ def apply( class UnquantizedFusedMoEMethod(FusedMoEMethodBase): """MoE method without quantization.""" - def create_weights(self, layer: torch.nn.Module, num_experts: int, - hidden_size: int, intermediate_size_per_partition: int, - params_dtype: torch.dtype, **extra_weight_attrs): + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): # Fused gate_up_proj (column parallel) - w13_weight = torch.nn.Parameter(torch.empty( - num_experts, - 2 * intermediate_size_per_partition, - hidden_size, - dtype=params_dtype), - requires_grad=False) + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) layer.register_parameter("w13_weight", w13_weight) set_weight_attrs(w13_weight, extra_weight_attrs) # down_proj (row parallel) - w2_weight = torch.nn.Parameter(torch.empty( - num_experts, - hidden_size, - intermediate_size_per_partition, - dtype=params_dtype), - requires_grad=False) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) @@ -95,20 +114,19 @@ def _maybe_pad_weight(self, weight: torch.Tensor) -> torch.Tensor: def process_weights_after_loading(self, layer: torch.nn.Module) -> None: super().process_weights_after_loading(layer) - layer.w13_weight = torch.nn.Parameter(self._maybe_pad_weight( - layer.w13_weight.data), - requires_grad=False) - layer.w2_weight = torch.nn.Parameter(self._maybe_pad_weight( - layer.w2_weight.data), - requires_grad=False) + layer.w13_weight = torch.nn.Parameter( + self._maybe_pad_weight(layer.w13_weight.data), requires_grad=False + ) + layer.w2_weight = torch.nn.Parameter( + self._maybe_pad_weight(layer.w2_weight.data), requires_grad=False + ) # reshaping weights is required for aiter moe kernel. shuffled_w13, shuffled_w2 = shuffle_weights( - layer.w13_weight.data, layer.w2_weight.data) + layer.w13_weight.data, layer.w2_weight.data + ) - layer.w13_weight = torch.nn.Parameter(shuffled_w13, - requires_grad=False) - layer.w2_weight = torch.nn.Parameter(shuffled_w2, - requires_grad=False) + layer.w13_weight = torch.nn.Parameter(shuffled_w13, requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, requires_grad=False) def apply( self, @@ -138,7 +156,8 @@ def apply( num_expert_group=num_expert_group, custom_routing_function=custom_routing_function, scoring_func=scoring_func, - e_score_correction_bias=e_score_correction_bias) + e_score_correction_bias=e_score_correction_bias, + ) return fused_moe( hidden_states=x, w1=layer.w13_weight, @@ -146,7 +165,8 @@ def apply( topk_weight=topk_weights, topk_ids=topk_ids, expert_mask=expert_map, - activation=activation) + activation=activation, + ) class Fp8MoEMethod(FusedMoEMethodBase): @@ -166,14 +186,25 @@ def __init__(self, quant_config: QuantizationConfig): self.quant_config = quant_config self.quant_type = self.quant_config["quant_type"] self.quant_dtype = self.quant_config["quant_dtype"] - self.block_quant = self.quant_type == QuantType.per_1x128 or self.quant_type == QuantType.per_1x32 - self.need_normalize_e4m3fn_to_e4m3fnuz = self.quant_dtype == torch.float8_e4m3fnuz + self.block_quant = ( + self.quant_type == QuantType.per_1x128 + or self.quant_type == QuantType.per_1x32 + ) + self.need_normalize_e4m3fn_to_e4m3fnuz = ( + self.quant_dtype == torch.float8_e4m3fnuz + ) - def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: int, - intermediate_size_per_partition: int, - params_dtype: torch.dtype, **extra_weight_attrs): + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): - #TODO hard code for now + # TODO hard code for now params_dtype = torch.float8_e4m3fn if self.block_quant: @@ -192,31 +223,38 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: raise ValueError( f"The output_size of gate's and up's weight = " f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_n = {block_n}.") - if (tp_size > 1 - and intermediate_size_per_partition % block_k != 0): + f"weight quantization block_n = {block_n}." + ) + if tp_size > 1 and intermediate_size_per_partition % block_k != 0: # Required by row parallel raise ValueError( f"The input_size of down's weight = " f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_k = {block_k}.") + f"weight quantization block_k = {block_k}." + ) # WEIGHTS - w13_weight = torch.nn.Parameter(torch.empty( - num_experts, - 2 * intermediate_size_per_partition, - hidden_size, - dtype=params_dtype), - requires_grad=False) + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) layer.register_parameter("w13_weight", w13_weight) set_weight_attrs(w13_weight, extra_weight_attrs) - w2_weight = torch.nn.Parameter(torch.empty( - num_experts, - hidden_size, - intermediate_size_per_partition, - dtype=params_dtype), - requires_grad=False) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) layer.register_parameter("w2_weight", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) @@ -224,20 +262,19 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: if not self.block_quant: # Allocate 2 scales for w1 and w3 respectively. # They will be combined to a single scale after weight loading. - w13_weight_scale = torch.nn.Parameter(torch.ones( - num_experts, 2, dtype=torch.float32), - requires_grad=False) - w2_weight_scale = torch.nn.Parameter(torch.ones( - num_experts, dtype=torch.float32), - requires_grad=False) + w13_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) layer.register_parameter("w13_weight_scale", w13_weight_scale) layer.register_parameter("w2_weight_scale", w2_weight_scale) else: w13_weight_scale = torch.nn.Parameter( torch.ones( num_experts, - 2 * ((intermediate_size_per_partition + block_n - 1) // - block_n), + 2 * ((intermediate_size_per_partition + block_n - 1) // block_n), (hidden_size + block_k - 1) // block_k, dtype=torch.float32, ), @@ -271,14 +308,14 @@ def create_weights(self, layer: torch.nn.Module, num_experts: int, hidden_size: # INPUT_SCALES if not self.quant_config["is_dynamic"]: - w13_input_scale = torch.nn.Parameter(torch.ones( - num_experts, dtype=torch.float32), - requires_grad=False) + w13_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) layer.register_parameter("w13_input_scale", w13_input_scale) set_weight_attrs(w13_input_scale, extra_weight_attrs) - w2_input_scale = torch.nn.Parameter(torch.ones( - num_experts, dtype=torch.float32), - requires_grad=False) + w2_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) layer.register_parameter("w2_input_scale", w2_input_scale) set_weight_attrs(w2_input_scale, extra_weight_attrs) else: @@ -298,14 +335,16 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: if self.block_quant: assert self.quant_config["is_dynamic"] == True if self.need_normalize_e4m3fn_to_e4m3fnuz: - w13_weight, w13_weight_scale, w13_input_scale = \ + w13_weight, w13_weight_scale, w13_input_scale = ( normalize_e4m3fn_to_e4m3fnuz( - layer.w13_weight, layer.w13_weight_scale, - layer.w13_input_scale) - w2_weight, w2_weight_scale, w2_input_scale = \ + layer.w13_weight, layer.w13_weight_scale, layer.w13_input_scale + ) + ) + w2_weight, w2_weight_scale, w2_input_scale = ( normalize_e4m3fn_to_e4m3fnuz( - layer.w2_weight, layer.w2_weight_scale, - layer.w2_input_scale) + layer.w2_weight, layer.w2_weight_scale, layer.w2_input_scale + ) + ) else: w13_weight = layer.w13_weight.data w13_weight_scale = layer.w13_weight_scale.data @@ -314,29 +353,26 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: # torch.compile() cannot use Parameter subclasses. layer.w13_weight = nn.Parameter(w13_weight, requires_grad=False) - layer.w13_weight_scale = nn.Parameter(w13_weight_scale, - requires_grad=False) + layer.w13_weight_scale = nn.Parameter(w13_weight_scale, requires_grad=False) layer.w2_weight = nn.Parameter(w2_weight, requires_grad=False) - layer.w2_weight_scale = nn.Parameter(w2_weight_scale, - requires_grad=False) + layer.w2_weight_scale = nn.Parameter(w2_weight_scale, requires_grad=False) shuffled_w13, shuffled_w2 = shuffle_weights( - layer.w13_weight.data, layer.w2_weight.data) + layer.w13_weight.data, layer.w2_weight.data + ) - layer.w13_weight = torch.nn.Parameter(shuffled_w13, - requires_grad=False) - layer.w2_weight = torch.nn.Parameter(shuffled_w2, - requires_grad=False) + layer.w13_weight = torch.nn.Parameter(shuffled_w13, requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, requires_grad=False) return else: # Fp8 moe kernels require a single activation scale. # We take the max of all the scales in case they differ. if not self.quant_config["is_dynamic"]: - if (layer.w13_input_scale is None - or layer.w2_input_scale is None): + if layer.w13_input_scale is None or layer.w2_input_scale is None: raise ValueError( "QuantConfig has static quantization, but found " - "activation scales are None.") + "activation scales are None." + ) # if (not all_close_1d(layer.w13_input_scale) # or not all_close_1d(layer.w2_input_scale)): # print( @@ -344,34 +380,40 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: # "fp8 MoE layer. Using the maximum across experts " # "for each layer.") layer.w13_input_scale = torch.nn.Parameter( - layer.w13_input_scale.max(), requires_grad=False) + layer.w13_input_scale.max(), requires_grad=False + ) layer.w2_input_scale = torch.nn.Parameter( - layer.w2_input_scale.max(), requires_grad=False) + layer.w2_input_scale.max(), requires_grad=False + ) if self.need_normalize_e4m3fn_to_e4m3fnuz: # Normalize the weights and scales - w13_weight, w13_weight_scale, w13_input_scale = \ + w13_weight, w13_weight_scale, w13_input_scale = ( normalize_e4m3fn_to_e4m3fnuz( - layer.w13_weight, layer.w13_weight_scale, - layer.w13_input_scale) - w2_weight, w2_weight_scale, w2_input_scale = \ + layer.w13_weight, layer.w13_weight_scale, layer.w13_input_scale + ) + ) + w2_weight, w2_weight_scale, w2_input_scale = ( normalize_e4m3fn_to_e4m3fnuz( - layer.w2_weight, layer.w2_weight_scale, - layer.w2_input_scale) + layer.w2_weight, layer.w2_weight_scale, layer.w2_input_scale + ) + ) # Reset the parameter - layer.w13_weight = torch.nn.Parameter(w13_weight, - requires_grad=False) + layer.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False) layer.w13_weight_scale = torch.nn.Parameter( - w13_weight_scale, requires_grad=False) + w13_weight_scale, requires_grad=False + ) if w13_input_scale is not None: layer.w13_input_scale = torch.nn.Parameter( - w13_input_scale, requires_grad=False) - layer.w2_weight = torch.nn.Parameter(w2_weight, - requires_grad=False) - layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, - requires_grad=False) + w13_input_scale, requires_grad=False + ) + layer.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter( + w2_weight_scale, requires_grad=False + ) if w2_input_scale is not None: layer.w2_input_scale = torch.nn.Parameter( - w2_input_scale, requires_grad=False) + w2_input_scale, requires_grad=False + ) # Fp8 moe kernel needs single weight scale for w13 per expert. # We take the max then dequant and requant each expert. @@ -382,25 +424,25 @@ def process_weights_after_loading(self, layer: nn.Module) -> None: start = 0 for shard_id in range(2): dq_weight = per_tensor_dequantize( - layer.w13_weight[expert_id][start:start + - shard_size, :], - layer.w13_weight_scale[expert_id][shard_id]) + layer.w13_weight[expert_id][start : start + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id], + ) quant_func = get_hip_quant(self.quant_type) - layer.w13_weight[expert_id][ - start:start + shard_size, :], _ = quant_func( - dq_weight, max_w13_scales[expert_id]) + layer.w13_weight[expert_id][start : start + shard_size, :], _ = ( + quant_func(dq_weight, max_w13_scales[expert_id]) + ) start += shard_size shuffled_w13, shuffled_w2 = shuffle_weights( - layer.w13_weight, layer.w2_weight) + layer.w13_weight, layer.w2_weight + ) - layer.w13_weight = torch.nn.Parameter(shuffled_w13, - requires_grad=False) - layer.w2_weight = torch.nn.Parameter(shuffled_w2, - requires_grad=False) + layer.w13_weight = torch.nn.Parameter(shuffled_w13, requires_grad=False) + layer.w2_weight = torch.nn.Parameter(shuffled_w2, requires_grad=False) - layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, - requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter( + max_w13_scales, requires_grad=False + ) return def apply( @@ -444,12 +486,15 @@ def apply( expert_mask=expert_map, activation=activation, quant_type=self.quant_type, - w1_scale=(layer.w13_weight_scale - if self.block_quant else layer.w13_weight_scale), - w2_scale=(layer.w2_weight_scale - if self.block_quant else layer.w2_weight_scale), + w1_scale=( + layer.w13_weight_scale if self.block_quant else layer.w13_weight_scale + ), + w2_scale=( + layer.w2_weight_scale if self.block_quant else layer.w2_weight_scale + ), a1_scale=layer.w13_input_scale, - a2_scale=layer.w2_input_scale) + a2_scale=layer.w2_input_scale, + ) def determine_expert_map( @@ -555,10 +600,8 @@ def __init__( tp_rank = 0 if self.tp_size == 1 else get_tp_group().rank_in_group self.global_num_experts = num_experts - # TODO: This slows load - # atom_config = get_current_atom_config() - # use_ep = self.tp_size > 1 and atom_config.enable_expert_parallel - use_ep = self.tp_size > 1 + atom_config = get_current_atom_config() + use_ep = self.tp_size > 1 and atom_config.enable_expert_parallel if use_ep: # Set TP size to 1 to adjust for EP and adjust EP size and rank @@ -585,11 +628,13 @@ def __init__( self.num_fused_shared_experts = ( config.n_shared_experts - if config is not None and hasattr(config, 'n_shared_experts') and is_rocm_aiter_fusion_shared_expert_enabled() + if config is not None + and hasattr(config, "n_shared_experts") + and is_rocm_aiter_fusion_shared_expert_enabled() else 0 ) self.routed_scaling_factor = ( - getattr(config, 'routed_scaling_factor', 1.0) + getattr(config, "routed_scaling_factor", 1.0) if config is not None and config.torch_dtype != torch.float16 else 1.0 ) @@ -625,7 +670,11 @@ def __init__( top_k=self.top_k, tp_rank=self.ep_rank if use_ep else tp_rank, tp_size=self.ep_size if use_ep else tp_size, - shared_experts_score=1.0 if is_rocm_aiter_fuse_routed_scaling_factor() else 1/self.routed_scaling_factor, + shared_experts_score=( + 1.0 + if is_rocm_aiter_fuse_routed_scaling_factor() + else 1 / self.routed_scaling_factor + ), max_num_tokens=16384, # Default max tokens is_EP=use_ep, ) @@ -656,8 +705,12 @@ def __init__( # for heuristic purposes, so it must be initialized first. if quant_config["quant_type"] == QuantType.No: self.quant_method: Optional[QuantizeMethodBase] = ( - UnquantizedFusedMoEMethod()) - elif quant_config["quant_dtype"] == torch.float8_e4m3fnuz or quant_config["quant_dtype"] == torch.float8_e4m3fn: + UnquantizedFusedMoEMethod() + ) + elif ( + quant_config["quant_dtype"] == torch.float8_e4m3fnuz + or quant_config["quant_dtype"] == torch.float8_e4m3fn + ): self.quant_method = Fp8MoEMethod(quant_config) else: raise ValueError(f"Unsupported quant dtype: {quant_config['quant_dtype']}") @@ -1027,7 +1080,9 @@ def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor) if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): # Default set to False. (May have to add shared expert outputs.) - final_hidden_states = get_tp_group().all_reduce(final_hidden_states, ca_fp8_quant=False) + final_hidden_states = get_tp_group().all_reduce( + final_hidden_states, ca_fp8_quant=False + ) return final_hidden_states From dece197060ee1e3b6ff307f5c084e9014551aa7a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 14 Oct 2025 06:27:02 +0000 Subject: [PATCH 056/119] check multiprocessing before initialization --- atom/model_engine/engine_core_mgr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 81e8f5f429..0f0b4b53e8 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -136,8 +136,8 @@ def launch_engine_core(config: Config): input_address = get_open_zmq_ipc_path() output_address = get_open_zmq_ipc_path() import torch - - torch.multiprocessing.set_start_method("spawn") + if torch.multiprocessing.get_start_method(allow_none=True) is None: + torch.multiprocessing.set_start_method("spawn", force=False) process = multiprocessing.Process( target=EngineCore.run_engine, name=f"EngineCore", From e569177232cd88434e2f780a611fed15f0ebd3be Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Tue, 14 Oct 2025 07:18:27 +0000 Subject: [PATCH 057/119] update readme --- README.md | 103 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index fbbff70d33..b0f49b55a8 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,87 @@ -# Nano-vLLM +# ATOM -A lightweight vLLM implementation built from scratch. +**ATOM** (AI Tensor Offline Model) is a lightweight vLLM-like implementation, focusing on integration and optimization based on [aiter](https://github.com/ROCm/aiter). -## Key Features +## πŸš€ Features -* πŸš€ **Fast offline inference** - Comparable inference speeds to vLLM -* πŸ“– **Readable codebase** - Clean implementation in ~ 1,200 lines of Python code -* ⚑ **Optimization Suite** - Prefix caching, Tensor Parallelism, Torch compilation, CUDA graph, etc. +- **ROCm Optimized**: Built on AMD's ROCm platform with torch compile support +- **Model Support**: Compatible with Qwen, Llama, and Mixtral models +- **Easy Integration**: Simple API for quick deployment -## Installation +## πŸ“‹ Requirements + +- AMD GPU with ROCm support +- Docker + +## πŸ› οΈ Installation + +### 1. Pull Docker Image + +```bash +docker pull rocm/pytorch:rocm7.0.2_ubuntu24.04_py3.12_pytorch_release_2.8.0 +``` + +### 2. Run Docker Container + +```bash +docker run -it --network=host \ + --device=/dev/kfd \ + --device=/dev/dri \ + --group-add video \ + --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -v $HOME:/home/$USER \ + -v /mnt:/mnt \ + -v /data:/data \ + --shm-size=16G \ + --ulimit memlock=-1 \ + --ulimit stack=67108864 \ + rocm/pytorch:rocm7.0.2_ubuntu24.04_py3.12_pytorch_release_2.8.0 +``` + +### 3. Install Dependencies ```bash -pip install git+https://github.com/GeeeekExplorer/nano-vllm.git +pip install transformers zmq ninja xxhash ``` -## Manual Download +### 4. Clone and Setup -If you prefer to download the model weights manually, use the following command: ```bash -huggingface-cli download --resume-download Qwen/Qwen3-0.6B \ - --local-dir ~/huggingface/Qwen3-0.6B/ \ - --local-dir-use-symlinks False +git clone https://github.com/valarLip/atom.git +git clone --recursive https://github.com/ROCm/aiter.git +cd aiter +python3 setup.py develop +cd .. ``` -## Quick Start +## πŸ’‘ Usage + +### Basic Example -See `example.py` for usage. The API mirrors vLLM's interface with minor differences in the `LLM.generate` method: -```python -from atom import LLM, SamplingParams -llm = LLM("/YOUR/MODEL/PATH", enforce_eager=True, tensor_parallel_size=1) -sampling_params = SamplingParams(temperature=0.6, max_tokens=256) -prompts = ["Hello, Nano-vLLM."] -outputs = llm.generate(prompts, sampling_params) -outputs[0]["text"] +The default optimization level is 3 (running with torch compile). Supported models include **Qwen**, **Llama**, and **Mixtral**. + +```bash +python example.py --model meta-llama/Meta-Llama-3-8B ``` -## Benchmark +> **Note:** First-time execution may take approximately 10 minutes for model compilation. -See `bench.py` for benchmark. +### Performance Benchmarking -**Test Configuration:** -- Hardware: RTX 4070 Laptop (8GB) -- Model: Qwen3-0.6B -- Total Requests: 256 sequences -- Input Length: Randomly sampled between 100–1024 tokens -- Output Length: Randomly sampled between 100–1024 tokens +Run performance tests to compare ATOM against vLLM: -**Performance Results:** -| Inference Engine | Output Tokens | Time (s) | Throughput (tokens/s) | -|----------------|-------------|----------|-----------------------| -| vLLM | 133,966 | 98.37 | 1361.84 | -| Nano-vLLM | 133,966 | 93.41 | 1434.13 | +```bash +python bench_test.py --model Qwen/Qwen3-0.6B +``` +## πŸ“Š Performance Comparison -## Star History +ATOM demonstrates significant performance improvements over vLLM: -[![Star History Chart](https://api.star-history.com/svg?repos=GeeeekExplorer/nano-vllm&type=Date)](https://www.star-history.com/#GeeeekExplorer/nano-vllm&Date) \ No newline at end of file +| Model | Framework | Tokens | Time | Throughput | +|-------|-----------|--------|------|------------| +| **Qwen3-0.6B** | ATOM | 4096 | 0.25s | **16,643.74 tok/s** | +| Qwen3-0.6B | vLLM | 4096 | 0.63s | 6,543.06 tok/s | +| **Llama-3.1-8B-Instruct-FP8-KV** | ATOM | 4096 | 0.68s | **5,983.37 tok/s** | +| Llama-3.1-8B-Instruct-FP8-KV | vLLM | 4096 | 1.68s | 2,432.62 tok/s | \ No newline at end of file From 5e62de1ad5535bc97f4ef1b04788521bdde3181b Mon Sep 17 00:00:00 2001 From: chenjun Date: Tue, 14 Oct 2025 13:48:13 +0000 Subject: [PATCH 058/119] use aiter concat_and_cache_mla --- atom/model_ops/attention_mla.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index 2e0a1c5cee..a064cfbb25 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -7,9 +7,8 @@ from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as aiter_triton_fp8_bmm, ) -from vllm import _custom_ops as ops -from aiter import flash_attn_varlen_func, dtypes +from aiter import flash_attn_varlen_func, concat_and_cache_mla, dtypes from aiter.mla import mla_decode_fwd from typing import Optional @@ -282,7 +281,7 @@ def forward( self.rotary_emb(positions, prefill_q_pe, k_pe) if kv_cache.numel() > 0: - ops.concat_and_cache_mla( + concat_and_cache_mla( k_c_normed, k_pe.squeeze(1), kv_cache, @@ -299,7 +298,7 @@ def forward( self.rotary_emb(positions, decode_q_pe, k_pe) if kv_cache.numel() > 0: - ops.concat_and_cache_mla( + concat_and_cache_mla( k_c_normed, k_pe.squeeze(1), kv_cache, From b893b57476022cc3a2f131c74ac511682b027d23 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Wed, 15 Oct 2025 17:00:23 +0800 Subject: [PATCH 059/119] update install flow; readme; (#13) --- .gitignore | 4 +- README.md | 15 +++----- example.py => atom/bench/example.py | 0 bench_test.py => atom/bench/one_batch.py | 7 ++-- atom/model_engine/model_runner.py | 44 +++++++++++++++------- atom/model_ops/topK.py | 5 ++- atom_example.md | 29 --------------- bench.py | 47 ------------------------ pyproject.toml | 19 ++++++---- 9 files changed, 55 insertions(+), 115 deletions(-) rename example.py => atom/bench/example.py (100%) rename bench_test.py => atom/bench/one_batch.py (95%) delete mode 100644 atom_example.md delete mode 100644 bench.py diff --git a/.gitignore b/.gitignore index 15b593a6e1..1cf7cc705b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -# aiter version -aiter/_version.py +# atom version +atom/_version.py # Prerequisites *.d diff --git a/README.md b/README.md index b0f49b55a8..9e750cd236 100644 --- a/README.md +++ b/README.md @@ -39,20 +39,15 @@ docker run -it --network=host \ rocm/pytorch:rocm7.0.2_ubuntu24.04_py3.12_pytorch_release_2.8.0 ``` -### 3. Install Dependencies - -```bash -pip install transformers zmq ninja xxhash -``` - -### 4. Clone and Setup +### 3. Clone and Setup ```bash git clone https://github.com/valarLip/atom.git git clone --recursive https://github.com/ROCm/aiter.git cd aiter python3 setup.py develop -cd .. +cd ../atom +pip install . ``` ## πŸ’‘ Usage @@ -62,7 +57,7 @@ cd .. The default optimization level is 3 (running with torch compile). Supported models include **Qwen**, **Llama**, and **Mixtral**. ```bash -python example.py --model meta-llama/Meta-Llama-3-8B +python -m atom.bench.example.py --model meta-llama/Meta-Llama-3-8B ``` > **Note:** First-time execution may take approximately 10 minutes for model compilation. @@ -72,7 +67,7 @@ python example.py --model meta-llama/Meta-Llama-3-8B Run performance tests to compare ATOM against vLLM: ```bash -python bench_test.py --model Qwen/Qwen3-0.6B +python -m atom.bench.one_batch.py --model Qwen/Qwen3-0.6B ``` ## πŸ“Š Performance Comparison diff --git a/example.py b/atom/bench/example.py similarity index 100% rename from example.py rename to atom/bench/example.py diff --git a/bench_test.py b/atom/bench/one_batch.py similarity index 95% rename from bench_test.py rename to atom/bench/one_batch.py index 4c7674bc10..fa4cbc8289 100644 --- a/bench_test.py +++ b/atom/bench/one_batch.py @@ -1,6 +1,4 @@ import argparse -import os -import pstats import time from random import randint, seed @@ -38,6 +36,8 @@ def main(): llm = LLMEngine( path, enforce_eager=False, + tensor_parallel_size=1, + kv_cache_block_size=16, max_model_len=4096, port=12345, torch_profiler_dir="./log", @@ -46,7 +46,6 @@ def main(): ), ) - sampling_params = [ SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=16) for _ in range(num_seqs) @@ -54,7 +53,7 @@ def main(): print("------start generating------") - #warm up + # warm up outputs_ = llm.generate(["Benchmark: "], SamplingParams()) t = time.time() outputs = llm.generate(LONG_PROMPT, sampling_params) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index b14835b400..3c88fd8a23 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -138,7 +138,6 @@ def __init__(self, rank: int, config: Config): if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") - def exit(self): if not self.still_running: return @@ -203,13 +202,23 @@ def allocate_decode_vars(self): "block_tables": CpuGpuBuffer( max_bs, max_num_blocks, dtype=torch.int32, device=self.device ), - "cu_seqlens_q": CpuGpuBuffer(max_bs + 1, dtype=torch.int32, device=self.device), - "kv_indptr": CpuGpuBuffer(max_bs + 1, dtype=torch.int32, device=self.device), - "kv_indices": CpuGpuBuffer(max_bs * max_num_blocks, dtype=torch.int32, device=self.device), - "kv_last_page_lens": CpuGpuBuffer(max_bs, dtype=torch.int32, device=self.device), + "cu_seqlens_q": CpuGpuBuffer( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "kv_indptr": CpuGpuBuffer( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "kv_indices": CpuGpuBuffer( + max_bs * max_num_blocks, dtype=torch.int32, device=self.device + ), + "kv_last_page_lens": CpuGpuBuffer( + max_bs, dtype=torch.int32, device=self.device + ), "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), } - self.decode_vars["cu_seqlens_q"].cpu.copy_(torch.arange(0, max_bs + 1, step=1, dtype=torch.int32)) + self.decode_vars["cu_seqlens_q"].cpu.copy_( + torch.arange(0, max_bs + 1, step=1, dtype=torch.int32) + ) self.decode_vars["cu_seqlens_q"].copy_to_gpu() self.decode_vars["kv_last_page_lens"].cpu.fill_(1) @@ -397,8 +406,8 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): positions = [] slot_mapping = [] context_lens = [] - cu_seqlens_q = [0] dropout_p = 0.0 + max_q_len = 1 self.total_blocks = 0 context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] @@ -418,10 +427,13 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self.kv_indices: list[int] = [] self.kv_indptr: list[int] = [0] self.kv_last_page_lens: list[int] = [] + qlens = [1 for seq in scheduled_batchs.seqs] # TODO support mtp + num_scheduled_tokens = np.array(qlens, dtype=np.int32) + max_q_len = max(qlens) + cu_seqlens_q = np.cumsum(num_scheduled_tokens, dtype=np.int32) for seq in scheduled_batchs.seqs: current_seq_len = len(seq) - cu_seqlens_q.append(cu_seqlens_q[-1] + 1) self._update_paged_kv_tensors(seq.block_table, current_seq_len) block_tables = self.prepare_block_tables(scheduled_batchs.seqs) @@ -436,11 +448,15 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): if isinstance(self.model, DeepseekV2ForCausalLM) and len(self.kv_indptr) > 0: # extend to the maximum number of blocks as returned by the scheduler self.kv_indices.extend([0] * (self.total_blocks - len(self.kv_indices))) - var["kv_indices"].np[: self.total_blocks] = np.array(self.kv_indices, dtype=np.int64) + var["kv_indices"].np[: self.total_blocks] = np.array( + self.kv_indices, dtype=np.int64 + ) var["kv_indptr"].np[: bs + 1] = np.array(self.kv_indptr, dtype=np.int64) var["kv_indptr"].np[bs + 1 : graph_bs + 1] = var["kv_indptr"].np[bs] - var["kv_last_page_lens"].np[:bs] = np.array(self.kv_last_page_lens, dtype=np.int64) - var["cu_seqlens_q"].np[:bs + 1] = np.array(cu_seqlens_q, dtype=np.int64) + var["kv_last_page_lens"].np[:bs] = np.array( + self.kv_last_page_lens, dtype=np.int64 + ) + var["cu_seqlens_q"].np[1 : bs + 1] = np.array(cu_seqlens_q, dtype=np.int64) for el in [ "slot_mapping", @@ -451,19 +467,19 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): "cu_seqlens_q", "kv_indices", "kv_indptr", - "kv_last_page_lens" + "kv_last_page_lens", ]: var[el].copy_to_gpu() set_context( False, batch_size=bs, graph_bs=graph_bs, - cu_seqlens_q=var["cu_seqlens_q"].gpu[:bs + 1], + cu_seqlens_q=var["cu_seqlens_q"].gpu[: bs + 1], slot_mapping=var["slot_mapping"].gpu[:bs], context_lens=var["context_lens"].gpu[:bs], block_tables=var["block_tables"].gpu[:bs, : block_tables.shape[1]], dropout_p=dropout_p, - max_q_len=1, + max_q_len=max_q_len, kv_indptr=var["kv_indptr"].gpu[: bs + 1], kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index 2b3420f849..64d76048ec 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -1,13 +1,14 @@ -import torch from functools import cache, lru_cache from typing import Optional +import torch + def is_rocm_aiter_fusion_shared_expert_enabled(): return True def is_rocm_aiter_fuse_routed_scaling_factor(): - return False + return True aiter_topK_meta_data = None diff --git a/atom_example.md b/atom_example.md deleted file mode 100644 index 2ad2fcb8fb..0000000000 --- a/atom_example.md +++ /dev/null @@ -1,29 +0,0 @@ -# Atom - -A lightweight vLLM implementation built from scratch. - -## Installation - -```bash -git clone https://github.com/valarLip/atom.git -cd atom -``` - -## Example - -```bash -python example.py --model meta-llama/Meta-Llama-3-8B -``` - -The default level is 3(running with torch compile), the model we support now is Qwen, Llama, Mixtral, you can set the model name or path using --model - -## Perf running - -```bash -python bench_test.py --model Qwen/Qwen3-0.6B -``` - -You can get the performance detail: -```bash -Total: 4096tok, Time: 0.31s, Throughput: 13333.85tok/s -``` \ No newline at end of file diff --git a/bench.py b/bench.py deleted file mode 100644 index 381008b60c..0000000000 --- a/bench.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import time -from random import randint, seed -from atom import LLMEngine, SamplingParams -from atom.config import CompilationConfig - -# from vllm import LLM, SamplingParams - - -def main(): - seed(0) - num_seqs = 256 - max_input_len = 1024 - max_ouput_len = 1024 - - path = os.path.expanduser("/mnt/raid0/lirong/Qwen3-0.6B/") - llm = LLMEngine(path, enforce_eager=False, max_model_len=4096, compilation_config=CompilationConfig( - level = 3, - # cudagraph_capture_sizes=[1, 2, 4, 8] + list(range(16, 512 + 1, 16)) - )) - - prompt_token_ids = [ - [randint(0, 10000) for _ in range(randint(100, max_input_len))] - for _ in range(num_seqs) - ] - sampling_params = [ - SamplingParams( - temperature=0.6, ignore_eos=True, max_tokens=randint(100, max_ouput_len) - ) - for _ in range(num_seqs) - ] - # uncomment the following line for vllm - # prompt_token_ids = [dict(prompt_token_ids=p) for p in prompt_token_ids] - - llm.generate(["Benchmark: "], SamplingParams()) - t = time.time() - llm.generate(prompt_token_ids, sampling_params, use_tqdm=False) - t = time.time() - t - total_tokens = sum(sp.max_tokens for sp in sampling_params) - throughput = total_tokens / t - print( - f"Total: {total_tokens}tok, Time: {t:.2f}s, Throughput: {throughput:.2f}tok/s" - ) - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index ac8cdfc342..c0027364d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,26 @@ [build-system] -requires = ["setuptools>=61"] -build-backend = "setuptools.build_meta" +requires = ["setuptools_scm[toml]>=6.2"] [project] name = "atom" -version = "0.1.0" authors = [{ name = "ValarLip" }] license = "MIT" readme = "README.md" description = "a lightweight vLLM implementation built from scratch" requires-python = ">=3.10,<3.13" -dependencies = [ -] +dynamic = ["version"] +dependencies = ["transformers", "zmq", "xxhash"] [project.urls] -Homepage="https://github.com/GeeeekExplorer/nano-vllm" +Homepage = "https://github.com/valarLip/atom" +Repository = "https://github.com/valarLip/atom" +Issues = "https://github.com/valarLip/atom/issues" + +[tool.setuptools_scm] +write_to = "atom/_version.py" +write_to_template = "__version__ = '{version}'\n" +fallback_version = "0.1.0" [tool.setuptools.packages.find] where = ["."] -include = ["python*"] \ No newline at end of file +include = ["atom*"] From ba7ca0ee8b00f5dff7057690869d371fa4166374 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Wed, 15 Oct 2025 15:43:37 +0000 Subject: [PATCH 060/119] log cudagraph info --- atom/model_engine/engine_core.py | 8 ++++++-- atom/model_engine/model_runner.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 33ec5fc54c..a4c7d0b68c 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -82,8 +82,12 @@ def __init__(self, config: Config, input_address: str, output_address: str): config.num_kvcache_blocks = num_blocks if not config.enforce_eager: - cap_cost = self.runner_mgr.call_func("capture_cudagraph", wait_out=True) - logger.info(f"{self.label}: cudagraph capture cost: {cap_cost} seconds") + cap_cost, bs = self.runner_mgr.call_func( + "capture_cudagraph", wait_out=True + ) + logger.info( + f"{self.label}: cudagraph capture{bs} cost: {cap_cost} seconds" + ) good = True finally: logger.info( diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 3c88fd8a23..d754c80c59 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -623,4 +623,4 @@ def capture_cudagraph(self): torch.cuda.synchronize() reset_context() self.graph_bs.sort(reverse=False) - return time.time() - start_time + return time.time() - start_time, self.graph_bs From 9dd7621c9645d3fef4f39a164f2ecd677bf302dd Mon Sep 17 00:00:00 2001 From: amd-ruitang3 <145657428+amd-ruitang3@users.noreply.github.com> Date: Thu, 16 Oct 2025 16:49:13 +0800 Subject: [PATCH 061/119] fix_minor_issue (#15) * fix_minor_issue * update --- README.md | 8 +++----- atom/config.py | 8 +++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9e750cd236..23e3786952 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,8 @@ docker run -it --network=host \ ### 3. Clone and Setup ```bash +pip install aiter -i https://mkmartifactory.amd.com/artifactory/api/pypi/hw-orc3pypi-prod-local/simple git clone https://github.com/valarLip/atom.git -git clone --recursive https://github.com/ROCm/aiter.git -cd aiter -python3 setup.py develop cd ../atom pip install . ``` @@ -57,7 +55,7 @@ pip install . The default optimization level is 3 (running with torch compile). Supported models include **Qwen**, **Llama**, and **Mixtral**. ```bash -python -m atom.bench.example.py --model meta-llama/Meta-Llama-3-8B +python -m atom.bench.example --model meta-llama/Meta-Llama-3-8B ``` > **Note:** First-time execution may take approximately 10 minutes for model compilation. @@ -67,7 +65,7 @@ python -m atom.bench.example.py --model meta-llama/Meta-Llama-3-8B Run performance tests to compare ATOM against vLLM: ```bash -python -m atom.bench.one_batch.py --model Qwen/Qwen3-0.6B +python -m atom.bench.one_batch --model Qwen/Qwen3-0.6B ``` ## πŸ“Š Performance Comparison diff --git a/atom/config.py b/atom/config.py index 30ed3923d1..a451b63813 100644 --- a/atom/config.py +++ b/atom/config.py @@ -376,9 +376,9 @@ def _set_cudagraph_sizes(self): def __post_init__(self): # assert os.path.isdir(self.model) - assert (self.kv_cache_block_size % 16 == 0 or self.kv_cache_block_size == 1), ( - f"kv_cache_block_size ({self.kv_cache_block_size}) must be a multiple of 16 or 1" - ) + assert ( + self.kv_cache_block_size % 16 == 0 or self.kv_cache_block_size == 1 + ), f"kv_cache_block_size ({self.kv_cache_block_size}) must be a multiple of 16 or 1" assert 1 <= self.tensor_parallel_size <= 8 self.hf_config = AutoConfig.from_pretrained(self.model) self.quant_config = get_quant_config(self.hf_config) @@ -386,6 +386,8 @@ def __post_init__(self): self.max_model_len, self.hf_config.max_position_embeddings ) assert self.max_num_batched_tokens >= self.max_model_len + if self.torch_profiler_dir is not None: + os.makedirs(self.torch_profiler_dir, exist_ok=True) assert self.torch_profiler_dir is None or os.path.isdir( self.torch_profiler_dir ), f"torch_profiler_dir {self.torch_profiler_dir} is not a valid directory" From 00d269c5b16ff2b10c2f4b87c32a20fd40e8fa27 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Fri, 17 Oct 2025 10:41:48 +0800 Subject: [PATCH 062/119] add openai server for accuracy testing (#12) * add openai server for accuracy testing * support custom stop string * refactor * fix readme --- README.md | 26 ++- atom/entrypoints/openai_server.py | 275 ++++++++++++++++++++++++++++++ atom/model_engine/llm_engine.py | 12 +- atom/model_engine/scheduler.py | 11 +- atom/model_engine/sequence.py | 4 +- atom/sampling_params.py | 1 + 6 files changed, 325 insertions(+), 4 deletions(-) create mode 100644 atom/entrypoints/openai_server.py diff --git a/README.md b/README.md index 23e3786952..968b6efdbd 100644 --- a/README.md +++ b/README.md @@ -77,4 +77,28 @@ ATOM demonstrates significant performance improvements over vLLM: | **Qwen3-0.6B** | ATOM | 4096 | 0.25s | **16,643.74 tok/s** | | Qwen3-0.6B | vLLM | 4096 | 0.63s | 6,543.06 tok/s | | **Llama-3.1-8B-Instruct-FP8-KV** | ATOM | 4096 | 0.68s | **5,983.37 tok/s** | -| Llama-3.1-8B-Instruct-FP8-KV | vLLM | 4096 | 1.68s | 2,432.62 tok/s | \ No newline at end of file +| Llama-3.1-8B-Instruct-FP8-KV | vLLM | 4096 | 1.68s | 2,432.62 tok/s | + +### Accuracy Benchmarking + +First, install `lm-eval` to test model accuracy: + +```bash +pip install lm-eval[api] +``` + +Next, start an OpenAI-compatible server using `openai_server.py`: + +```bash +python -m atom.entrypoints.openai_server --model meta-llama/Meta-Llama-3-8B +``` + +Finally, run the evaluation by choosing your datasets: + +```bash +lm_eval --model local-completions \ + --model_args model=meta-llama/Meta-Llama-3-8B,base_url=http://localhost:8000/v1/completions,num_concurrent=8,max_retries=3,tokenized_requests=False \ + --tasks gsm8k \ + --num_fewshot 3 \ + --batch_size 16 +``` \ No newline at end of file diff --git a/atom/entrypoints/openai_server.py b/atom/entrypoints/openai_server.py new file mode 100644 index 0000000000..3dac7be376 --- /dev/null +++ b/atom/entrypoints/openai_server.py @@ -0,0 +1,275 @@ +import argparse +import asyncio +import json +import time +import uuid +from datetime import datetime +from typing import List, Optional, Dict, Any + +import torch +import uvicorn +from fastapi import FastAPI, HTTPException +from fastapi.responses import StreamingResponse, JSONResponse +from pydantic import BaseModel +from transformers import AutoTokenizer + +from atom import LLMEngine, SamplingParams +from atom.config import CompilationConfig + + +# Request models +class CompletionRequest(BaseModel): + model: str = "atom" + prompt: str | List[str] + max_tokens: Optional[int] = 256 + temperature: Optional[float] = 0.7 + top_p: Optional[float] = 1.0 + n: Optional[int] = 1 + stream: Optional[bool] = False + stop: Optional[str | List[str]] = None + presence_penalty: Optional[float] = 0.0 + frequency_penalty: Optional[float] = 0.0 + + +class ChatMessage(BaseModel): + role: str + content: str + + +class ChatCompletionRequest(BaseModel): + model: str = "atom" + messages: List[ChatMessage] + max_tokens: Optional[int] = 256 + temperature: Optional[float] = 0.7 + top_p: Optional[float] = 1.0 + n: Optional[int] = 1 + stream: Optional[bool] = False + stop: Optional[str | List[str]] = None + presence_penalty: Optional[float] = 0.0 + frequency_penalty: Optional[float] = 0.0 + + +# Global engine and tokenizer +llm_engine: Optional[LLMEngine] = None +tokenizer: Optional[Any] = None +# Lock to serialize generation calls +generation_lock: Optional[asyncio.Lock] = None +# Log file path +log_file_path: Optional[str] = None + + +def parse_size_list(size_str: str) -> List[int]: + import ast + try: + return ast.literal_eval(size_str) + except (ValueError, SyntaxError) as e: + raise ValueError(f"Error list size: {size_str}") from e + + +def log_request_response(request_id: str, prompts: List[str], outputs: List[Dict], endpoint: str): + """Log request inputs and outputs to JSONL file""" + if log_file_path is None: + return + + log_entry = { + "timestamp": datetime.now().isoformat(), + "request_id": request_id, + "endpoint": endpoint, + "prompts": prompts, + "outputs": [{"text": out["text"]} for out in outputs] + } + + try: + with open(log_file_path, "a") as f: + f.write(json.dumps(log_entry) + "\n") + except Exception as e: + print(f"Error writing to log file: {e}") + + +def init_engine(args): + global llm_engine, tokenizer, generation_lock, log_file_path + + tokenizer = AutoTokenizer.from_pretrained(args.model) + # LLMEngine needs a separate port for distributed communication + engine_port = args.port + 1000 # e.g., if API port is 8000, engine uses 9000 + llm_engine = LLMEngine( + args.model, + enforce_eager=args.enforce_eager, + tensor_parallel_size=args.tensor_parallel_size, + kv_cache_dtype=args.kv_cache_dtype, + port=engine_port, + torch_profiler_dir=args.torch_profiler_dir, + compilation_config=CompilationConfig( + level=args.level, + cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes) + ) + ) + generation_lock = asyncio.Lock() + + # Set log file path if enabled + if args.log_requests: + log_file_path = args.log_file + print(f"Request/Response logging enabled: {log_file_path}") + + +app = FastAPI(title="Atom OpenAI-Compatible API") + + +@app.post("/v1/completions") +async def create_completion(request: CompletionRequest): + if llm_engine is None or generation_lock is None: + raise HTTPException(status_code=503, detail="Engine not initialized") + + request_id = f"cmpl-{uuid.uuid4().hex[:8]}" + prompts = [request.prompt] if isinstance(request.prompt, str) else request.prompt + + sampling_params = SamplingParams( + temperature=request.temperature, + max_tokens=request.max_tokens, + stop_strings=request.stop, + ) + + # Use lock to serialize generation calls and run in thread pool to avoid blocking + async with generation_lock: + loop = asyncio.get_event_loop() + outputs = await loop.run_in_executor( + None, + llm_engine.generate, + prompts, + sampling_params + ) + + # Log request and response + log_request_response(request_id, prompts, outputs, "/v1/completions") + + choices = [] + for i, output in enumerate(outputs): + choices.append({ + "text": output["text"], + "index": i, + "logprobs": None, + "finish_reason": "stop" + }) + + return JSONResponse({ + "id": request_id, + "object": "text_completion", + "created": int(time.time()), + "model": request.model, + "choices": choices, + "usage": { + "prompt_tokens": sum(len(tokenizer.encode(p)) for p in prompts), + "completion_tokens": sum(len(tokenizer.encode(c["text"])) for c in choices), + "total_tokens": sum(len(tokenizer.encode(p)) for p in prompts) + sum(len(tokenizer.encode(c["text"])) for c in choices) + } + }) + + +@app.post("/v1/chat/completions") +async def create_chat_completion(request: ChatCompletionRequest): + if llm_engine is None or tokenizer is None or generation_lock is None: + raise HTTPException(status_code=503, detail="Engine not initialized") + + request_id = f"chatcmpl-{uuid.uuid4().hex[:8]}" + + # Convert chat messages to prompt + messages = [{"role": msg.role, "content": msg.content} for msg in request.messages] + + try: + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + ) + except Exception as e: + # Fallback: simple concatenation + prompt = "\n".join([f"{msg.role}: {msg.content}" for msg in request.messages]) + + sampling_params = SamplingParams( + temperature=request.temperature, + max_tokens=request.max_tokens, + stop_strings=request.stop, + ) + + # Use lock to serialize generation calls and run in thread pool to avoid blocking + async with generation_lock: + loop = asyncio.get_event_loop() + outputs = await loop.run_in_executor( + None, + llm_engine.generate, + [prompt], + sampling_params + ) + + # Log request and response + log_request_response(request_id, [prompt], outputs, "/v1/chat/completions") + + completion_text = outputs[0]["text"] if outputs else "" + + return JSONResponse({ + "id": request_id, + "object": "chat.completion", + "created": int(time.time()), + "model": request.model, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": completion_text, + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": len(tokenizer.encode(prompt)), + "completion_tokens": len(tokenizer.encode(completion_text)), + "total_tokens": len(tokenizer.encode(prompt)) + len(tokenizer.encode(completion_text)) + } + }) + + +@app.get("/v1/models") +async def list_models(): + return JSONResponse({ + "object": "list", + "data": [{ + "id": "atom", + "object": "model", + "created": int(time.time()), + "owned_by": "atom" + }] + }) + + +@app.get("/health") +async def health(): + return {"status": "healthy", "engine_ready": llm_engine is not None} + + +def main(): + parser = argparse.ArgumentParser(description="Atom OpenAI-Compatible Serving") + + parser.add_argument("--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path") + parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host") + parser.add_argument("--port", type=int, default=8000, help="API server port (engine will use port+1000)") + parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1) + parser.add_argument("--kv_cache_dtype", choices=["bf16", "fp8"], type=str, default="bf16") + parser.add_argument("--enforce-eager", action="store_true", help="Enforce eager mode") + parser.add_argument("--enable_prefix_caching", action="store_true", help="Enable prefix caching") + parser.add_argument("--cudagraph-capture-sizes", type=str, default="[1,2,4,8,16]") + parser.add_argument("--level", type=int, default=3, help="Compilation level") + parser.add_argument("--torch-profiler-dir", type=str, default=None) + parser.add_argument("--log-requests", action="store_true", help="Enable request/response logging") + parser.add_argument("--log-file", type=str, default="serving_requests.jsonl", help="Path to log file for requests/responses") + + args = parser.parse_args() + + print(f"Initializing engine with model: {args.model}") + init_engine(args) + print("Engine initialized successfully") + + print(f"Starting server at http://{args.host}:{args.port}") + uvicorn.run(app, host=args.host, port=args.port, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 9e5ca73424..42494caca2 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -93,7 +93,17 @@ def preprocess( if isinstance(prompt_or_tokens, str) else prompt_or_tokens ) - seq = Sequence(tokens, self.block_size, sampling_params) + + stop_token_sequences = [] + if sampling_params.stop_strings: + stops = [sampling_params.stop_strings] if isinstance(sampling_params.stop_strings, str) else sampling_params.stop_strings + for stop_str in stops: + # Encode the full stop string as a sequence of tokens + stop_tokens = self.tokenizer.encode(stop_str, add_special_tokens=False) + if stop_tokens: + stop_token_sequences.append(stop_tokens) + + seq = Sequence(tokens, self.block_size, sampling_params, stop_token_sequences) seq.arrive_time = time.time() self.requests[seq.id] = seq print( diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index cc0dfdb3ce..d72117b560 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -101,7 +101,16 @@ def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): seq.append_token(token_id) # seq.token_ids[-2] = token_id leave_reason = None - if not seq.ignore_eos and token_id == self.eos_token_id: + + # Check if sequence ends with any stop sequence + stop_matched = False + for stop_seq in seq.stop_token_sequences: + if len(seq.token_ids) >= len(stop_seq): + if seq.token_ids[-len(stop_seq):] == stop_seq: + stop_matched = True + break + + if (not seq.ignore_eos and token_id == self.eos_token_id) or stop_matched: leave_reason = "eos" elif seq.num_completion_tokens == seq.max_tokens: leave_reason = "max_tokens" diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index e7cd8e0f50..da5abdcc0b 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -22,7 +22,7 @@ class Sequence: counter = count() def __init__( - self, token_ids: list[int], block_size: int, sampling_params=SamplingParams() + self, token_ids: list[int], block_size: int, sampling_params=SamplingParams(), stop_token_sequences: list[list[int]] = None ): self.block_size = block_size self.id = next(Sequence.counter) @@ -36,6 +36,8 @@ def __init__( self.temperature = sampling_params.temperature self.max_tokens = sampling_params.max_tokens self.ignore_eos = sampling_params.ignore_eos + self.stop_strings = sampling_params.stop_strings + self.stop_token_sequences = stop_token_sequences or [] # statistics fields self.arrive_time = 0.0 diff --git a/atom/sampling_params.py b/atom/sampling_params.py index 67d60bb465..667ab28e88 100644 --- a/atom/sampling_params.py +++ b/atom/sampling_params.py @@ -6,3 +6,4 @@ class SamplingParams: temperature: float = 1.0 max_tokens: int = 64 ignore_eos: bool = False + stop_strings: str | list[str] = None From 7843c55a15402d4ce3120e3c8c1c19807dfccbf7 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 15 Oct 2025 14:05:24 +0800 Subject: [PATCH 063/119] draft deffer output --- atom/model_engine/engine_core.py | 5 +- atom/model_engine/model_runner.py | 330 ++++++++++++++++++++++++++++-- atom/model_engine/scheduler.py | 174 +++++++++++++++- 3 files changed, 475 insertions(+), 34 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index a4c7d0b68c..e3264b6080 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -138,8 +138,9 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() - out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) - self.scheduler.postprocess(scheduled_batchs.seqs, out) + out, prev_seqs = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) + # scheduled_batchs.prev_seqs = prev_seqs + self.scheduler.postprocess(scheduled_batchs.seqs, out, prev_seqs) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] ) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index d754c80c59..3b4c242b18 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -3,7 +3,9 @@ import signal import time import weakref -from typing import Optional +from typing import Optional, Union, Any +import queue +from collections import deque import numpy as np import torch @@ -13,7 +15,7 @@ from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture from atom.config import Config, set_current_atom_config -from atom.model_engine.scheduler import ScheduledBatchs +from atom.model_engine.scheduler import ScheduledBatchs, InputBatch from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler @@ -26,6 +28,7 @@ logger = logging.getLogger("atom") from atom.utils.forward_context import AttentionMetadata, set_forward_context +from atom.model_engine.outputs import ModelRunnerOutput, AsyncModelRunnerOutput suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -34,6 +37,54 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } +# Wrapper for ModelRunnerOutput to support overlapped execution. +class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): + + def __init__( + self, + # model_runner_output: ModelRunnerOutput, + sampled_token_ids: torch.Tensor, + # invalid_req_indices: list[int], + async_output_copy_stream: torch.cuda.Stream, + ): + # self._model_runner_output = model_runner_output + # self._invalid_req_indices = invalid_req_indices + + # Event on the copy stream so we can synchronize the non-blocking copy. + # self._async_copy_ready_event = torch.cuda.Event() + + # Keep a reference to the device tensor to avoid it being + # deallocated until we finish copying it to the host. + self._sampled_token_ids = sampled_token_ids + + # Initiate the copy on a separate stream, but do not synchronize it. + default_stream = torch.cuda.current_stream() + with torch.cuda.stream(async_output_copy_stream): + async_output_copy_stream.wait_stream(default_stream) + self._sampled_token_ids_cpu = self._sampled_token_ids.to( + 'cpu', non_blocking=True) + # self._async_copy_ready_event.record() + + def get_output(self) -> ModelRunnerOutput: + """Copy the device tensors to the host and return a ModelRunnerOutput. + + This function blocks until the copy is finished. + """ + # self._async_copy_ready_event.synchronize() + + # Release the device tensor once the copy has completed + del self._sampled_token_ids + + valid_sampled_token_ids = self._sampled_token_ids_cpu.tolist() + # for i in self._invalid_req_indices: + # valid_sampled_token_ids[i].clear() + + return valid_sampled_token_ids + # output = self._model_runner_output + # output.sampled_token_ids = valid_sampled_token_ids + # return output + + class OutputProcessor: @@ -47,34 +98,59 @@ def __init__(self): self.token_ids_gpu: list[torch.Tensor] = [] self.token_ids_cpu: list[torch.Tensor] = [] - def send_to_cpu_async(self, gpu_tensor: torch.Tensor): + self.token_req_ids: list[list[int]] = [] + + self.pending_outputs: deque = deque() # saving (cpu_tensor, seqs) + + + def send_to_cpu_async(self, batch: ScheduledBatchs, gpu_tensor: torch.Tensor): default_stream = torch.cuda.current_stream() with torch.cuda.stream(self.async_copy_stream): self.async_copy_stream.wait_stream(default_stream) cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) self.async_copy_event.record(self.async_copy_stream) - self.token_ids_gpu.append(gpu_tensor) + # self.token_ids_gpu.append(gpu_tensor) self.token_ids_cpu.append(cpu_tensor) + self.pending_outputs.append((cpu_tensor, batch.seqs)) + + def revc_async_output_tuple(self) -> tuple[list[int], list[Sequence]]: + if not self.pending_outputs: + return [], [] + + self.async_copy_event.synchronize() + if self.pending_outputs: + cpu_tensor, seqs = self.pending_outputs.popleft() + token_ids = cpu_tensor.tolist() + return token_ids, seqs + return [], [] def recv_async_output(self) -> list[int]: self.async_copy_event.synchronize() - for _ in self.token_ids_gpu: + for _ in self.token_ids_cpu: token_ids = self.token_ids_cpu.pop(0).tolist() return token_ids return [] + def clean_token(self): + self.token_ids_cpu: list[torch.Tensor] = [] + self.pending_outputs.clear() + def update_and_ret( self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor ) -> list[int]: - token_ids = self.recv_async_output() - self.send_to_cpu_async(sampled_token_ids) + # token_ids = self.recv_async_output() + prev_token_ids, prev_seqs = self.revc_async_output_tuple() + self.send_to_cpu_async(batch, sampled_token_ids) + # return token_ids + # return prev_token_ids, prev_seqs + deferred_reqIDs = [] returned_reqIDs = [] for req in batch.seqs: if req.id in self.deferred_request_id: returned_reqIDs.append(req.id) else: - deferred_reqIDs.append(req.id) + self.deferred_request_id.append(req.id) if returned_reqIDs: for ids in self.token_ids_cpu[:-1]: @@ -84,11 +160,9 @@ def update_and_ret( req_index = batch.unfinished_prev_req.index(req_id) batch.unfinished_prev_req.remove(req_id) req = batch.seqs[req_index] - req.append_token(ids_list[req_index]) - - self.async_copy_event.synchronize() - - return [] + req.append_token(prev_token_ids[req_index]) + return prev_token_ids, prev_seqs + # self.async_copy_event.synchronize() class ModelRunner: @@ -125,6 +199,22 @@ def __init__(self, rank: int, config: Config): torch.set_default_device("cuda") self.out_processor = OutputProcessor() self.sampler = Sampler() + + self.input_batch = InputBatch( + max_num_reqs=self.config.max_num_seqs, + max_model_len=self.config.max_model_len) + + self.input_ids = self._make_buffer(self.config.max_num_batched_tokens, + dtype=torch.int32) + self.positions = self._make_buffer(self.config.max_num_batched_tokens, + dtype=torch.int64) + self.use_async_scheduling = True + self.async_output_copy_stream = torch.cuda.Stream() if \ + self.use_async_scheduling else None + + self.out_processor = OutputProcessor() + + self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) torch.set_default_device(None) load_model(self.model, config.model, config.hf_config, config.load_dummy) @@ -138,6 +228,21 @@ def __init__(self, rank: int, config: Config): if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") + + def _make_buffer(self, + *size: Union[int, torch.SymInt], + dtype: torch.dtype, + numpy: bool = True) -> CpuGpuBuffer: + # Bfloat16 torch tensors cannot be directly cast to a numpy array, so + # if a bfloat16 buffer is needed without a corresponding numpy array, + # don't bother instantiating the numpy array. + return CpuGpuBuffer(*size, + dtype=dtype, + device=self.device, + pin_memory=True, + with_numpy=numpy) + + def exit(self): if not self.still_running: return @@ -184,8 +289,17 @@ def warmup_model(self): Sequence([0] * max_model_len, block_size=self.block_size) for _ in range(num_seqs) ] - dummy_batch = ScheduledBatchs(seqs, True, False) + + num_scheduled_tokens: dict[str, int] = {} + + for seq in seqs: + num_scheduled_tokens[seq.id] = seq.num_tokens + + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) + dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) self.forward(dummy_batch) + # self.out_processor.clean_token() torch.cuda.empty_cache() def allocate_decode_vars(self): @@ -344,6 +458,79 @@ def prepare_block_tables(self, seqs: list[Sequence]): block_tables[i, : lens[i]] = seq.block_table return block_tables + def _prepare_input_ids(self, total_num_scheduled_tokens: int, + cu_num_tokens: np.ndarray) -> None: + """Prepare the input IDs for the current batch. + + Carefully handles the `prev_sampled_token_ids` which can be cached + from the previous engine iteration, in which case those tokens on the + GPU need to be copied into the corresponding slots into input_ids.""" + + if self.input_batch.prev_sampled_token_ids is None: + # Normal scheduling case + self.input_ids.copy_to_gpu(total_num_scheduled_tokens) + return + + # Async scheduling case, where some decode requests from the previous + # iteration won't have entries in input_ids_cpu and need to be copied + # on the GPU from prev_sampled_token_ids. + prev_req_id_to_index = self.input_batch.prev_req_id_to_index + assert prev_req_id_to_index is not None + flattened_indices = [] + prev_common_req_indices = [] + indices_match = True + max_flattened_index = -1 + for req_id, cur_index in self.input_batch.req_id_to_index.items(): + if (prev_index := prev_req_id_to_index.get(req_id)) is not None: + prev_common_req_indices.append(prev_index) + # We need to compute the flattened input_ids index of the + # last token in each common request. + flattened_index = cu_num_tokens[cur_index].item() - 1 + # flattened_indices.append(flattened_index) + flattened_indices.append(prev_index) + indices_match &= (prev_index == flattened_index) + max_flattened_index = max(max_flattened_index, flattened_index) + num_commmon_tokens = len(flattened_indices) + if num_commmon_tokens < total_num_scheduled_tokens: + # If not all requests are decodes from the last iteration, + # We need to copy the input_ids_cpu to the GPU first. + self.input_ids.copy_to_gpu(total_num_scheduled_tokens) + if num_commmon_tokens == 0: + # No requests in common with the previous iteration + # So input_ids_cpu will have all the input ids. + return + if indices_match and max_flattened_index == (num_commmon_tokens - 1): + # Common-case optimization: the batch is unchanged + # and no reordering happened. + # The indices are both the same permutation of 0..N-1 so + # we can copy directly using a single slice. + self.input_ids.gpu[:num_commmon_tokens].copy_( + self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, + 0], + non_blocking=True) + return + # Upload the index tensors asynchronously + # so the scatter can be non-blocking. + input_ids_index_tensor = torch.tensor(flattened_indices, + dtype=torch.int64, + pin_memory=True).to( + self.device, + non_blocking=True) + prev_common_req_indices_tensor = torch.tensor( + prev_common_req_indices, + dtype=torch.int64, + pin_memory=True).to(self.device, non_blocking=True) + # print('self.input_batch.prev_sampled_token_ids', self.input_batch.prev_sampled_token_ids) + self.input_ids.gpu.scatter_( + dim=0, + index=input_ids_index_tensor, + src=self.input_batch.prev_sampled_token_ids[ + prev_common_req_indices_tensor]) + # prev_common_req_indices_tensor, 0]) + + self.positions.copy_to_gpu(total_num_scheduled_tokens) + + def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): input_ids = [] positions = [] @@ -484,7 +671,11 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) - + num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens + num_input_tokens = num_scheduled_tokens + input_ids = self.input_ids.gpu[:num_input_tokens] + positions = self.positions.gpu[:num_input_tokens] + return input_ids, positions return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): @@ -514,7 +705,47 @@ def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: ).cuda(non_blocking=True) return temperatures + # def _get_cumsum_and_arange( + # self, + # num_tokens: np.ndarray, + # cumsum_dtype: Optional[np.dtype] = None, + # ) -> tuple[np.ndarray, np.ndarray]: + # """Get the cumulative sum and batched arange of the given array. + # # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) + # # Equivalent to but faster than: + # # np.concatenate([np.arange(n) for n in num_tokens]) + # """ + # # Step 1. [2, 5, 3] -> [2, 7, 10] + # cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) + # total_num_tokens = cu_num_tokens[-1] + # # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] + # cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) + # # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + # arange = self.arange_np[:total_num_tokens] - cumsums_offsets + + # return cu_num_tokens, arange + + def prepare_model(self, scheduled_batchs: ScheduledBatchs): + self._update_states(scheduled_batchs) + + total_num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens + assert total_num_scheduled_tokens > 0 + + _req_ids = self.input_batch.req_ids + req_ids = [req_id for req_id in _req_ids if req_id is not None] + + # print("scheduled_batchs.num_scheduled_tokens", scheduled_batchs.num_scheduled_tokens) + # print("req_ids", req_ids) + tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] + num_scheduled_tokens = np.array(tokens, dtype=np.int32) + # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] + # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + # cu_num_tokens, arange = self._get_cumsum_and_arange( + # num_scheduled_tokens) + cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) + + self._prepare_input_ids(total_num_scheduled_tokens, cu_num_tokens) seqs = scheduled_batchs.seqs is_prefill = scheduled_batchs.is_prefill prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode @@ -543,20 +774,75 @@ def postprocess( ) -> list[int]: if self.rank == 0: sampled_tokens = self.sampler(logits, temperatures) - token_ids = sampled_tokens.tolist() - # token_ids = self.prev_sampled.update_and_ret( - # batch, - # sampled_tokens, + + # num_sampled_tokens = sampled_tokens.shape[0] + sampled_token_ids = sampled_tokens + if self.use_async_scheduling: + # assert sampled_token_ids.shape[-1] == 1 + + # Cache the sampled tokens on the GPU and avoid CPU sync. + # These will be copied into input_ids in the next step + # when preparing inputs. + # if self.input_batch.prev_sampled_token_ids is None: + # output = None + # else: + # output = self.input_batch.prev_sampled_token_ids + + self.input_batch.prev_sampled_token_ids = \ + sampled_token_ids + # print("self.input_batch.prev_sampled_token_ids in postprocess", self.input_batch.prev_sampled_token_ids) + self.input_batch.prev_req_id_to_index = { + req_id: i + for i, req_id in enumerate(self.input_batch.req_ids) + } + + + # output = AsyncGPUModelRunnerOutput( + # sampled_token_ids=sampled_token_ids, + # # invalid_req_indices=invalid_req_indices, + # async_output_copy_stream=self.async_output_copy_stream, # ) + + # .get_output() + # self.handle_output(output) + + # token_ids = sampled_tokens.tolist() + output, prev_seqs = self.out_processor.update_and_ret( + batch, + sampled_tokens, + ) + print("sampled_token_ids", sampled_token_ids) + print("prev_token_ids", output) + print('----------') else: - token_ids = [] - return token_ids + output = None + prev_seqs = None + # self.worker_response_mq.enqueue(result) + return output, prev_seqs + + def _update_states(self, batch: ScheduledBatchs): + for req_id in self.input_batch.finished_req_ids: + self.input_batch.remove_request(req_id) + + for request in batch.seqs: + self.input_batch.add_request(request) + + def finish_req(self, batch: ScheduledBatchs): + request_id = self.input_batch.req_ids + for id in request_id: + self.input_batch.finished_req_ids.add(id) + # print("finish_req:", self.input_batch.finished_req_ids) + @torch.inference_mode() - def forward(self, batch: ScheduledBatchs) -> list[int]: + def forward(self, batch: ScheduledBatchs, is_wramup = False) -> list[int]: + # self._update_states(batch) input_ids, positions, temperatures = self.prepare_model(batch) logits = self.run_model(input_ids, positions, batch.is_prefill) reset_context() + self.finish_req(batch) + # self.postprocess(batch, logits, temperatures) + # return self.async_get_output() return self.postprocess(batch, logits, temperatures) @torch.inference_mode() diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index d72117b560..d978a8dce5 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -1,10 +1,13 @@ from collections import deque -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Optional, cast, Any from atom.config import Config from atom.model_engine.block_manager import BlockManager from atom.model_engine.sequence import Sequence, SequenceStatus +import torch +import numpy as np @dataclass class ScheduledBatchs: @@ -12,6 +15,109 @@ class ScheduledBatchs: is_prefill: bool deferred_reqID: list[int] + unfinished_prev_req: list[Sequence] + + # req_id -> num_scheduled_tokens + # Number of tokens scheduled for each request. + num_scheduled_tokens: dict[str, int] + # Total number of tokens scheduled for all requests. + # Equal to sum(num_scheduled_tokens.values()) + total_num_scheduled_tokens: int + # Cached reference to the GPU tensor of previously sampled tokens + # prev_sampled_token_ids: Optional[torch.Tensor] = None + # prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None + # prev_req_id_to_index: Optional[dict[str, int]] = None + + +class InputBatch: + + def __init__( + self, + max_num_reqs: int, + max_model_len: int + ): + self.max_num_reqs = max_num_reqs + self.max_model_len = max_model_len + self.token_ids_cpu_tensor = torch.zeros( + (max_num_reqs, max_model_len), + device="cpu", + dtype=torch.int32, + pin_memory=False, + ) + self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() + self.num_tokens = np.zeros(max_num_reqs, dtype=np.int32) + self.req_id_to_index: dict[str, int] = {} + + # Cached reference to the GPU tensor of previously sampled tokens + self.prev_sampled_token_ids: Optional[torch.Tensor] = None + self.prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None + self.prev_req_id_to_index: Optional[dict[str, int]] = None + + self._req_ids: list[Optional[str]] = [] + self.req_output_token_ids: list[Optional[list[int]]] = [] + + self.finished_req_ids: set[str] = set() + + @property + def num_reqs(self) -> int: + return len(self.req_id_to_index) + + def _register_add_request(self, request: "Sequence") -> int: + """Track add-request operations for logits processors. + Not applicable to pooling models. + """ + + new_req_index = self.num_reqs + + assert new_req_index < self.max_num_reqs + return new_req_index + + + def add_request( + self, + request: "Sequence", + ) -> int: + req_index = self._register_add_request(request) + + req_id = request.id + + if req_index == len(self._req_ids): + self._req_ids.append(req_id) + # self.req_output_token_ids.append(request.output_token_ids) + else: + self._req_ids[req_index] = req_id + # self.req_output_token_ids[req_index] = request.output_token_ids + self.req_id_to_index[req_id] = req_index + + # new_req_index = request.last_token + # self.req_id_to_index[req_id] = new_req_index + + # self._req_ids.append(req_id) + # print("req_id:", req_id) + + def remove_request(self, req_id: str) -> Optional[int]: + """This method must always be followed by a call to condense(). + + Args: + req_id: request to remove + + Returns: + Removed request index, or `None` if `req_id` not recognized + """ + req_index = self.req_id_to_index.pop(req_id, None) + if req_index is None: + return None + + self._req_ids[req_index] = None + # self.req_output_token_ids[req_index] = None + + + @property + def req_ids(self) -> list[str]: + # None elements should only be present transiently + # while performing state updates to the batch. + return cast(list[str], self._req_ids) + class Scheduler: @@ -39,6 +145,9 @@ def schedule(self) -> ScheduledBatchs: scheduled_seqs = [] num_seqs = 0 num_batched_tokens = 0 + + num_scheduled_tokens: dict[str, int] = {} + if not self.running and not self.waiting: self.block_manager.reset() @@ -55,9 +164,16 @@ def schedule(self) -> ScheduledBatchs: self.waiting.popleft() self.running.append(seq) scheduled_seqs.append(seq) + # in vllm, using num_new_tokens + num_scheduled_tokens[seq.id] = seq.num_tokens + + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + if scheduled_seqs: return ScheduledBatchs( - seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[] + seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[], unfinished_prev_req=scheduled_seqs, + num_scheduled_tokens=num_scheduled_tokens, + total_num_scheduled_tokens=total_num_scheduled_tokens, ) # decode @@ -73,12 +189,19 @@ def schedule(self) -> ScheduledBatchs: num_seqs += 1 self.block_manager.may_append(seq) scheduled_seqs.append(seq) + num_scheduled_tokens[seq.id] = seq.num_tokens + + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + assert scheduled_seqs self.running.extendleft(reversed(scheduled_seqs)) return ScheduledBatchs( seqs=scheduled_seqs, is_prefill=False, deferred_reqID=self.deferred_reqID, + unfinished_prev_req=scheduled_seqs, + num_scheduled_tokens=num_scheduled_tokens, + total_num_scheduled_tokens=total_num_scheduled_tokens, ) def preempt(self, seq: Sequence): @@ -86,20 +209,51 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): + def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int], prev_seqs: list[Sequence]): # # placeholder for the each decode step - # token_ids = [self.eos_token_id] * len(seqs) - # for seq, token_id in zip(seqs, token_ids): - # seq.append_token(token_id) + token_ids = [self.eos_token_id] * len(seqs) + for seq, token_id in zip(seqs, prev_token_ids): + seq.append_token(token_id) + + if not prev_token_ids or not prev_seqs: + return + + # seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} + + # self.unfinished_req = [] + # for i, seq in enumerate(seqs): + # if seq.id in seq_id_to_index: + # token_idx = seq_id_to_index[seq.id] + # token_id = prev_token_ids[token_idx] + + # seq.append_token(token_id) + + # leave_reason = None + # if not seq.ignore_eos and token_id == self.eos_token_id: + # leave_reason = "eos" + # elif seq.num_completion_tokens == seq.max_tokens: + # leave_reason = "max_tokens" + + # if leave_reason is not None: + # seq.leave_reason = leave_reason + # seq.status = SequenceStatus.FINISHED + # self.block_manager.deallocate(seq) + # self.running.remove(seq) + # else: + # self.unfinished_req.append(i) + # else: + # print(f"New sequence {seq.id} found in current batch") + - # if not prev_token_ids: - # return # update token_ids with the actual sampled token ids self.unfinished_req = [] + # prev_token_ids = out.get_output() + # worker_response_mq.dequeue( token_ids = prev_token_ids for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): - seq.append_token(token_id) - # seq.token_ids[-2] = token_id + # if seq.id in prev_seqs: + # seq.append_token(token_id) + seq.token_ids[-2] = token_id leave_reason = None # Check if sequence ends with any stop sequence From 3be94c4f8189dbe9d02196f4d4be63e5ad690c2e Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 15 Oct 2025 17:01:48 +0800 Subject: [PATCH 064/119] draft deffer output 2 --- atom/model_engine/model_runner.py | 98 +++++-------------------------- atom/model_engine/scheduler.py | 79 +++++++++++++++---------- 2 files changed, 63 insertions(+), 114 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 3b4c242b18..d246910ded 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -28,7 +28,6 @@ logger = logging.getLogger("atom") from atom.utils.forward_context import AttentionMetadata, set_forward_context -from atom.model_engine.outputs import ModelRunnerOutput, AsyncModelRunnerOutput suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -37,55 +36,6 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } -# Wrapper for ModelRunnerOutput to support overlapped execution. -class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): - - def __init__( - self, - # model_runner_output: ModelRunnerOutput, - sampled_token_ids: torch.Tensor, - # invalid_req_indices: list[int], - async_output_copy_stream: torch.cuda.Stream, - ): - # self._model_runner_output = model_runner_output - # self._invalid_req_indices = invalid_req_indices - - # Event on the copy stream so we can synchronize the non-blocking copy. - # self._async_copy_ready_event = torch.cuda.Event() - - # Keep a reference to the device tensor to avoid it being - # deallocated until we finish copying it to the host. - self._sampled_token_ids = sampled_token_ids - - # Initiate the copy on a separate stream, but do not synchronize it. - default_stream = torch.cuda.current_stream() - with torch.cuda.stream(async_output_copy_stream): - async_output_copy_stream.wait_stream(default_stream) - self._sampled_token_ids_cpu = self._sampled_token_ids.to( - 'cpu', non_blocking=True) - # self._async_copy_ready_event.record() - - def get_output(self) -> ModelRunnerOutput: - """Copy the device tensors to the host and return a ModelRunnerOutput. - - This function blocks until the copy is finished. - """ - # self._async_copy_ready_event.synchronize() - - # Release the device tensor once the copy has completed - del self._sampled_token_ids - - valid_sampled_token_ids = self._sampled_token_ids_cpu.tolist() - # for i in self._invalid_req_indices: - # valid_sampled_token_ids[i].clear() - - return valid_sampled_token_ids - # output = self._model_runner_output - # output.sampled_token_ids = valid_sampled_token_ids - # return output - - - class OutputProcessor: def __init__(self): @@ -152,15 +102,20 @@ def update_and_ret( else: self.deferred_request_id.append(req.id) - if returned_reqIDs: - for ids in self.token_ids_cpu[:-1]: - ids_list = ids.tolist() - for req_id in returned_reqIDs: - self.deferred_request_id.remove(req_id) - req_index = batch.unfinished_prev_req.index(req_id) - batch.unfinished_prev_req.remove(req_id) - req = batch.seqs[req_index] - req.append_token(prev_token_ids[req_index]) + # prev_seqs = [] + # if returned_reqIDs: + # # for ids in self.token_ids_cpu[:-1]: + # # ids_list = ids.tolist() + # for req_id in returned_reqIDs: + # self.deferred_request_id.remove(req_id) + # for i, seq in enumerate(batch.seqs): + # if seq.id in returned_reqIDs: + # # batch.unfinished_prev_req.remove(i) + # prev_seqs.append(seq) + # # req_index = batch.unfinished_prev_req.index(req_id) + # # batch.unfinished_prev_req.remove(req_id) + # # req = batch.seqs[req_index] + # # req.append_token(prev_token_ids[req_index]) return prev_token_ids, prev_seqs # self.async_copy_event.synchronize() @@ -705,26 +660,6 @@ def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: ).cuda(non_blocking=True) return temperatures - # def _get_cumsum_and_arange( - # self, - # num_tokens: np.ndarray, - # cumsum_dtype: Optional[np.dtype] = None, - # ) -> tuple[np.ndarray, np.ndarray]: - # """Get the cumulative sum and batched arange of the given array. - # # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) - # # Equivalent to but faster than: - # # np.concatenate([np.arange(n) for n in num_tokens]) - # """ - # # Step 1. [2, 5, 3] -> [2, 7, 10] - # cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) - # total_num_tokens = cu_num_tokens[-1] - # # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] - # cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) - # # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - # arange = self.arange_np[:total_num_tokens] - cumsums_offsets - - # return cu_num_tokens, arange - def prepare_model(self, scheduled_batchs: ScheduledBatchs): self._update_states(scheduled_batchs) @@ -735,8 +670,6 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): _req_ids = self.input_batch.req_ids req_ids = [req_id for req_id in _req_ids if req_id is not None] - # print("scheduled_batchs.num_scheduled_tokens", scheduled_batchs.num_scheduled_tokens) - # print("req_ids", req_ids) tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] num_scheduled_tokens = np.array(tokens, dtype=np.int32) # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] @@ -811,9 +744,6 @@ def postprocess( batch, sampled_tokens, ) - print("sampled_token_ids", sampled_token_ids) - print("prev_token_ids", output) - print('----------') else: output = None prev_seqs = None diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index d978a8dce5..fbea489c5d 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -211,41 +211,60 @@ def preempt(self, seq: Sequence): def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int], prev_seqs: list[Sequence]): # # placeholder for the each decode step - token_ids = [self.eos_token_id] * len(seqs) - for seq, token_id in zip(seqs, prev_token_ids): - seq.append_token(token_id) + # token_ids = [self.eos_token_id] * len(seqs) + # for seq, token_id in zip(seqs, prev_token_ids): + # seq.append_token(token_id) + + # print('--------------------') + # for i, seq in enumerate(prev_seqs): + # print("prev_seqs", seq.id) + # print(seq.token_ids) + # for i, seq in enumerate(seqs): + # print("seqs", seq.id) + # print(seq.token_ids) + # print("prev_token_ids", prev_token_ids) if not prev_token_ids or not prev_seqs: return - # seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} - # self.unfinished_req = [] - # for i, seq in enumerate(seqs): - # if seq.id in seq_id_to_index: - # token_idx = seq_id_to_index[seq.id] - # token_id = prev_token_ids[token_idx] - - # seq.append_token(token_id) - - # leave_reason = None - # if not seq.ignore_eos and token_id == self.eos_token_id: - # leave_reason = "eos" - # elif seq.num_completion_tokens == seq.max_tokens: - # leave_reason = "max_tokens" - - # if leave_reason is not None: - # seq.leave_reason = leave_reason - # seq.status = SequenceStatus.FINISHED - # self.block_manager.deallocate(seq) - # self.running.remove(seq) - # else: - # self.unfinished_req.append(i) - # else: - # print(f"New sequence {seq.id} found in current batch") - - - # update token_ids with the actual sampled token ids + # processed_seqs = set() + + # for i, prev_seq in enumerate(prev_seqs): + # token_id = prev_token_ids[i] + + # current_seq = None + # for seq in seqs: + # if seq.id == prev_seq.id: + # current_seq = seq + + # if current_seq and current_seq.id not in processed_seqs: + # processed_seqs.add(current_seq.id) + + # # Check if this token has been used before (to avoid outputting a word repeatedly) + # if (len(current_seq.token_ids) > prev_seq.num_tokens and + # current_seq.token_ids[-1] == token_id): + # print(f"Token {token_id} already applied to sequence {current_seq.id}") + # continue + + # current_seq.append_token(token_id) + # print(f"Applied token {token_id} to sequence {current_seq.id}") + + # leave_reason = None + # if not current_seq.ignore_eos and token_id == self.eos_token_id: + # leave_reason = "eos" + # elif current_seq.num_completion_tokens == current_seq.max_tokens: + # leave_reason = "max_tokens" + + # if leave_reason is not None: + # current_seq.leave_reason = leave_reason + # current_seq.status = SequenceStatus.FINISHED + # self.block_manager.deallocate(current_seq) + # self.running.remove(current_seq) + # else: + # self.unfinished_req.append(i) + + seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} self.unfinished_req = [] # prev_token_ids = out.get_output() # worker_response_mq.dequeue( From 4d4a6d6ffa3791c684b26547c991dec40da65990 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 11:01:55 +0800 Subject: [PATCH 065/119] fix acc issue --- atom/model_engine/engine_core.py | 4 +- atom/model_engine/model_runner.py | 71 +++++++++++++++++-------------- atom/model_engine/scheduler.py | 64 +++++----------------------- 3 files changed, 50 insertions(+), 89 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index e3264b6080..6300e32225 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -138,9 +138,9 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() - out, prev_seqs = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) + out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) # scheduled_batchs.prev_seqs = prev_seqs - self.scheduler.postprocess(scheduled_batchs.seqs, out, prev_seqs) + self.scheduler.postprocess(scheduled_batchs.seqs, out) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] ) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index d246910ded..33a4ccc7f1 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -88,13 +88,13 @@ def clean_token(self): def update_and_ret( self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor ) -> list[int]: - # token_ids = self.recv_async_output() - prev_token_ids, prev_seqs = self.revc_async_output_tuple() + token_ids = self.recv_async_output() + # prev_token_ids, prev_seqs = self.revc_async_output_tuple() self.send_to_cpu_async(batch, sampled_token_ids) # return token_ids # return prev_token_ids, prev_seqs - deferred_reqIDs = [] + # deferred_reqIDs = [] returned_reqIDs = [] for req in batch.seqs: if req.id in self.deferred_request_id: @@ -102,23 +102,19 @@ def update_and_ret( else: self.deferred_request_id.append(req.id) - # prev_seqs = [] - # if returned_reqIDs: - # # for ids in self.token_ids_cpu[:-1]: - # # ids_list = ids.tolist() - # for req_id in returned_reqIDs: - # self.deferred_request_id.remove(req_id) - # for i, seq in enumerate(batch.seqs): - # if seq.id in returned_reqIDs: - # # batch.unfinished_prev_req.remove(i) - # prev_seqs.append(seq) - # # req_index = batch.unfinished_prev_req.index(req_id) - # # batch.unfinished_prev_req.remove(req_id) - # # req = batch.seqs[req_index] - # # req.append_token(prev_token_ids[req_index]) - return prev_token_ids, prev_seqs + if returned_reqIDs: + # for ids in self.token_ids_cpu[:-1]: + # ids_list = ids.tolist() + for req_id in returned_reqIDs: + self.deferred_request_id.remove(req_id) + # req_index = batch.unfinished_prev_req.index(req_id) + # batch.unfinished_prev_req.remove(req_id) + # req = batch.seqs[req_index] + # req.append_token(ids_list[req_index]) + # self.async_copy_event.synchronize() + return token_ids class ModelRunner: @@ -254,7 +250,7 @@ def warmup_model(self): # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) self.forward(dummy_batch) - # self.out_processor.clean_token() + self.out_processor.clean_token() torch.cuda.empty_cache() def allocate_decode_vars(self): @@ -483,7 +479,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, prev_common_req_indices_tensor]) # prev_common_req_indices_tensor, 0]) - self.positions.copy_to_gpu(total_num_scheduled_tokens) + # self.positions.gpu[:num_commmon_tokens].copy_() def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): @@ -545,7 +541,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): ) assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" input_ids = [] - positions = [] + # positions = [] slot_mapping = [] context_lens = [] dropout_p = 0.0 @@ -553,7 +549,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self.total_blocks = 0 context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] - positions = np.array(context_lens, dtype=np.int64) + # positions = np.array(context_lens, dtype=np.int64) context_lens = np.array(context_lens, dtype=np.int32) input_ids = [seq.last_token for seq in scheduled_batchs.seqs] input_ids = np.array(input_ids, dtype=np.int64) @@ -579,11 +575,20 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self._update_paged_kv_tensors(seq.block_table, current_seq_len) block_tables = self.prepare_block_tables(scheduled_batchs.seqs) + # cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + # cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) + num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens + num_input_tokens = num_scheduled_tokens + prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] + prev_positions_ = self.input_batch.prev_position_ids var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping - var["input_ids"].np[:bs] = input_ids - var["positions"].np[:bs] = positions + var["input_ids"].np[:bs] = prev_input_ids_[:bs].cpu() + var["positions"].np[:bs] = prev_positions_[:bs] + # var["input_ids"].np[:bs] = input_ids + # var["positions"].np[:bs] = positions + var["context_lens"].np[:bs] = context_lens var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables @@ -626,11 +631,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) - num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens - num_input_tokens = num_scheduled_tokens - input_ids = self.input_ids.gpu[:num_input_tokens] - positions = self.positions.gpu[:num_input_tokens] - return input_ids, positions + return prev_input_ids_, prev_positions_ return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): @@ -728,7 +729,12 @@ def postprocess( req_id: i for i, req_id in enumerate(self.input_batch.req_ids) } - + # plus 1 to fit next iter seq len + context_lens = [seq.num_tokens + 1 for seq in batch.seqs] + # positions = np.array(context_lens, dtype=np.int64) + + self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) + # output = AsyncGPUModelRunnerOutput( # sampled_token_ids=sampled_token_ids, @@ -740,15 +746,14 @@ def postprocess( # self.handle_output(output) # token_ids = sampled_tokens.tolist() - output, prev_seqs = self.out_processor.update_and_ret( + output = self.out_processor.update_and_ret( batch, sampled_tokens, ) else: output = None - prev_seqs = None # self.worker_response_mq.enqueue(result) - return output, prev_seqs + return output def _update_states(self, batch: ScheduledBatchs): for req_id in self.input_batch.finished_req_ids: diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index fbea489c5d..9bd2ff61a4 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -52,6 +52,7 @@ def __init__( self.prev_sampled_token_ids: Optional[torch.Tensor] = None self.prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None self.prev_req_id_to_index: Optional[dict[str, int]] = None + self.prev_position_ids: Optional[torch.Tensor] = None self._req_ids: list[Optional[str]] = [] self.req_output_token_ids: list[Optional[list[int]]] = [] @@ -209,62 +210,17 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int], prev_seqs: list[Sequence]): + def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): + # # placeholder for the each decode step - # token_ids = [self.eos_token_id] * len(seqs) - # for seq, token_id in zip(seqs, prev_token_ids): - # seq.append_token(token_id) - - # print('--------------------') - # for i, seq in enumerate(prev_seqs): - # print("prev_seqs", seq.id) - # print(seq.token_ids) - # for i, seq in enumerate(seqs): - # print("seqs", seq.id) - # print(seq.token_ids) - # print("prev_token_ids", prev_token_ids) - - if not prev_token_ids or not prev_seqs: + token_ids = [self.eos_token_id] * len(seqs) + for seq, token_id in zip(seqs, token_ids): + seq.append_token(token_id) + + if not prev_token_ids: return - - # self.unfinished_req = [] - # processed_seqs = set() - - # for i, prev_seq in enumerate(prev_seqs): - # token_id = prev_token_ids[i] - - # current_seq = None - # for seq in seqs: - # if seq.id == prev_seq.id: - # current_seq = seq - - # if current_seq and current_seq.id not in processed_seqs: - # processed_seqs.add(current_seq.id) - - # # Check if this token has been used before (to avoid outputting a word repeatedly) - # if (len(current_seq.token_ids) > prev_seq.num_tokens and - # current_seq.token_ids[-1] == token_id): - # print(f"Token {token_id} already applied to sequence {current_seq.id}") - # continue - - # current_seq.append_token(token_id) - # print(f"Applied token {token_id} to sequence {current_seq.id}") - - # leave_reason = None - # if not current_seq.ignore_eos and token_id == self.eos_token_id: - # leave_reason = "eos" - # elif current_seq.num_completion_tokens == current_seq.max_tokens: - # leave_reason = "max_tokens" - - # if leave_reason is not None: - # current_seq.leave_reason = leave_reason - # current_seq.status = SequenceStatus.FINISHED - # self.block_manager.deallocate(current_seq) - # self.running.remove(current_seq) - # else: - # self.unfinished_req.append(i) - - seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} + + # update token_ids with the actual sampled token ids self.unfinished_req = [] # prev_token_ids = out.get_output() # worker_response_mq.dequeue( From 29d2c4938f689165ae472a55a85fddec51b82175 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 15:33:34 +0800 Subject: [PATCH 066/119] commit on opt deffer --- atom/model_engine/model_runner.py | 58 ++++++++++--------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 33a4ccc7f1..e74030d4e6 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -91,28 +91,6 @@ def update_and_ret( token_ids = self.recv_async_output() # prev_token_ids, prev_seqs = self.revc_async_output_tuple() self.send_to_cpu_async(batch, sampled_token_ids) - # return token_ids - # return prev_token_ids, prev_seqs - - # deferred_reqIDs = [] - returned_reqIDs = [] - for req in batch.seqs: - if req.id in self.deferred_request_id: - returned_reqIDs.append(req.id) - else: - self.deferred_request_id.append(req.id) - - if returned_reqIDs: - # for ids in self.token_ids_cpu[:-1]: - # ids_list = ids.tolist() - for req_id in returned_reqIDs: - self.deferred_request_id.remove(req_id) - # req_index = batch.unfinished_prev_req.index(req_id) - # batch.unfinished_prev_req.remove(req_id) - # req = batch.seqs[req_index] - # req.append_token(ids_list[req_index]) - - # self.async_copy_event.synchronize() return token_ids @@ -159,6 +137,10 @@ def __init__(self, rank: int, config: Config): dtype=torch.int32) self.positions = self._make_buffer(self.config.max_num_batched_tokens, dtype=torch.int64) + + self.input_ids_index_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) + self.prev_common_req_indices_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) + self.use_async_scheduling = True self.async_output_copy_stream = torch.cuda.Stream() if \ self.use_async_scheduling else None @@ -462,24 +444,22 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, return # Upload the index tensors asynchronously # so the scatter can be non-blocking. - input_ids_index_tensor = torch.tensor(flattened_indices, - dtype=torch.int64, - pin_memory=True).to( - self.device, - non_blocking=True) - prev_common_req_indices_tensor = torch.tensor( - prev_common_req_indices, - dtype=torch.int64, - pin_memory=True).to(self.device, non_blocking=True) - # print('self.input_batch.prev_sampled_token_ids', self.input_batch.prev_sampled_token_ids) + + self.input_ids_index_tensor.cpu[:num_commmon_tokens] = torch.tensor( + flattened_indices, dtype=torch.int64, device="cpu") + self.prev_common_req_indices_tensor.cpu[:num_commmon_tokens] = torch.tensor( + prev_common_req_indices, dtype=torch.int64, device="cpu") + + self.input_ids_index_tensor.copy_to_gpu(num_commmon_tokens) + self.prev_common_req_indices_tensor.copy_to_gpu(num_commmon_tokens) + self.input_ids.gpu.scatter_( dim=0, - index=input_ids_index_tensor, + index=self.input_ids_index_tensor.gpu[:num_commmon_tokens], src=self.input_batch.prev_sampled_token_ids[ - prev_common_req_indices_tensor]) + self.prev_common_req_indices_tensor.gpu[:num_commmon_tokens]]) # prev_common_req_indices_tensor, 0]) - # self.positions.gpu[:num_commmon_tokens].copy_() def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): @@ -549,7 +529,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self.total_blocks = 0 context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] - # positions = np.array(context_lens, dtype=np.int64) + positions = np.array(context_lens, dtype=np.int64) context_lens = np.array(context_lens, dtype=np.int32) input_ids = [seq.last_token for seq in scheduled_batchs.seqs] input_ids = np.array(input_ids, dtype=np.int64) @@ -585,7 +565,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping var["input_ids"].np[:bs] = prev_input_ids_[:bs].cpu() - var["positions"].np[:bs] = prev_positions_[:bs] + var["positions"].np[:bs] = positions # var["input_ids"].np[:bs] = input_ids # var["positions"].np[:bs] = positions @@ -631,7 +611,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) - return prev_input_ids_, prev_positions_ + return prev_input_ids_, positions return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): @@ -724,7 +704,6 @@ def postprocess( self.input_batch.prev_sampled_token_ids = \ sampled_token_ids - # print("self.input_batch.prev_sampled_token_ids in postprocess", self.input_batch.prev_sampled_token_ids) self.input_batch.prev_req_id_to_index = { req_id: i for i, req_id in enumerate(self.input_batch.req_ids) @@ -766,7 +745,6 @@ def finish_req(self, batch: ScheduledBatchs): request_id = self.input_batch.req_ids for id in request_id: self.input_batch.finished_req_ids.add(id) - # print("finish_req:", self.input_batch.finished_req_ids) @torch.inference_mode() From b2436c258a5fc7658fd040e78f79a38386b5b6d6 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 17:16:39 +0800 Subject: [PATCH 067/119] remove d2h in prepare decode --- atom/model_engine/model_runner.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index e74030d4e6..d1a894812d 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -560,11 +560,12 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens num_input_tokens = num_scheduled_tokens prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] - prev_positions_ = self.input_batch.prev_position_ids + # prev_positions_ = self.input_batch.prev_position_ids var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping - var["input_ids"].np[:bs] = prev_input_ids_[:bs].cpu() + # var["input_ids"].np[:bs] = prev_input_ids_[:bs] + var["input_ids"].gpu[:bs] = prev_input_ids_[:bs] var["positions"].np[:bs] = positions # var["input_ids"].np[:bs] = input_ids # var["positions"].np[:bs] = positions @@ -587,7 +588,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): for el in [ "slot_mapping", - "input_ids", + # "input_ids", "positions", "context_lens", "block_tables", @@ -697,11 +698,6 @@ def postprocess( # Cache the sampled tokens on the GPU and avoid CPU sync. # These will be copied into input_ids in the next step # when preparing inputs. - # if self.input_batch.prev_sampled_token_ids is None: - # output = None - # else: - # output = self.input_batch.prev_sampled_token_ids - self.input_batch.prev_sampled_token_ids = \ sampled_token_ids self.input_batch.prev_req_id_to_index = { @@ -709,20 +705,10 @@ def postprocess( for i, req_id in enumerate(self.input_batch.req_ids) } # plus 1 to fit next iter seq len - context_lens = [seq.num_tokens + 1 for seq in batch.seqs] + # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] # positions = np.array(context_lens, dtype=np.int64) - self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) - - - # output = AsyncGPUModelRunnerOutput( - # sampled_token_ids=sampled_token_ids, - # # invalid_req_indices=invalid_req_indices, - # async_output_copy_stream=self.async_output_copy_stream, - # ) - - # .get_output() - # self.handle_output(output) + # self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) # token_ids = sampled_tokens.tolist() output = self.out_processor.update_and_ret( From b1da4f6ecdbc8e7e83332cd706f47097b4656852 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 18:05:27 +0800 Subject: [PATCH 068/119] use np --- atom/model_engine/model_runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index d1a894812d..5ecb6de863 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -445,10 +445,8 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # Upload the index tensors asynchronously # so the scatter can be non-blocking. - self.input_ids_index_tensor.cpu[:num_commmon_tokens] = torch.tensor( - flattened_indices, dtype=torch.int64, device="cpu") - self.prev_common_req_indices_tensor.cpu[:num_commmon_tokens] = torch.tensor( - prev_common_req_indices, dtype=torch.int64, device="cpu") + self.input_ids_index_tensor.np[:num_commmon_tokens] = np.array(flattened_indices, dtype=np.int64) + self.prev_common_req_indices_tensor.np[:num_commmon_tokens] = np.array(prev_common_req_indices, dtype=np.int64) self.input_ids_index_tensor.copy_to_gpu(num_commmon_tokens) self.prev_common_req_indices_tensor.copy_to_gpu(num_commmon_tokens) From 557f949a7acc29e6a5864e2cb7d9a64888b841cd Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 17 Oct 2025 16:50:06 +0800 Subject: [PATCH 069/119] reduce prepare_input_ids overhead in prefill --- atom/model_engine/model_runner.py | 41 +++++++++++++++++++++++++------ atom/model_engine/scheduler.py | 6 +++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 5ecb6de863..0bdb272787 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -138,6 +138,12 @@ def __init__(self, rank: int, config: Config): self.positions = self._make_buffer(self.config.max_num_batched_tokens, dtype=torch.int64) + self.arange_np = np.arange(max(self.config.max_num_seqs + 1, + self.config.max_model_len, + self.config.max_num_batched_tokens), + dtype=np.int64) + + self.input_ids_index_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) self.prev_common_req_indices_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) @@ -226,7 +232,7 @@ def warmup_model(self): num_scheduled_tokens: dict[str, int] = {} for seq in seqs: - num_scheduled_tokens[seq.id] = seq.num_tokens + num_scheduled_tokens[seq.id] = 1 total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) @@ -438,8 +444,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # The indices are both the same permutation of 0..N-1 so # we can copy directly using a single slice. self.input_ids.gpu[:num_commmon_tokens].copy_( - self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, - 0], + self.input_batch.prev_sampled_token_ids[:num_commmon_tokens], non_blocking=True) return # Upload the index tensors asynchronously @@ -640,6 +645,26 @@ def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: ).cuda(non_blocking=True) return temperatures + def _get_cumsum_and_arange( + self, + num_tokens: np.ndarray, + cumsum_dtype: Optional[np.dtype] = None, + ) -> tuple[np.ndarray, np.ndarray]: + """Get the cumulative sum and batched arange of the given array. + # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) + # Equivalent to but faster than: + # np.concatenate([np.arange(n) for n in num_tokens]) + """ + # Step 1. [2, 5, 3] -> [2, 7, 10] + cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) + total_num_tokens = cu_num_tokens[-1] + # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] + cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) + # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + arange = self.arange_np[:total_num_tokens] - cumsums_offsets + + return cu_num_tokens, arange + def prepare_model(self, scheduled_batchs: ScheduledBatchs): self._update_states(scheduled_batchs) @@ -654,9 +679,9 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): num_scheduled_tokens = np.array(tokens, dtype=np.int32) # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - # cu_num_tokens, arange = self._get_cumsum_and_arange( - # num_scheduled_tokens) - cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) + cu_num_tokens, arange = self._get_cumsum_and_arange( + num_scheduled_tokens) + # cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) self._prepare_input_ids(total_num_scheduled_tokens, cu_num_tokens) seqs = scheduled_batchs.seqs @@ -669,7 +694,7 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): def run_model( self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool ): - torch.cuda.empty_cache() + # torch.cuda.empty_cache() ctx = get_context() bs = ctx.batch_size if is_prefill or self.enforce_eager or bs > self.graph_bs[-1]: @@ -696,11 +721,13 @@ def postprocess( # Cache the sampled tokens on the GPU and avoid CPU sync. # These will be copied into input_ids in the next step # when preparing inputs. + invalid_req_indices_set = [None] self.input_batch.prev_sampled_token_ids = \ sampled_token_ids self.input_batch.prev_req_id_to_index = { req_id: i for i, req_id in enumerate(self.input_batch.req_ids) + if req_id not in invalid_req_indices_set } # plus 1 to fit next iter seq len # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 9bd2ff61a4..aff2a460c9 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -166,7 +166,8 @@ def schedule(self) -> ScheduledBatchs: self.running.append(seq) scheduled_seqs.append(seq) # in vllm, using num_new_tokens - num_scheduled_tokens[seq.id] = seq.num_tokens + num_new_tokens = 1 + num_scheduled_tokens[seq.id] = num_new_tokens total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) @@ -190,7 +191,8 @@ def schedule(self) -> ScheduledBatchs: num_seqs += 1 self.block_manager.may_append(seq) scheduled_seqs.append(seq) - num_scheduled_tokens[seq.id] = seq.num_tokens + num_new_tokens = 1 + num_scheduled_tokens[seq.id] = num_new_tokens total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) From 721e7a3f7214bd9b4bf0e10c38bbe6447459682f Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 17 Oct 2025 23:43:30 +0800 Subject: [PATCH 070/119] rename inputbatch to prevbatch --- atom/model_engine/model_runner.py | 4 ++-- atom/model_engine/scheduler.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 0bdb272787..4eb62caa60 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -15,7 +15,7 @@ from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture from atom.config import Config, set_current_atom_config -from atom.model_engine.scheduler import ScheduledBatchs, InputBatch +from atom.model_engine.scheduler import ScheduledBatchs, PrevScheduledBatchs from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler @@ -129,7 +129,7 @@ def __init__(self, rank: int, config: Config): self.out_processor = OutputProcessor() self.sampler = Sampler() - self.input_batch = InputBatch( + self.input_batch = PrevScheduledBatchs( max_num_reqs=self.config.max_num_seqs, max_model_len=self.config.max_model_len) diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index aff2a460c9..39c22d34ef 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -29,7 +29,7 @@ class ScheduledBatchs: # prev_req_id_to_index: Optional[dict[str, int]] = None -class InputBatch: +class PrevScheduledBatchs: def __init__( self, From d996e19f360441ff8039c1272e4bb13506094ae1 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 17 Oct 2025 23:51:40 +0800 Subject: [PATCH 071/119] fix profile by push cover --- atom/model_engine/engine_core.py | 2 +- atom/model_engine/model_runner.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 6300e32225..60a258a8bc 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -220,5 +220,5 @@ def start_profiler(self): def stop_profiler(self): if self.profile_enbaled: print("Stopping profiler...") - self.runner_mgr.call_func("stop_profiler") + self.runner_mgr.call_func("stop_profiler", wait_out=True) print("Profiler stopped.") diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 4eb62caa60..9d1957de48 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -213,6 +213,7 @@ def stop_profiler(self): if self.profiler is not None: self.profiler.__exit__(None, None, None) self.profiler = None + return True def warmup_model(self): torch.cuda.empty_cache() From 374c30e1adca5e307ce5eb46c48bb69a38379bc5 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sat, 18 Oct 2025 00:27:27 +0800 Subject: [PATCH 072/119] rename and remove some useless code --- atom/model_engine/engine_core.py | 1 - atom/model_engine/model_runner.py | 44 +++++++++++-------------------- atom/model_engine/scheduler.py | 10 +------ 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 60a258a8bc..d3e1c21885 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -139,7 +139,6 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) - # scheduled_batchs.prev_seqs = prev_seqs self.scheduler.postprocess(scheduled_batchs.seqs, out) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 9d1957de48..5861ce065d 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -59,7 +59,6 @@ def send_to_cpu_async(self, batch: ScheduledBatchs, gpu_tensor: torch.Tensor): self.async_copy_stream.wait_stream(default_stream) cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) self.async_copy_event.record(self.async_copy_stream) - # self.token_ids_gpu.append(gpu_tensor) self.token_ids_cpu.append(cpu_tensor) self.pending_outputs.append((cpu_tensor, batch.seqs)) @@ -89,7 +88,6 @@ def update_and_ret( self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor ) -> list[int]: token_ids = self.recv_async_output() - # prev_token_ids, prev_seqs = self.revc_async_output_tuple() self.send_to_cpu_async(batch, sampled_token_ids) return token_ids @@ -129,7 +127,7 @@ def __init__(self, rank: int, config: Config): self.out_processor = OutputProcessor() self.sampler = Sampler() - self.input_batch = PrevScheduledBatchs( + self.prev_input_batch = PrevScheduledBatchs( max_num_reqs=self.config.max_num_seqs, max_model_len=self.config.max_model_len) @@ -236,7 +234,6 @@ def warmup_model(self): num_scheduled_tokens[seq.id] = 1 total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) - # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) self.forward(dummy_batch) self.out_processor.clean_token() @@ -406,7 +403,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, from the previous engine iteration, in which case those tokens on the GPU need to be copied into the corresponding slots into input_ids.""" - if self.input_batch.prev_sampled_token_ids is None: + if self.prev_input_batch.prev_sampled_token_ids is None: # Normal scheduling case self.input_ids.copy_to_gpu(total_num_scheduled_tokens) return @@ -414,13 +411,13 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # Async scheduling case, where some decode requests from the previous # iteration won't have entries in input_ids_cpu and need to be copied # on the GPU from prev_sampled_token_ids. - prev_req_id_to_index = self.input_batch.prev_req_id_to_index + prev_req_id_to_index = self.prev_input_batch.prev_req_id_to_index assert prev_req_id_to_index is not None flattened_indices = [] prev_common_req_indices = [] indices_match = True max_flattened_index = -1 - for req_id, cur_index in self.input_batch.req_id_to_index.items(): + for req_id, cur_index in self.prev_input_batch.req_id_to_index.items(): if (prev_index := prev_req_id_to_index.get(req_id)) is not None: prev_common_req_indices.append(prev_index) # We need to compute the flattened input_ids index of the @@ -445,7 +442,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # The indices are both the same permutation of 0..N-1 so # we can copy directly using a single slice. self.input_ids.gpu[:num_commmon_tokens].copy_( - self.input_batch.prev_sampled_token_ids[:num_commmon_tokens], + self.prev_input_batch.prev_sampled_token_ids[:num_commmon_tokens], non_blocking=True) return # Upload the index tensors asynchronously @@ -460,7 +457,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, self.input_ids.gpu.scatter_( dim=0, index=self.input_ids_index_tensor.gpu[:num_commmon_tokens], - src=self.input_batch.prev_sampled_token_ids[ + src=self.prev_input_batch.prev_sampled_token_ids[ self.prev_common_req_indices_tensor.gpu[:num_commmon_tokens]]) # prev_common_req_indices_tensor, 0]) @@ -525,7 +522,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): ) assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" input_ids = [] - # positions = [] + positions = [] slot_mapping = [] context_lens = [] dropout_p = 0.0 @@ -564,15 +561,11 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens num_input_tokens = num_scheduled_tokens prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] - # prev_positions_ = self.input_batch.prev_position_ids var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping - # var["input_ids"].np[:bs] = prev_input_ids_[:bs] var["input_ids"].gpu[:bs] = prev_input_ids_[:bs] var["positions"].np[:bs] = positions - # var["input_ids"].np[:bs] = input_ids - # var["positions"].np[:bs] = positions var["context_lens"].np[:bs] = context_lens var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables @@ -673,7 +666,7 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): total_num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens assert total_num_scheduled_tokens > 0 - _req_ids = self.input_batch.req_ids + _req_ids = self.prev_input_batch.req_ids req_ids = [req_id for req_id in _req_ids if req_id is not None] tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] @@ -723,19 +716,17 @@ def postprocess( # These will be copied into input_ids in the next step # when preparing inputs. invalid_req_indices_set = [None] - self.input_batch.prev_sampled_token_ids = \ + self.prev_input_batch.prev_sampled_token_ids = \ sampled_token_ids - self.input_batch.prev_req_id_to_index = { + self.prev_input_batch.prev_req_id_to_index = { req_id: i - for i, req_id in enumerate(self.input_batch.req_ids) + for i, req_id in enumerate(self.prev_input_batch.req_ids) if req_id not in invalid_req_indices_set } # plus 1 to fit next iter seq len # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] # positions = np.array(context_lens, dtype=np.int64) - # self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) - # token_ids = sampled_tokens.tolist() output = self.out_processor.update_and_ret( batch, @@ -743,20 +734,19 @@ def postprocess( ) else: output = None - # self.worker_response_mq.enqueue(result) return output def _update_states(self, batch: ScheduledBatchs): - for req_id in self.input_batch.finished_req_ids: - self.input_batch.remove_request(req_id) + for req_id in self.prev_input_batch.finished_req_ids: + self.prev_input_batch.remove_request(req_id) for request in batch.seqs: - self.input_batch.add_request(request) + self.prev_input_batch.add_request(request) def finish_req(self, batch: ScheduledBatchs): - request_id = self.input_batch.req_ids + request_id = self.prev_input_batch.req_ids for id in request_id: - self.input_batch.finished_req_ids.add(id) + self.prev_input_batch.finished_req_ids.add(id) @torch.inference_mode() @@ -766,8 +756,6 @@ def forward(self, batch: ScheduledBatchs, is_wramup = False) -> list[int]: logits = self.run_model(input_ids, positions, batch.is_prefill) reset_context() self.finish_req(batch) - # self.postprocess(batch, logits, temperatures) - # return self.async_get_output() return self.postprocess(batch, logits, temperatures) @torch.inference_mode() diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 39c22d34ef..2afa33b783 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -23,10 +23,6 @@ class ScheduledBatchs: # Total number of tokens scheduled for all requests. # Equal to sum(num_scheduled_tokens.values()) total_num_scheduled_tokens: int - # Cached reference to the GPU tensor of previously sampled tokens - # prev_sampled_token_ids: Optional[torch.Tensor] = None - # prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None - # prev_req_id_to_index: Optional[dict[str, int]] = None class PrevScheduledBatchs: @@ -94,7 +90,6 @@ def add_request( # self.req_id_to_index[req_id] = new_req_index # self._req_ids.append(req_id) - # print("req_id:", req_id) def remove_request(self, req_id: str) -> Optional[int]: """This method must always be followed by a call to condense(). @@ -224,12 +219,9 @@ def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): # update token_ids with the actual sampled token ids self.unfinished_req = [] - # prev_token_ids = out.get_output() - # worker_response_mq.dequeue( token_ids = prev_token_ids for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): - # if seq.id in prev_seqs: - # seq.append_token(token_id) + # seq.append_token(token_id) seq.token_ids[-2] = token_id leave_reason = None From 2b6d8a7416acb3ab1b4173979b41bd4ebd1ceeb7 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 19 Oct 2025 15:47:04 +0000 Subject: [PATCH 073/119] finalize deferred_out --- atom/bench/example.py | 29 +- atom/model_engine/engine_core.py | 11 +- atom/model_engine/model_runner.py | 716 ++++++++++++++---------------- atom/model_engine/scheduler.py | 274 +++++------- atom/model_engine/sequence.py | 8 + atom/utils/context.py | 1 + 6 files changed, 474 insertions(+), 565 deletions(-) diff --git a/atom/bench/example.py b/atom/bench/example.py index 238d7aa7a2..aeeff6f577 100644 --- a/atom/bench/example.py +++ b/atom/bench/example.py @@ -40,34 +40,46 @@ parser.add_argument( - "--cudagraph-capture-sizes", type=str, default="[1,2,4,8,16]", help="Sizes to capture cudagraph." + "--cudagraph-capture-sizes", + type=str, + default="[1,2,4,8,16]", + help="Sizes to capture cudagraph.", ) parser.add_argument("--level", type=int, default=3, help="The level of compilation") parser.add_argument( - "--torch-profiler-dir", type=str, default=None, help="Directory to save torch profiler traces" + "--torch-profiler-dir", + type=str, + default=None, + help="Directory to save torch profiler traces", ) parser.add_argument( - "--enable-expert-parallel", action="store_true", help="Enable expert parallel(EP MoE)." + "--enable-expert-parallel", + action="store_true", + help="Enable expert parallel(EP MoE).", ) +parser.add_argument("--block-size", type=int, default=16, help="KV cache block size.") parser.add_argument( - "--block-size", type=int, default=16, help="KV cache block size." + "--temperature", type=float, default=0.6, help="temperature for sampling" ) parser.add_argument( "--load_dummy", action="store_true", help="Skip loading model weights." ) + def parse_size_list(size_str: str) -> List[int]: import ast + try: return ast.literal_eval(size_str) except (ValueError, SyntaxError) as e: raise ValueError(f"Error list size: {size_str}") from e + def main(): args = parser.parse_args() model_name_or_path = args.model @@ -84,12 +96,12 @@ def main(): enable_expert_parallel=args.enable_expert_parallel, torch_profiler_dir=args.torch_profiler_dir, compilation_config=CompilationConfig( - level = args.level, - cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes) - ) + level=args.level, + cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes), + ), ) - sampling_params = SamplingParams(temperature=0.6, max_tokens=256) + sampling_params = SamplingParams(temperature=args.temperature, max_tokens=256) prompts = [ "introduce yourself", "list all prime numbers within 100", @@ -122,6 +134,7 @@ def main(): # _ = llm.generate(warmup_prompts, sampling_params) # generate (with profiling) + # outputs_ = llm.generate(["Benchmark: "], SamplingParams()) outputs = llm.generate(prompts, sampling_params) for prompt, output in zip(prompts, outputs): diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index d3e1c21885..704e54c9ac 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -137,12 +137,11 @@ def busy_loop(self): break def _process_engine_step(self): - scheduled_batchs = self.scheduler.schedule() - out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) - self.scheduler.postprocess(scheduled_batchs.seqs, out) - self.output_queue.put_nowait( - [seq for seq in scheduled_batchs.seqs if seq.is_finished] - ) + scheduled_batch = self.scheduler.schedule() + out = self.runner_mgr.call_func("forward", scheduled_batch, wait_out=True) + seqs = scheduled_batch.seqs.values() + self.scheduler.postprocess(seqs, out) + self.output_queue.put_nowait([seq for seq in seqs if seq.is_finished]) def pull_and_process_input_queue(self): recv_reqs = [] diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 5861ce065d..41c30a7c41 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -1,21 +1,17 @@ import logging import os -import signal import time -import weakref -from typing import Optional, Union, Any -import queue -from collections import deque +from typing import Any, Optional, Union import numpy as np import torch import torch.profiler as torch_profiler import tqdm from aiter import destroy_dist_env, dtypes, init_dist_env -from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture +from aiter.dist.parallel_state import graph_capture from atom.config import Config, set_current_atom_config -from atom.model_engine.scheduler import ScheduledBatchs, PrevScheduledBatchs +from atom.model_engine.scheduler import ScheduledBatch from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler @@ -36,42 +32,31 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } -class OutputProcessor: - def __init__(self): +class tokenIDProcessor: + + def __init__(self, max_num_batched_tokens: int, device: torch.device): """Asynchronously copy the sampled_token_ids tensor to the host.""" + self.is_deferred_out = True + self.input_ids = CpuGpuBuffer( + max_num_batched_tokens, dtype=torch.int32, device=device + ) + self.input_ids_loc = CpuGpuBuffer( + max_num_batched_tokens, dtype=torch.int64, device=device + ) # Event on the copy stream so we can synchronize the non-blocking copy. self.async_copy_event = torch.cuda.Event() self.async_copy_stream = torch.cuda.Stream() - self.deferred_request_id: list[list[int]] = [] - self.deferred_token_id: list[list[int]] = [] - self.token_ids_gpu: list[torch.Tensor] = [] - self.token_ids_cpu: list[torch.Tensor] = [] - - self.token_req_ids: list[list[int]] = [] - - self.pending_outputs: deque = deque() # saving (cpu_tensor, seqs) - + self.clean() - def send_to_cpu_async(self, batch: ScheduledBatchs, gpu_tensor: torch.Tensor): + def send_to_cpu_async(self, gpu_tensor: torch.Tensor): default_stream = torch.cuda.current_stream() with torch.cuda.stream(self.async_copy_stream): self.async_copy_stream.wait_stream(default_stream) cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) self.async_copy_event.record(self.async_copy_stream) self.token_ids_cpu.append(cpu_tensor) - self.pending_outputs.append((cpu_tensor, batch.seqs)) - - def revc_async_output_tuple(self) -> tuple[list[int], list[Sequence]]: - if not self.pending_outputs: - return [], [] - - self.async_copy_event.synchronize() - if self.pending_outputs: - cpu_tensor, seqs = self.pending_outputs.popleft() - token_ids = cpu_tensor.tolist() - return token_ids, seqs - return [], [] + self.token_ids_gpu.append(gpu_tensor) def recv_async_output(self) -> list[int]: self.async_copy_event.synchronize() @@ -80,18 +65,125 @@ def recv_async_output(self) -> list[int]: return token_ids return [] - def clean_token(self): + def clean(self): + self.token_ids_gpu: list[torch.Tensor] = [] self.token_ids_cpu: list[torch.Tensor] = [] - self.pending_outputs.clear() - def update_and_ret( - self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor - ) -> list[int]: + self.prev_batch: Optional[ScheduledBatch] = None + + def prepare_sampled_ids( + self, batch: ScheduledBatch, sampled_token_ids: torch.Tensor + ) -> dict[int, int]: + if not self.is_deferred_out: + token_ids = sampled_token_ids.tolist() + seq_ids = batch.seqs.keys() + return {seq_id: token_id for seq_id, token_id in zip(seq_ids, token_ids)} token_ids = self.recv_async_output() - self.send_to_cpu_async(batch, sampled_token_ids) + self.send_to_cpu_async(sampled_token_ids) + if self.prev_batch is not None: + seq_ids = self.prev_batch.seqs.keys() + token_ids = { + seq_id: token_id for seq_id, token_id in zip(seq_ids, token_ids) + } + else: + # first time, no previous tokens + token_ids = {} + + self.prev_batch = batch + self.prev_token_ids = sampled_token_ids return token_ids + def get_prev_alive_locations(self, batch: ScheduledBatch) -> tuple[list[int], bool]: + token_ids = self.prev_token_ids + deferred_prev_indices = [ + i + for i, seq_id in enumerate(self.prev_batch.seqs.keys()) + if seq_id in batch.seqs + ] + return ( + deferred_prev_indices, + len(deferred_prev_indices) == token_ids.size(0), + ) + + def prepare_input_ids( + self, + batch: ScheduledBatch, + ) -> torch.Tensor: + """Prepare the input IDs for the current batch. + + Carefully handles the `prev_sampled_token_ids` which can be cached + from the previous engine iteration, in which case those tokens on the + GPU need to be copied into the corresponding slots into input_ids.""" + seqs = list(batch.seqs.values()) + token_nums = batch.num_scheduled_tokens + total_tokens = batch.total_tokens_num + total_tokens_prefill = batch.total_tokens_num_prefill + total_tokens_decode = batch.total_tokens_num_decode + total_reqs_prefill = batch.total_seqs_num_prefill + total_reqs_decode = batch.total_seqs_num_decode + """for prefill: all input ids are new""" + start_loc = 0 + for seq, new_token_num in zip( + seqs[:total_reqs_prefill], token_nums[:total_reqs_prefill] + ): + self.input_ids.np[start_loc : start_loc + new_token_num] = seq[ + seq.num_cached_tokens : + ] + start_loc += new_token_num + self.input_ids.copy_to_gpu(total_tokens_prefill) + + # TODO: remove this when we support mixed prefill and decode in one batch + if total_reqs_prefill > 0: + return self.input_ids.gpu[:total_tokens_prefill] + + if not self.is_deferred_out: + token_ids = [ + seq.token_ids[-1] + for seq in seqs[ + total_reqs_prefill : total_reqs_prefill + total_reqs_decode + ] + ] + self.input_ids.np[:total_tokens_decode] = token_ids + self.input_ids.copy_to_gpu(total_tokens_decode) + return self.input_ids.gpu[:total_tokens_decode] + + """for decode: input ids are from prev_sampled_token_ids""" + locations, is_all_alive = self.get_prev_alive_locations(batch) + num_deferred_tokens = len(locations) + if is_all_alive: + num_norm_tokens = total_tokens_decode - num_deferred_tokens + if num_norm_tokens > 0: + token_ids = [ + seq.token_ids[-1] + for seq in seqs[ + total_reqs_prefill : total_reqs_prefill + num_norm_tokens + ] + ] + self.input_ids.np[:num_norm_tokens] = token_ids + self.input_ids.copy_to_gpu(num_norm_tokens) + # no new requests added and old requests finished + self.input_ids.gpu[ + num_norm_tokens : num_norm_tokens + num_deferred_tokens + ] = self.prev_token_ids + + elif num_deferred_tokens == total_tokens_decode: + # no new requests added but some old requests finished + self.input_ids_loc.np[:num_deferred_tokens] = locations + self.input_ids_loc.copy_to_gpu(num_deferred_tokens) + torch.gather( + self.prev_token_ids, + 0, + self.input_ids_loc.gpu[:num_deferred_tokens], + out=self.input_ids.gpu[:num_deferred_tokens], + ) + else: + # TODO: new requests' input_ids need to be filled in + assert False, "TODO new requests' input_ids need to be filled in" + # print(f"{self.input_ids.gpu[:total_tokens]=}") + return self.input_ids.gpu[:total_tokens] + + class ModelRunner: def __init__(self, rank: int, config: Config): @@ -124,61 +216,61 @@ def __init__(self, rank: int, config: Config): default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") - self.out_processor = OutputProcessor() + self.tokenID_processor = tokenIDProcessor( + self.config.max_num_batched_tokens, self.device + ) self.sampler = Sampler() - - self.prev_input_batch = PrevScheduledBatchs( - max_num_reqs=self.config.max_num_seqs, - max_model_len=self.config.max_model_len) - - self.input_ids = self._make_buffer(self.config.max_num_batched_tokens, - dtype=torch.int32) - self.positions = self._make_buffer(self.config.max_num_batched_tokens, - dtype=torch.int64) - - self.arange_np = np.arange(max(self.config.max_num_seqs + 1, - self.config.max_model_len, - self.config.max_num_batched_tokens), - dtype=np.int64) - - - self.input_ids_index_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) - self.prev_common_req_indices_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) - - self.use_async_scheduling = True - self.async_output_copy_stream = torch.cuda.Stream() if \ - self.use_async_scheduling else None - - self.out_processor = OutputProcessor() - - + self.arange_np = np.arange( + max( + self.config.max_num_seqs + 1, + self.config.max_model_len, + self.config.max_num_batched_tokens, + ), + dtype=np.int64, + ) + self.async_output_copy_stream = torch.cuda.Stream() self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) torch.set_default_device(None) load_model(self.model, config.model, config.hf_config, config.load_dummy) torch.set_default_device("cuda") + self.allocate_forward_vars() self.warmup_model() logger.info(f"Model warmup done: {config.model}") - self.allocate_decode_vars() torch.set_default_device("cpu") torch.set_default_dtype(default_dtype) if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") - - def _make_buffer(self, - *size: Union[int, torch.SymInt], - dtype: torch.dtype, - numpy: bool = True) -> CpuGpuBuffer: + def _make_buffer( + self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True + ) -> CpuGpuBuffer: # Bfloat16 torch tensors cannot be directly cast to a numpy array, so # if a bfloat16 buffer is needed without a corresponding numpy array, # don't bother instantiating the numpy array. - return CpuGpuBuffer(*size, - dtype=dtype, - device=self.device, - pin_memory=True, - with_numpy=numpy) + return CpuGpuBuffer( + *size, dtype=dtype, device=self.device, pin_memory=True, with_numpy=numpy + ) + + def _get_cumsum_and_arange( + self, + num_tokens: np.ndarray, + cumsum_dtype: Optional[np.dtype] = None, + ) -> tuple[np.ndarray, np.ndarray]: + """Get the cumulative sum and batched arange of the given array. + # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) + # Equivalent to but faster than: + # np.concatenate([np.arange(n) for n in num_tokens]) + """ + # Step 1. [2, 5, 3] -> [2, 7, 10] + cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) + total_num_tokens = cu_num_tokens[-1] + # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] + cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) + # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + arange = self.arange_np[:total_num_tokens] - cumsums_offsets + return cu_num_tokens, arange def exit(self): if not self.still_running: @@ -227,53 +319,53 @@ def warmup_model(self): Sequence([0] * max_model_len, block_size=self.block_size) for _ in range(num_seqs) ] - - num_scheduled_tokens: dict[str, int] = {} - - for seq in seqs: - num_scheduled_tokens[seq.id] = 1 - - total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) - dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) + seqs = {seq.id: seq for seq in seqs} + + num_scheduled_tokens = np.array([max_model_len] * num_seqs, dtype=np.int32) + total_tokens_num = num_scheduled_tokens.sum() + + dummy_batch = ScheduledBatch( + seqs=seqs, + num_scheduled_tokens=num_scheduled_tokens, + total_tokens_num=total_tokens_num, + total_tokens_num_prefill=total_tokens_num, + total_seqs_num=num_seqs, + total_seqs_num_prefill=num_seqs, + ) self.forward(dummy_batch) - self.out_processor.clean_token() + self.tokenID_processor.clean() torch.cuda.empty_cache() - def allocate_decode_vars(self): + def allocate_forward_vars(self): config = self.config hidden_size = config.hf_config.hidden_size hidden_type = config.hf_config.torch_dtype max_bs = min(self.config.max_num_seqs, 512) max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size - self.decode_vars = { - "input_ids": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), - "positions": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), - "slot_mapping": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), - "context_lens": CpuGpuBuffer(max_bs, dtype=torch.int32, device=self.device), - "block_tables": CpuGpuBuffer( - max_bs, max_num_blocks, dtype=torch.int32, device=self.device - ), - "cu_seqlens_q": CpuGpuBuffer( - max_bs + 1, dtype=torch.int32, device=self.device - ), - "kv_indptr": CpuGpuBuffer( - max_bs + 1, dtype=torch.int32, device=self.device - ), - "kv_indices": CpuGpuBuffer( - max_bs * max_num_blocks, dtype=torch.int32, device=self.device - ), - "kv_last_page_lens": CpuGpuBuffer( - max_bs, dtype=torch.int32, device=self.device - ), + max_num_batched_tokens = config.max_num_batched_tokens + i64_kwargs = {"dtype": torch.int64, "device": self.device} + i32_kwargs = {"dtype": torch.int32, "device": self.device} + f32_kwargs = {"dtype": torch.float, "device": self.device} + self.forward_vars = { + "input_ids": self.tokenID_processor.input_ids, + "positions": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), + "slot_mapping": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), + "context_lens": CpuGpuBuffer(max_bs, **i32_kwargs), + "block_tables": CpuGpuBuffer(max_bs, max_num_blocks, **i32_kwargs), + "temperatures": CpuGpuBuffer(max_bs, **f32_kwargs), + "cu_seqlens_q": CpuGpuBuffer(max_bs + 1, **i32_kwargs), + "kv_indptr": CpuGpuBuffer(max_bs + 1, **i32_kwargs), + "kv_indices": CpuGpuBuffer(max_bs * max_num_blocks, **i32_kwargs), + "kv_last_page_lens": CpuGpuBuffer(max_bs, **i32_kwargs), "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), } - self.decode_vars["cu_seqlens_q"].cpu.copy_( + self.forward_vars["cu_seqlens_q"].cpu.copy_( torch.arange(0, max_bs + 1, step=1, dtype=torch.int32) ) - self.decode_vars["cu_seqlens_q"].copy_to_gpu() + self.forward_vars["cu_seqlens_q"].copy_to_gpu() - self.decode_vars["kv_last_page_lens"].cpu.fill_(1) - self.decode_vars["kv_last_page_lens"].copy_to_gpu() + self.forward_vars["kv_last_page_lens"].cpu.fill_(1) + self.forward_vars["kv_last_page_lens"].copy_to_gpu() def get_num_blocks(self): torch.set_default_device("cuda") @@ -389,96 +481,27 @@ def allocate_kv_cache(self, num_kvcache_blocks): return True def prepare_block_tables(self, seqs: list[Sequence]): - lens = [len(seq.block_table) for seq in seqs] - block_tables = np.empty((len(seqs), max(lens)), dtype=np.int32) + block_tables = self.forward_vars["block_tables"].np for i, seq in enumerate(seqs): - block_tables[i, : lens[i]] = seq.block_table - return block_tables - - def _prepare_input_ids(self, total_num_scheduled_tokens: int, - cu_num_tokens: np.ndarray) -> None: - """Prepare the input IDs for the current batch. - - Carefully handles the `prev_sampled_token_ids` which can be cached - from the previous engine iteration, in which case those tokens on the - GPU need to be copied into the corresponding slots into input_ids.""" - - if self.prev_input_batch.prev_sampled_token_ids is None: - # Normal scheduling case - self.input_ids.copy_to_gpu(total_num_scheduled_tokens) - return - - # Async scheduling case, where some decode requests from the previous - # iteration won't have entries in input_ids_cpu and need to be copied - # on the GPU from prev_sampled_token_ids. - prev_req_id_to_index = self.prev_input_batch.prev_req_id_to_index - assert prev_req_id_to_index is not None - flattened_indices = [] - prev_common_req_indices = [] - indices_match = True - max_flattened_index = -1 - for req_id, cur_index in self.prev_input_batch.req_id_to_index.items(): - if (prev_index := prev_req_id_to_index.get(req_id)) is not None: - prev_common_req_indices.append(prev_index) - # We need to compute the flattened input_ids index of the - # last token in each common request. - flattened_index = cu_num_tokens[cur_index].item() - 1 - # flattened_indices.append(flattened_index) - flattened_indices.append(prev_index) - indices_match &= (prev_index == flattened_index) - max_flattened_index = max(max_flattened_index, flattened_index) - num_commmon_tokens = len(flattened_indices) - if num_commmon_tokens < total_num_scheduled_tokens: - # If not all requests are decodes from the last iteration, - # We need to copy the input_ids_cpu to the GPU first. - self.input_ids.copy_to_gpu(total_num_scheduled_tokens) - if num_commmon_tokens == 0: - # No requests in common with the previous iteration - # So input_ids_cpu will have all the input ids. - return - if indices_match and max_flattened_index == (num_commmon_tokens - 1): - # Common-case optimization: the batch is unchanged - # and no reordering happened. - # The indices are both the same permutation of 0..N-1 so - # we can copy directly using a single slice. - self.input_ids.gpu[:num_commmon_tokens].copy_( - self.prev_input_batch.prev_sampled_token_ids[:num_commmon_tokens], - non_blocking=True) - return - # Upload the index tensors asynchronously - # so the scatter can be non-blocking. - - self.input_ids_index_tensor.np[:num_commmon_tokens] = np.array(flattened_indices, dtype=np.int64) - self.prev_common_req_indices_tensor.np[:num_commmon_tokens] = np.array(prev_common_req_indices, dtype=np.int64) - - self.input_ids_index_tensor.copy_to_gpu(num_commmon_tokens) - self.prev_common_req_indices_tensor.copy_to_gpu(num_commmon_tokens) - - self.input_ids.gpu.scatter_( - dim=0, - index=self.input_ids_index_tensor.gpu[:num_commmon_tokens], - src=self.prev_input_batch.prev_sampled_token_ids[ - self.prev_common_req_indices_tensor.gpu[:num_commmon_tokens]]) - # prev_common_req_indices_tensor, 0]) - + block_tables[i] = 0 + block_tables[i, : seq.num_blocks] = seq.block_table - - def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): - input_ids = [] + def prepare_prefill(self, batch: ScheduledBatch): + bs = batch.total_seqs_num_prefill + sum_scheduled_tokens = batch.total_tokens_num_prefill + var = self.forward_vars positions = [] - cu_seqlens_q = [0] cu_seqlens_k = [0] max_seqlen_q = 0 max_seqlen_k = 0 slot_mapping = [] - block_tables = None - for seq in scheduled_batchs.seqs: - seqlen = len(seq) - input_ids.extend(seq[seq.num_cached_tokens :]) + seqs = list(batch.seqs.values()) + seqs = seqs[:bs] + for seq in seqs: + seqlen = seq.num_tokens positions.extend(list(range(seq.num_cached_tokens, seqlen))) seqlen_q = seqlen - seq.num_cached_tokens seqlen_k = seqlen - cu_seqlens_q.append(cu_seqlens_q[-1] + seqlen_q) cu_seqlens_k.append(cu_seqlens_k[-1] + seqlen_k) max_seqlen_q = max(seqlen_q, max_seqlen_q) max_seqlen_k = max(seqlen_k, max_seqlen_k) @@ -491,84 +514,78 @@ def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): else: end = start + seq.last_block_num_tokens slot_mapping.extend(list(range(start, end))) - if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache - block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - block_tables = torch.from_numpy(block_tables).cuda(non_blocking=True) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) - cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + if cu_seqlens_k[-1] > batch.total_tokens_num: # prefix cache + self.prepare_block_tables(seqs) + var["positions"].np[:sum_scheduled_tokens] = positions + var["slot_mapping"].np[: len(slot_mapping)] = slot_mapping cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) + min_seqlen_q = 0 dropout_p = 0.0 + vars_used = [ + ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("slot_mapping", len(slot_mapping)), + ] + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} set_context( True, - len(scheduled_batchs.seqs), - cu_seqlens_q=cu_seqlens_q.cuda(non_blocking=True), + batch_size=bs, cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), + # slot_mapping=slot_mapping.cuda(non_blocking=True), max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping.cuda(non_blocking=True), - block_tables=block_tables, dropout_p=dropout_p, + **ctx, ) - return input_ids.cuda(non_blocking=True), positions.cuda(non_blocking=True) - - def prepare_decode(self, scheduled_batchs: ScheduledBatchs): - bs = len(scheduled_batchs.seqs) - graph_bs = ( - bs if self.enforce_eager else next(x for x in self.graph_bs if x >= bs) + return var["positions"].copy_to_gpu(sum_scheduled_tokens) + + def prepare_decode(self, batch: ScheduledBatch): + scheduled_bs = batch.total_seqs_num_decode + seqs = list(batch.seqs.values()) + seqs = seqs[batch.total_seqs_num_prefill :] + assert len(seqs) == scheduled_bs + bs = ( + scheduled_bs + if self.enforce_eager + else next(x for x in self.graph_bs if x >= scheduled_bs) ) - assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" - input_ids = [] - positions = [] - slot_mapping = [] - context_lens = [] + assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" dropout_p = 0.0 max_q_len = 1 self.total_blocks = 0 - context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] - positions = np.array(context_lens, dtype=np.int64) - context_lens = np.array(context_lens, dtype=np.int32) - input_ids = [seq.last_token for seq in scheduled_batchs.seqs] - input_ids = np.array(input_ids, dtype=np.int64) + context_lens = [seq.num_tokens for seq in seqs] + positions = context_lens slot_mapping = [ seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 - for seq in scheduled_batchs.seqs + for seq in seqs ] - slot_mapping.extend([-1] * (graph_bs - bs)) - slot_mapping = np.array(slot_mapping, dtype=np.int64) + slot_mapping.extend([-1] * (bs - scheduled_bs)) if isinstance(self.model, DeepseekV2ForCausalLM): assert self.block_size == 1, "AITER MLA requires only block size 1." self.kv_indices: list[int] = [] self.kv_indptr: list[int] = [0] self.kv_last_page_lens: list[int] = [] - qlens = [1 for seq in scheduled_batchs.seqs] # TODO support mtp - num_scheduled_tokens = np.array(qlens, dtype=np.int32) - max_q_len = max(qlens) - cu_seqlens_q = np.cumsum(num_scheduled_tokens, dtype=np.int32) - for seq in scheduled_batchs.seqs: - current_seq_len = len(seq) + for seq in seqs: + current_seq_len = seq.num_tokens self._update_paged_kv_tensors(seq.block_table, current_seq_len) - block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - # cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) - # cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens - num_input_tokens = num_scheduled_tokens - prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] - - var = self.decode_vars - var["slot_mapping"].np[:graph_bs] = slot_mapping - var["input_ids"].gpu[:bs] = prev_input_ids_[:bs] - var["positions"].np[:bs] = positions - - var["context_lens"].np[:bs] = context_lens - var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables + self.prepare_block_tables(seqs) + + sum_scheduled_tokens = batch.total_tokens_num_decode + var = self.forward_vars + var["slot_mapping"].np[:bs] = slot_mapping + var["positions"].np[:sum_scheduled_tokens] = positions + var["context_lens"].np[:scheduled_bs] = context_lens + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + ("block_tables", bs), + ] if isinstance(self.model, DeepseekV2ForCausalLM) and len(self.kv_indptr) > 0: # extend to the maximum number of blocks as returned by the scheduler @@ -576,41 +593,37 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): var["kv_indices"].np[: self.total_blocks] = np.array( self.kv_indices, dtype=np.int64 ) - var["kv_indptr"].np[: bs + 1] = np.array(self.kv_indptr, dtype=np.int64) - var["kv_indptr"].np[bs + 1 : graph_bs + 1] = var["kv_indptr"].np[bs] + var["kv_indptr"].np[: scheduled_bs + 1] = np.array( + self.kv_indptr, dtype=np.int64 + ) + var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[bs] var["kv_last_page_lens"].np[:bs] = np.array( self.kv_last_page_lens, dtype=np.int64 ) - var["cu_seqlens_q"].np[1 : bs + 1] = np.array(cu_seqlens_q, dtype=np.int64) - - for el in [ - "slot_mapping", - # "input_ids", - "positions", - "context_lens", - "block_tables", - "cu_seqlens_q", - "kv_indices", - "kv_indptr", - "kv_last_page_lens", - ]: - var[el].copy_to_gpu() + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("kv_indices", bs), + ("kv_indptr", bs), + ("kv_last_page_lens", bs), + ] + + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} set_context( False, - batch_size=bs, - graph_bs=graph_bs, - cu_seqlens_q=var["cu_seqlens_q"].gpu[: bs + 1], - slot_mapping=var["slot_mapping"].gpu[:bs], - context_lens=var["context_lens"].gpu[:bs], - block_tables=var["block_tables"].gpu[:bs, : block_tables.shape[1]], + batch_size=scheduled_bs, + graph_bs=bs, dropout_p=dropout_p, max_q_len=max_q_len, - kv_indptr=var["kv_indptr"].gpu[: bs + 1], - kv_indices=var["kv_indices"].gpu, - kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], + **ctx, ) - return prev_input_ids_, positions - return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) + # for el in ctx: + # print(f"{el}: {ctx[el]}") + # print(f"positions: {positions}") + return positions def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): # Get the number of valid blocks based on sequence length. @@ -632,145 +645,80 @@ def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): last_page_len = self.block_size self.kv_last_page_lens.append(last_page_len) - def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: - temperatures = [seq.temperature for seq in seqs] - temperatures = torch.tensor( - temperatures, dtype=torch.float32, pin_memory=True - ).cuda(non_blocking=True) - return temperatures - - def _get_cumsum_and_arange( - self, - num_tokens: np.ndarray, - cumsum_dtype: Optional[np.dtype] = None, - ) -> tuple[np.ndarray, np.ndarray]: - """Get the cumulative sum and batched arange of the given array. - # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) - # Equivalent to but faster than: - # np.concatenate([np.arange(n) for n in num_tokens]) - """ - # Step 1. [2, 5, 3] -> [2, 7, 10] - cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) - total_num_tokens = cu_num_tokens[-1] - # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] - cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) - # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - arange = self.arange_np[:total_num_tokens] - cumsums_offsets - - return cu_num_tokens, arange - + def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: + temperatures = [seq.temperature for seq in batch.seqs.values()] + bs = batch.total_seqs_num + buffer = self.forward_vars["temperatures"] + buffer.np[:bs] = temperatures + return buffer.copy_to_gpu(bs) - def prepare_model(self, scheduled_batchs: ScheduledBatchs): - self._update_states(scheduled_batchs) + def prepare_model(self, batch: ScheduledBatch): + total_tokens_num = batch.total_tokens_num + assert total_tokens_num > 0 + bs = batch.total_seqs_num + seqs = batch.seqs.values() - total_num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens - assert total_num_scheduled_tokens > 0 + num_scheduled_tokens = batch.num_scheduled_tokens + cu_seqlens_q, arange = self._get_cumsum_and_arange(num_scheduled_tokens) + self.forward_vars["cu_seqlens_q"].np[1 : bs + 1] = cu_seqlens_q - _req_ids = self.prev_input_batch.req_ids - req_ids = [req_id for req_id in _req_ids if req_id is not None] + input_ids = self.tokenID_processor.prepare_input_ids(batch) - tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] - num_scheduled_tokens = np.array(tokens, dtype=np.int32) - # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] - # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - cu_num_tokens, arange = self._get_cumsum_and_arange( - num_scheduled_tokens) - # cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) - - self._prepare_input_ids(total_num_scheduled_tokens, cu_num_tokens) - seqs = scheduled_batchs.seqs - is_prefill = scheduled_batchs.is_prefill + is_prefill = batch.total_tokens_num_prefill > 0 prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode - input_ids, positions = prepare_func(scheduled_batchs) - temperatures = self.prepare_sample(seqs) if self.rank == 0 else None - return input_ids, positions, temperatures - - def run_model( - self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool - ): - # torch.cuda.empty_cache() + positions = prepare_func(batch) + temperatures = self.prepare_sample(batch) + return ( + input_ids, + positions, + temperatures, + ) + + def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor): ctx = get_context() bs = ctx.batch_size + is_prefill = ctx.is_prefill if is_prefill or self.enforce_eager or bs > self.graph_bs[-1]: - return self.model.compute_logits(self.model(input_ids, positions)) + hidden_states = self.model(input_ids, positions) else: graph_bs = ctx.graph_bs self.graphs[graph_bs].replay() - return self.model.compute_logits(self.decode_vars["outputs"][:bs]) + hidden_states = self.forward_vars["outputs"][:bs] + return self.model.compute_logits(hidden_states) def postprocess( self, - batch: ScheduledBatchs, + batch: ScheduledBatch, logits: torch.Tensor, - temperatures: Optional[torch.Tensor], - ) -> list[int]: - if self.rank == 0: - sampled_tokens = self.sampler(logits, temperatures) - - # num_sampled_tokens = sampled_tokens.shape[0] - sampled_token_ids = sampled_tokens - if self.use_async_scheduling: - # assert sampled_token_ids.shape[-1] == 1 - - # Cache the sampled tokens on the GPU and avoid CPU sync. - # These will be copied into input_ids in the next step - # when preparing inputs. - invalid_req_indices_set = [None] - self.prev_input_batch.prev_sampled_token_ids = \ - sampled_token_ids - self.prev_input_batch.prev_req_id_to_index = { - req_id: i - for i, req_id in enumerate(self.prev_input_batch.req_ids) - if req_id not in invalid_req_indices_set - } - # plus 1 to fit next iter seq len - # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] - # positions = np.array(context_lens, dtype=np.int64) - - # token_ids = sampled_tokens.tolist() - output = self.out_processor.update_and_ret( - batch, - sampled_tokens, - ) - else: - output = None - return output - - def _update_states(self, batch: ScheduledBatchs): - for req_id in self.prev_input_batch.finished_req_ids: - self.prev_input_batch.remove_request(req_id) - - for request in batch.seqs: - self.prev_input_batch.add_request(request) - - def finish_req(self, batch: ScheduledBatchs): - request_id = self.prev_input_batch.req_ids - for id in request_id: - self.prev_input_batch.finished_req_ids.add(id) - + temperatures: torch.Tensor, + ) -> dict[int, int]: + sampled_tokens = self.sampler(logits, temperatures) + token_ids = self.tokenID_processor.prepare_sampled_ids( + batch, + sampled_tokens, + ) + return token_ids @torch.inference_mode() - def forward(self, batch: ScheduledBatchs, is_wramup = False) -> list[int]: - # self._update_states(batch) + def forward(self, batch: ScheduledBatch) -> dict[int, int]: input_ids, positions, temperatures = self.prepare_model(batch) - logits = self.run_model(input_ids, positions, batch.is_prefill) + logits = self.run_model(input_ids, positions) reset_context() - self.finish_req(batch) return self.postprocess(batch, logits, temperatures) @torch.inference_mode() def capture_cudagraph(self): start_time = time.time() - input_ids = self.decode_vars["input_ids"].gpu - positions = self.decode_vars["positions"].gpu - slot_mapping = self.decode_vars["slot_mapping"].gpu - context_lens = self.decode_vars["context_lens"].gpu - block_tables = self.decode_vars["block_tables"].gpu - cu_seqlens_q = self.decode_vars["cu_seqlens_q"].gpu - kv_indptr = self.decode_vars["kv_indptr"].gpu - kv_indices = self.decode_vars["kv_indices"].gpu - kv_last_page_lens = self.decode_vars["kv_last_page_lens"].gpu - outputs = self.decode_vars["outputs"] + input_ids = self.forward_vars["input_ids"].gpu + positions = self.forward_vars["positions"].gpu + slot_mapping = self.forward_vars["slot_mapping"].gpu + context_lens = self.forward_vars["context_lens"].gpu + block_tables = self.forward_vars["block_tables"].gpu + cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu + kv_indptr = self.forward_vars["kv_indptr"].gpu + kv_indices = self.forward_vars["kv_indices"].gpu + kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu + outputs = self.forward_vars["outputs"] # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) if self.config.compilation_config.cudagraph_capture_sizes: @@ -790,12 +738,10 @@ def capture_cudagraph(self): with graph_capture() as gc: capture_range = ( - tqdm.tqdm(self.graph_bs) - if get_tensor_model_parallel_rank() == 0 - else self.graph_bs + tqdm.tqdm(self.graph_bs) if self.rank == 0 else self.graph_bs ) for bs in capture_range: - if get_tensor_model_parallel_rank() == 0: + if self.rank == 0: capture_range.set_description(f"Capturing {bs=}") graph = torch.cuda.CUDAGraph() diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 2afa33b783..146422ab63 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -1,118 +1,45 @@ +import logging from collections import deque from dataclasses import dataclass, field -from typing import Optional, cast, Any +from typing import Any, Optional, cast -from atom.config import Config -from atom.model_engine.block_manager import BlockManager -from atom.model_engine.sequence import Sequence, SequenceStatus - -import torch import numpy as np +import torch -@dataclass -class ScheduledBatchs: - seqs: list[Sequence] - is_prefill: bool - deferred_reqID: list[int] - - unfinished_prev_req: list[Sequence] - - # req_id -> num_scheduled_tokens - # Number of tokens scheduled for each request. - num_scheduled_tokens: dict[str, int] - # Total number of tokens scheduled for all requests. - # Equal to sum(num_scheduled_tokens.values()) - total_num_scheduled_tokens: int +from atom.config import Config +from atom.model_engine.block_manager import BlockManager +from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType +logger = logging.getLogger("atom") -class PrevScheduledBatchs: +class ScheduledBatch: def __init__( self, - max_num_reqs: int, - max_model_len: int + seqs: dict[int, Sequence], + num_scheduled_tokens: np.ndarray, + total_tokens_num: int, + total_tokens_num_prefill: int = 0, + total_tokens_num_decode: int = 0, + total_seqs_num: int = 0, + total_seqs_num_prefill: int = 0, + total_seqs_num_decode: int = 0, ): - self.max_num_reqs = max_num_reqs - self.max_model_len = max_model_len - self.token_ids_cpu_tensor = torch.zeros( - (max_num_reqs, max_model_len), - device="cpu", - dtype=torch.int32, - pin_memory=False, - ) - self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() - self.num_tokens = np.zeros(max_num_reqs, dtype=np.int32) - self.req_id_to_index: dict[str, int] = {} - - # Cached reference to the GPU tensor of previously sampled tokens - self.prev_sampled_token_ids: Optional[torch.Tensor] = None - self.prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None - self.prev_req_id_to_index: Optional[dict[str, int]] = None - self.prev_position_ids: Optional[torch.Tensor] = None - - self._req_ids: list[Optional[str]] = [] - self.req_output_token_ids: list[Optional[list[int]]] = [] - - self.finished_req_ids: set[str] = set() - - @property - def num_reqs(self) -> int: - return len(self.req_id_to_index) - - def _register_add_request(self, request: "Sequence") -> int: - """Track add-request operations for logits processors. - Not applicable to pooling models. - """ - - new_req_index = self.num_reqs - - assert new_req_index < self.max_num_reqs - return new_req_index - - - def add_request( - self, - request: "Sequence", - ) -> int: - req_index = self._register_add_request(request) - - req_id = request.id - - if req_index == len(self._req_ids): - self._req_ids.append(req_id) - # self.req_output_token_ids.append(request.output_token_ids) - else: - self._req_ids[req_index] = req_id - # self.req_output_token_ids[req_index] = request.output_token_ids - self.req_id_to_index[req_id] = req_index - - # new_req_index = request.last_token - # self.req_id_to_index[req_id] = new_req_index - - # self._req_ids.append(req_id) + # len(seqs) == total_seqs_num == total_seqs_num_prefill + total_seqs_num_decode + self.seqs = seqs - def remove_request(self, req_id: str) -> Optional[int]: - """This method must always be followed by a call to condense(). + # num_scheduled_tokens for each sequence in the batch + self.num_scheduled_tokens = num_scheduled_tokens - Args: - req_id: request to remove + # Total number of tokens scheduled for all requests. + self.total_tokens_num = total_tokens_num + self.total_tokens_num_prefill = total_tokens_num_prefill + self.total_tokens_num_decode = total_tokens_num_decode - Returns: - Removed request index, or `None` if `req_id` not recognized - """ - req_index = self.req_id_to_index.pop(req_id, None) - if req_index is None: - return None - - self._req_ids[req_index] = None - # self.req_output_token_ids[req_index] = None - - - @property - def req_ids(self) -> list[str]: - # None elements should only be present transiently - # while performing state updates to the batch. - return cast(list[str], self._req_ids) + # Total number of reqs scheduled for all requests. + self.total_seqs_num = total_seqs_num + self.total_seqs_num_prefill = total_seqs_num_prefill + self.total_seqs_num_decode = total_seqs_num_decode class Scheduler: @@ -125,7 +52,6 @@ def __init__(self, config: Config): self.block_manager = BlockManager(config) self.waiting: deque[Sequence] = deque() self.running: deque[Sequence] = deque() - self.deferred_reqID = [] def is_finished(self): return not self.waiting and not self.running @@ -136,45 +62,55 @@ def add(self, seq: Sequence): def extend(self, seqs: list[Sequence]): self.waiting.extend(seqs) - def schedule(self) -> ScheduledBatchs: + def schedule(self) -> ScheduledBatch: # prefill - scheduled_seqs = [] - num_seqs = 0 + scheduled_seqs = {} + num_seqs_prefill = 0 num_batched_tokens = 0 - num_scheduled_tokens: dict[str, int] = {} + num_scheduled_tokens: list[int] = [] if not self.running and not self.waiting: self.block_manager.reset() - while self.waiting and num_seqs < self.max_num_seqs: + while self.waiting and num_seqs_prefill < self.max_num_seqs: seq = self.waiting[0] - if num_batched_tokens + len( - seq - ) > self.max_num_batched_tokens or not self.block_manager.can_allocate(seq): + num_new_tokens = seq.num_tokens - seq.num_cached_tokens + if ( + num_batched_tokens + num_new_tokens > self.max_num_batched_tokens + or not self.block_manager.can_allocate(seq) + ): break - num_seqs += 1 + num_seqs_prefill += 1 self.block_manager.allocate(seq) - num_batched_tokens += len(seq) - seq.num_cached_tokens + num_batched_tokens += num_new_tokens seq.status = SequenceStatus.RUNNING + seq.type = SequenceType.PREFILL self.waiting.popleft() self.running.append(seq) - scheduled_seqs.append(seq) - # in vllm, using num_new_tokens - num_new_tokens = 1 - num_scheduled_tokens[seq.id] = num_new_tokens - - total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) - - if scheduled_seqs: - return ScheduledBatchs( - seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[], unfinished_prev_req=scheduled_seqs, - num_scheduled_tokens=num_scheduled_tokens, - total_num_scheduled_tokens=total_num_scheduled_tokens, + scheduled_seqs[seq.id] = seq + num_scheduled_tokens.append(num_new_tokens) + + num_scheduled_tokens_np = np.array(num_scheduled_tokens, dtype=np.int32) + total_tokens_num_prefill = num_scheduled_tokens_np.sum() + + if num_seqs_prefill > 0: + logger.info( + f"scheduled prefill batch: {num_seqs_prefill} reqs, {total_tokens_num_prefill} tokens" + ) + # lip: TODO for prefill/decode mixed batch + return ScheduledBatch( + seqs=scheduled_seqs, + num_scheduled_tokens=num_scheduled_tokens_np, + total_tokens_num=total_tokens_num_prefill, + total_tokens_num_prefill=total_tokens_num_prefill, + total_seqs_num=num_seqs_prefill, + total_seqs_num_prefill=num_seqs_prefill, ) # decode - while self.running and num_seqs < self.max_num_seqs: + num_seqs_decode = 0 + while self.running and num_seqs_decode < self.max_num_seqs: seq = self.running.popleft() while not self.block_manager.can_append(seq): if self.running: @@ -183,23 +119,29 @@ def schedule(self) -> ScheduledBatchs: self.preempt(seq) break else: - num_seqs += 1 + num_seqs_decode += 1 self.block_manager.may_append(seq) - scheduled_seqs.append(seq) - num_new_tokens = 1 - num_scheduled_tokens[seq.id] = num_new_tokens - - total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + scheduled_seqs[seq.id] = seq + seq.type = SequenceType.DECODE + num_new_tokens = 1 + num_scheduled_tokens.append(num_new_tokens) + + num_scheduled_tokens_np = np.array(num_scheduled_tokens, dtype=np.int32) + total_tokens_num_decode = num_scheduled_tokens_np.sum() assert scheduled_seqs - self.running.extendleft(reversed(scheduled_seqs)) - return ScheduledBatchs( + self.running.extendleft(reversed(scheduled_seqs.values())) + # logger.info( + # f"Scheduled decode batch: {num_seqs_decode} reqs, {total_tokens_num_decode} tokens" + # ) + return ScheduledBatch( seqs=scheduled_seqs, - is_prefill=False, - deferred_reqID=self.deferred_reqID, - unfinished_prev_req=scheduled_seqs, - num_scheduled_tokens=num_scheduled_tokens, - total_num_scheduled_tokens=total_num_scheduled_tokens, + num_scheduled_tokens=num_scheduled_tokens_np, + total_tokens_num=total_tokens_num_decode, + total_tokens_num_decode=total_tokens_num_decode, + total_seqs_num=num_seqs_prefill + num_seqs_decode, + total_seqs_num_prefill=num_seqs_prefill, + total_seqs_num_decode=num_seqs_decode, ) def preempt(self, seq: Sequence): @@ -207,40 +149,40 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): - - # # placeholder for the each decode step - token_ids = [self.eos_token_id] * len(seqs) - for seq, token_id in zip(seqs, token_ids): - seq.append_token(token_id) - - if not prev_token_ids: - return - + def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): + is_deferred_out = True # update token_ids with the actual sampled token ids - self.unfinished_req = [] - token_ids = prev_token_ids - for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): - # seq.append_token(token_id) - seq.token_ids[-2] = token_id + finished_seqs = [] + for seq in self.running: + if seq.id not in prev_token_ids: + continue + token_id = prev_token_ids[seq.id] + if is_deferred_out: + seq.token_ids[-1] = token_id + else: + seq.append_token(token_id) + leave_reason = None - # Check if sequence ends with any stop sequence - stop_matched = False for stop_seq in seq.stop_token_sequences: if len(seq.token_ids) >= len(stop_seq): - if seq.token_ids[-len(stop_seq):] == stop_seq: - stop_matched = True + if seq.token_ids[-len(stop_seq) :] == stop_seq: + leave_reason = "stop_sequence" break - - if (not seq.ignore_eos and token_id == self.eos_token_id) or stop_matched: - leave_reason = "eos" - elif seq.num_completion_tokens == seq.max_tokens: - leave_reason = "max_tokens" + else: + if not seq.ignore_eos and token_id == self.eos_token_id: + leave_reason = "eos" + elif seq.num_completion_tokens == seq.max_tokens: + leave_reason = "max_tokens" if leave_reason is not None: seq.leave_reason = leave_reason seq.status = SequenceStatus.FINISHED - self.block_manager.deallocate(seq) - self.running.remove(seq) - else: - self.unfinished_req.append(i) + finished_seqs.append(seq) + if is_deferred_out: + # placeholder for the each decode step + for seq in seqs: + if seq.status == SequenceStatus.RUNNING: + seq.append_token(self.eos_token_id) + for seq in finished_seqs: + self.block_manager.deallocate(seq) + self.running.remove(seq) diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index da5abdcc0b..64ab59b5ab 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -12,6 +12,12 @@ class SequenceStatus(Enum): EXIT_ENGINE = auto() +class SequenceType(Enum): + DUMMY = auto() + PREFILL = auto() + DECODE = auto() + + def get_exit_sequence(): exit_seq = Sequence([-1], 1) exit_seq.status = SequenceStatus.EXIT_ENGINE @@ -27,6 +33,7 @@ def __init__( self.block_size = block_size self.id = next(Sequence.counter) self.status = SequenceStatus.WAITING + self.type = SequenceType.DUMMY self.token_ids = copy(token_ids) self.last_token = token_ids[-1] self.num_tokens = len(self.token_ids) @@ -52,6 +59,7 @@ def __getitem__(self, key): @property def num_tokens(self): + '''The total number of tokens in the sequence. i.e. prompt + completion''' return self._num_tokens @num_tokens.setter diff --git a/atom/utils/context.py b/atom/utils/context.py index 84e15c3089..2038ebcdaf 100644 --- a/atom/utils/context.py +++ b/atom/utils/context.py @@ -24,6 +24,7 @@ class Context: kv_indices: Optional[torch.Tensor] = None kv_last_page_lens: Optional[torch.Tensor] = None + _CONTEXT = Context() From 0659952eb105a0c465f4b19921d28bba8cf28e36 Mon Sep 17 00:00:00 2001 From: jiayyu Date: Tue, 14 Oct 2025 16:16:58 +0800 Subject: [PATCH 074/119] support torch compile --- atom/model_engine/model_runner.py | 7 + atom/model_ops/attention_mla.py | 638 ++++++++++++++++++++++-------- atom/model_ops/topK.py | 24 +- atom/models/deepseek_v2.py | 153 +------ 4 files changed, 487 insertions(+), 335 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index d754c80c59..dd3e8b6d91 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -334,6 +334,13 @@ def allocate_kv_cache(self, num_kvcache_blocks): 576, ) module.max_model_len = self.config.max_model_len + attention_metadata = AttentionMetadata( + k_cache=module.kv_cache, + v_cache=None, + k_scale=None, + v_scale=None, + ) + set_forward_context(module.layer_num, attention_metadata) layer_id += 1 return True diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index a064cfbb25..78a9f4ca80 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -1,25 +1,31 @@ import torch from torch import nn from atom.utils.context import get_context, Context +from atom.utils.custom_register import direct_register_custom_op +from atom.utils.forward_context import ForwardContext, get_forward_context from atom.model_ops.linear import ColumnParallelLinear, RowParallelLinear from atom.model_ops.utils import get_and_maybe_dequant_weights from aiter.rotary_embedding import RotaryEmbedding from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip - batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as aiter_triton_fp8_bmm, + batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as _aiter_triton_fp8_bmm, +) +from aiter import ( + flash_attn_varlen_func, + concat_and_cache_mla, + dtypes, + QuantType, + gemm_a8w8_blockscale, + get_hip_quant, ) -from aiter import flash_attn_varlen_func, concat_and_cache_mla, dtypes from aiter.mla import mla_decode_fwd -from typing import Optional +from typing import Optional, Tuple from tqdm import tqdm import logging -from aiter.dist.parallel_state import ( - get_pp_group, - get_tp_group, - get_tensor_model_parallel_world_size, -) +from aiter.dist.parallel_state import get_tp_group + logger = logging.getLogger("atom") @@ -35,6 +41,440 @@ def dynamic_per_batched_tensor_quant( return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal() +def aiter_triton_fp8_bmm( + X: torch.Tensor, + WQ: torch.Tensor, + w_scale: torch.Tensor, + group_size: int = 128, + transpose_bm: Optional[bool] = False, +) -> torch.Tensor: + return _aiter_triton_fp8_bmm(X, WQ, w_scale, group_size=group_size, transpose_bm=transpose_bm) + + +def fake_bmm( + X: torch.Tensor, + WQ: torch.Tensor, + w_scale: torch.Tensor, + group_size: int = 128, + transpose_bm: Optional[bool] = False, +) -> torch.Tensor: + return torch.zeros([X.shape[0], X.shape[1], WQ.shape[1]], dtype=X.dtype, device=X.device) + + +direct_register_custom_op( + op_name="aiter_triton_fp8_bmm", + op_func=aiter_triton_fp8_bmm, + mutates_args=[], + fake_impl=fake_bmm, +) + + +def aiter_mla_decode_fwd( + q: torch.Tensor, + kv_buffer: torch.Tensor, + o: torch.Tensor, + qo_indptr: torch.Tensor, + max_seqlen_qo: int, + kv_indptr: Optional[torch.Tensor] = None, + kv_indices: Optional[torch.Tensor] = None, + kv_last_page_lens: Optional[torch.Tensor] = None, + sm_scale: float = 1.0, + logit_cap: float = 0.0, +) -> None: + mla_decode_fwd( + q, + kv_buffer.view(-1, 1, 1, q.shape[-1]), + o, + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_lens, + max_seqlen_qo, + sm_scale=sm_scale, + logit_cap=logit_cap, + ) + + +def _forward_rope( + positions: torch.Tensor, + # if is_nope_first + # [[batch_size, seq_len, num_heads, nope_size+rope_size] + # if NOT is_nope_first + # [[batch_size, seq_len, num_heads, rope_size+nope_size], + query: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, + head_size: int, + rotary_dim: int, + key: Optional[torch.Tensor] = None, + offsets: Optional[torch.Tensor] = None, + is_nope_first=False, +) -> Tuple[torch.Tensor, torch.Tensor]: + import aiter + + assert ( + cos.dtype == query.dtype + ), f"cos_cache dtype ({cos.dtype}) does not match query dtype ({query.dtype})" + + rotate_style = 0 if is_neox_style else 1 + + num_tokens = positions.numel() + + query_shape = query.shape + query = query.view(1, num_tokens, -1, head_size) + if key is not None: + key_shape = key.shape + key = key.view(1, num_tokens, -1, head_size) + + positions = positions.view(*query.shape[:2]) + if offsets is not None: + offsets = offsets.view(*query.shape[:2]) + + if not is_nope_first: + query_ = query[..., : rotary_dim] + key_ = key[..., : rotary_dim] if key is not None else None + else: + query_ = query[..., -rotary_dim:] + key_ = key[..., -rotary_dim:] if key is not None else None + + if key_ is not None: + if offsets is None: + aiter.rope_cached_positions_2c_fwd_inplace( + query_, + key_, + cos, + sin, + positions, + rotate_style, + reuse_freqs_front_part=True, + nope_first=is_nope_first, + ) + else: + aiter.rope_cached_positions_offsets_2c_fwd_inplace( + query_, + key_, + cos, + sin, + positions, + offsets, + rotate_style, + reuse_freqs_front_part=True, + nope_first=is_nope_first, + ) + return query.view(query_shape), key.view(key_shape) + else: + if offsets is None: + aiter.rope_cached_positions_fwd_inplace( + query_, + cos, + sin, + positions, + rotate_style, + reuse_freqs_front_part=True, + nope_first=is_nope_first, + ) + else: + aiter.rope_cached_positions_offsets_fwd_inplace( + query_, + cos, + sin, + positions, + offsets, + rotate_style, + reuse_freqs_front_part=True, + nope_first=is_nope_first, + ) + return query.view(query_shape) + + +def _q_proj_and_k_up_proj(x, q_proj_weight_data, q_proj_weight_scale_data, q_proj_tp_dim, q_proj_tp_size, q_proj_reduce_results, W_K, W_K_scale, num_heads, qk_head_dim, qk_nope_head_dim, qk_rope_head_dim): + q_nope, q_pe = ( + torch.split(_forward_linear_fp8_no_bias(x, q_proj_weight_data, q_proj_weight_scale_data, q_proj_tp_dim, q_proj_tp_size, q_proj_reduce_results) + .view(-1, num_heads, qk_head_dim), + [qk_nope_head_dim, qk_rope_head_dim], dim=-1) + ) + + # Convert from (B, N, P) to (N, B, P) + q_nope = q_nope.transpose(0, 1) + # Multiply (N, B, P) x (N, P, L) -> (N, B, L), Convert from (N, B, L) to (B, N, L) + # ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) + ql_nope = torch.ops.aiter.aiter_triton_fp8_bmm( + q_nope, W_K, W_K_scale, group_size=128, transpose_bm=True + ) + return ql_nope, q_pe + + +def _forward_prefill( + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + kv_b_proj_weight_data, + kv_b_proj_weight_scale_data, + kv_b_proj_tp_dim, + kv_b_proj_tp_size, + kv_b_proj_reduce_results, + o_proj_weight_data, + o_proj_weight_scale_data, + o_proj_tp_dim, + o_proj_tp_size, + o_proj_reduce_results, + num_heads, + qk_nope_head_dim, + v_head_dim, + scale, + context: Context, +) -> torch.Tensor: + assert context is not None + + kv_nope = _forward_linear_fp8_no_bias(kv_c_normed, kv_b_proj_weight_data, kv_b_proj_weight_scale_data, kv_b_proj_tp_dim, kv_b_proj_tp_size, kv_b_proj_reduce_results).view( + -1, num_heads, qk_nope_head_dim + v_head_dim + ) + k_nope, v = torch.split(kv_nope, [qk_nope_head_dim, v_head_dim], dim=-1) + + if k_pe.dim() == 2: + k_pe = k_pe.unsqueeze(1) + k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1) + + output = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=context.cu_seqlens_q, + cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q=context.min_seqlen_q, + dropout_p=context.dropout_p, + softmax_scale=scale, + causal=True, + ) + + # return o_proj(output.flatten(start_dim=-2)) + return _forward_linear_fp8_no_bias(output.flatten(start_dim=-2), o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results) + + +def _v_up_proj_and_o_proj(x, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, W_V, W_V_scale, num_heads, kv_lora_rank, v_head_dim): + # Convert from (B, N, L) to (N, B, L) + x = x.view(-1, num_heads, kv_lora_rank).transpose(0, 1) + # Multiply (N, B, L) x (N, L, V) -> (N, B, V), Convert from (N, B, V) to (B, N, V) + # x = torch.bmm(x, self.W_UV).transpose(0, 1) + x = torch.ops.aiter.aiter_triton_fp8_bmm( + x, W_V, W_V_scale, group_size=128, transpose_bm=True + ) + # Convert from (B, N, V) to (B, N * V) + x = x.reshape(-1, num_heads * v_head_dim) + # return o_proj(x) + return _forward_linear_fp8_no_bias(x, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results) + + +def _forward_decode( + q_nope: torch.Tensor, + q_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + o_proj_weight_data, + o_proj_weight_scale_data, + o_proj_tp_dim, + o_proj_tp_size, + o_proj_reduce_results, + W_V, + W_V_scale, + num_heads, + kv_lora_rank, + v_head_dim, + scale, + context: Context, +) -> torch.Tensor: + assert kv_c_and_k_pe_cache.numel() > 0 + assert context is not None + B = q_nope.shape[0] + + q = torch.cat([q_nope, q_pe], dim=-1) + o = torch.empty( + B, num_heads, kv_lora_rank, dtype=q.dtype, device=q.device + ) + + kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2) + + aiter_mla_decode_fwd( + q, + kv_buffer, + o, + context.cu_seqlens_q, + context.max_q_len, + context.kv_indptr, + context.kv_indices, + context.kv_last_page_lens, + scale, + ) + + return _v_up_proj_and_o_proj(o, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, W_V, W_V_scale, num_heads, kv_lora_rank, v_head_dim) + + +def _forward_linear_fp8_no_bias(x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, + tp_dim: int, tp_size: int, reduce_results: bool + ) -> torch.Tensor: + quant_type = QuantType.per_1x128 + quant_func = get_hip_quant(quant_type) + x, x_scale = quant_func( + x, + quant_dtype=torch.float8_e4m3fnuz, + scale=None, + ) + y = gemm_a8w8_blockscale( + x, weight, x_scale, weight_scale, dtype=dtypes.bf16 + ) + if tp_dim == 1 and tp_size > 1 and reduce_results: + y = get_tp_group().all_reduce(y, ca_fp8_quant=False) + return y + + +def mla_attention( + q: torch.Tensor, # query in unified attn + k_c_normed: torch.Tensor, # key in unified attn + k_pe: torch.Tensor, # value in unified attn + positions: torch.Tensor, + W_K: torch.Tensor, + W_V: torch.Tensor, + W_K_scale: torch.Tensor, + W_V_scale: torch.Tensor, + rotary_emb_cos_cache: torch.Tensor, + rotary_emb_sin_cache: torch.Tensor, + rotary_emb_is_neox_style: bool, + rotary_emb_head_size: int, + rotary_emb_rotary_dim: int, + q_proj_weight_data: torch.Tensor, + q_proj_weight_scale_data: torch.Tensor, + q_proj_tp_dim: int, + q_proj_tp_size: int, + q_proj_reduce_results: bool, + kv_b_proj_weight_data: torch.Tensor, + kv_b_proj_weight_scale_data: torch.Tensor, + kv_b_proj_tp_dim: int, + kv_b_proj_tp_size: int, + kv_b_proj_reduce_results: bool, + o_proj_weight_data: torch.Tensor, + o_proj_weight_scale_data: torch.Tensor, + o_proj_tp_dim: int, + o_proj_tp_size: int, + o_proj_reduce_results: bool, + kv_lora_rank: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + qk_head_dim: int, + v_head_dim: int, + kv_cache_dtype: str, + scale: float, + layer_num: int +) -> torch.Tensor: + context = get_context() + if context.slot_mapping.numel(): + forward_context: ForwardContext = get_forward_context() + attn_metadata_ = forward_context.no_compile_layers[layer_num] + kv_cache = attn_metadata_.k_cache + else: + # dummy run before allocate kv_cache, thus we create manually + kv_cache = torch.tensor([]) + + _k_scale = torch.tensor(1.0, dtype=torch.float32) + + if context.is_prefill: + prefill_q = _forward_linear_fp8_no_bias(q, q_proj_weight_data, q_proj_weight_scale_data, + q_proj_tp_dim, q_proj_tp_size, q_proj_reduce_results).view(-1, num_heads, qk_head_dim) + prefill_q_pe = prefill_q[..., qk_nope_head_dim:] + _forward_rope(positions, prefill_q_pe, rotary_emb_cos_cache, rotary_emb_sin_cache, + rotary_emb_is_neox_style, rotary_emb_head_size, rotary_emb_rotary_dim, k_pe) + + if kv_cache.numel() > 0: + concat_and_cache_mla( + k_c_normed, + k_pe.squeeze(1), + kv_cache, + context.slot_mapping.flatten(), + kv_cache_dtype=kv_cache_dtype, + scale=_k_scale, + ) + + output = _forward_prefill( + prefill_q, k_c_normed, k_pe, kv_cache, kv_b_proj_weight_data, kv_b_proj_weight_scale_data, kv_b_proj_tp_dim, kv_b_proj_tp_size, kv_b_proj_reduce_results, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, num_heads, qk_nope_head_dim, v_head_dim, scale, context + ) + else: + decode_ql_nope, decode_q_pe = _q_proj_and_k_up_proj(q, q_proj_weight_data, q_proj_weight_scale_data, q_proj_tp_dim, + q_proj_tp_size, q_proj_reduce_results, W_K, W_K_scale, num_heads, qk_head_dim, qk_nope_head_dim, qk_rope_head_dim) + _forward_rope(positions, decode_q_pe, rotary_emb_cos_cache, rotary_emb_sin_cache, + rotary_emb_is_neox_style, rotary_emb_head_size, rotary_emb_rotary_dim, k_pe) + + if kv_cache.numel() > 0: + concat_and_cache_mla( + k_c_normed, + k_pe.squeeze(1), + kv_cache, + context.slot_mapping.flatten(), + kv_cache_dtype=kv_cache_dtype, + scale=_k_scale, + ) + + output = _forward_decode( + decode_ql_nope, decode_q_pe, kv_cache, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, W_V, W_V_scale, num_heads, kv_lora_rank, v_head_dim, scale, context + ) + + return output + + +def fake_mla_attention( + q: torch.Tensor, # query in unified attn + k_c_normed: torch.Tensor, # key in unified attn + k_pe: torch.Tensor, # value in unified attn + positions: torch.Tensor, + W_K: torch.Tensor, + W_V: torch.Tensor, + W_K_scale: torch.Tensor, + W_V_scale: torch.Tensor, + rotary_emb_cos_cache: torch.Tensor, + rotary_emb_sin_cache: torch.Tensor, + rotary_emb_is_neox_style: bool, + rotary_emb_head_size: int, + rotary_emb_rotary_dim: int, + q_proj_weight_data: torch.Tensor, + q_proj_weight_scale_data: torch.Tensor, + q_proj_tp_dim: int, + q_proj_tp_size: int, + q_proj_reduce_results: bool, + kv_b_proj_weight_data: torch.Tensor, + kv_b_proj_weight_scale_data: torch.Tensor, + kv_b_proj_tp_dim: int, + kv_b_proj_tp_size: int, + kv_b_proj_reduce_results: bool, + o_proj_weight_data: torch.Tensor, + o_proj_weight_scale_data: torch.Tensor, + o_proj_tp_dim: int, + o_proj_tp_size: int, + o_proj_reduce_results: bool, + kv_lora_rank: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + qk_head_dim: int, + v_head_dim: int, + kv_cache_dtype: str, + scale: float, + layer_num: int +) -> torch.Tensor: + output_shape = list(q.shape) + output_shape[-1] = 7168 # TODO: read from hidden_size + return torch.zeros(output_shape, dtype=q.dtype, device=q.device) + + +direct_register_custom_op( + op_name="mla_attention", + op_func=mla_attention, + mutates_args=[], + fake_impl=fake_mla_attention, +) + + class MLAAttention(nn.Module): def __init__( self, @@ -54,6 +494,7 @@ def __init__( q_proj: ColumnParallelLinear, kv_b_proj: ColumnParallelLinear, o_proj: RowParallelLinear, + layer_num: int = 0, ) -> None: super().__init__() self.num_heads = num_heads @@ -74,6 +515,7 @@ def __init__( self.kv_b_proj = kv_b_proj self.kv_cache = torch.tensor([]) self._k_scale = torch.tensor(1.0, dtype=torch.float32) + self.layer_num = layer_num def process_weights_after_loading(self): kv_b_proj_weight = get_and_maybe_dequant_weights(self.kv_b_proj).T @@ -92,9 +534,9 @@ def process_weights_after_loading(self): self.num_heads, self.qk_nope_head_dim + self.v_head_dim, ) - W_UK, W_UV = kv_b_proj_weight.split( - [self.qk_nope_head_dim, self.v_head_dim], dim=-1 - ) + W_UK, W_UV = torch.split(kv_b_proj_weight, + [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + ) if True: # is_rocm_aiter_fp8bmm_enabled(): W_K = W_UK.transpose(0, 1) # 16 512 128 @@ -125,7 +567,7 @@ def process_weights_after_loading(self): dtype=torch.bfloat16, device=self.W_K.device, ) - aiter_triton_fp8_bmm( + torch.ops.aiter.aiter_triton_fp8_bmm( x, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True ) @@ -134,7 +576,7 @@ def process_weights_after_loading(self): dtype=torch.bfloat16, device=self.W_V.device, ) - aiter_triton_fp8_bmm( + torch.ops.aiter.aiter_triton_fp8_bmm( x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True ) else: @@ -143,127 +585,6 @@ def process_weights_after_loading(self): # Convert from (L, N, P) to (N, P, L) self.W_UK_T = W_UK.permute(1, 2, 0) - def _aiter_mla_decode_fwd( - self, - q: torch.Tensor, - kv_buffer: torch.Tensor, - o: torch.Tensor, - qo_indptr: torch.Tensor, - max_seqlen_qo: int, - kv_indptr: Optional[torch.Tensor] = None, - kv_indices: Optional[torch.Tensor] = None, - kv_last_page_lens: Optional[torch.Tensor] = None, - sm_scale: float = 1.0, - logit_cap: float = 0.0, - ) -> None: - mla_decode_fwd( - q, - kv_buffer.view(-1, 1, 1, q.shape[-1]), - o, - qo_indptr, - kv_indptr, - kv_indices, - kv_last_page_lens, - max_seqlen_qo, - sm_scale=sm_scale, - logit_cap=logit_cap, - ) - - def _v_up_proj_and_o_proj(self, x): - # Convert from (B, N, L) to (N, B, L) - x = x.view(-1, self.num_heads, self.kv_lora_rank).transpose(0, 1) - # Multiply (N, B, L) x (N, L, V) -> (N, B, V), Convert from (N, B, V) to (B, N, V) - # x = torch.bmm(x, self.W_UV).transpose(0, 1) - x = aiter_triton_fp8_bmm( - x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True - ) - # Convert from (B, N, V) to (B, N * V) - x = x.reshape(-1, self.num_heads * self.v_head_dim) - return self.o_proj(x) - - def _q_proj_and_k_up_proj(self, x): - q_nope, q_pe = ( - self.q_proj(x) - .view(-1, self.num_heads, self.qk_head_dim) - .split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - ) - - # Convert from (B, N, P) to (N, B, P) - q_nope = q_nope.transpose(0, 1) - # Multiply (N, B, P) x (N, P, L) -> (N, B, L), Convert from (N, B, L) to (B, N, L) - # ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) - ql_nope = aiter_triton_fp8_bmm( - q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True - ) - return ql_nope, q_pe - - def _forward_prefill( - self, - q: torch.Tensor, - kv_c_normed: torch.Tensor, - k_pe: torch.Tensor, - kv_c_and_k_pe_cache: torch.Tensor, - context: Context, - ) -> torch.Tensor: - assert context is not None - - kv_nope = self.kv_b_proj(kv_c_normed).view( - -1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim - ) - k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) - - if k_pe.dim() == 2: - k_pe = k_pe.unsqueeze(1) - k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1) - - output = flash_attn_varlen_func( - q=q, - k=k, - v=v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q=context.min_seqlen_q, - dropout_p=context.dropout_p, - softmax_scale=self.scale, - causal=True, - ) - - return self.o_proj(output.flatten(start_dim=-2)) - - def _forward_decode( - self, - q_nope: torch.Tensor, - q_pe: torch.Tensor, - kv_c_and_k_pe_cache: torch.Tensor, - context: Context, - ) -> torch.Tensor: - assert kv_c_and_k_pe_cache.numel() > 0 - assert context is not None - B = q_nope.shape[0] - - q = torch.cat([q_nope, q_pe], dim=-1) - o = torch.empty( - B, self.num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device - ) - - kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2) - - self._aiter_mla_decode_fwd( - q, - kv_buffer, - o, - context.cu_seqlens_q, - context.max_q_len, - context.kv_indptr, - context.kv_indices, - context.kv_last_page_lens, - self.scale, - ) - - return self._v_up_proj_and_o_proj(o) - def forward( self, q: torch.Tensor, # query in unified attn @@ -271,44 +592,11 @@ def forward( k_pe: torch.Tensor, # value in unified attn positions: torch.Tensor, ) -> torch.Tensor: - # TODO: kv cache and context - kv_cache = self.kv_cache - context = get_context() - - if context.is_prefill: - prefill_q = self.q_proj(q).view(-1, self.num_heads, self.qk_head_dim) - prefill_q_pe = prefill_q[..., self.qk_nope_head_dim :] - self.rotary_emb(positions, prefill_q_pe, k_pe) - - if kv_cache.numel() > 0: - concat_and_cache_mla( - k_c_normed, - k_pe.squeeze(1), - kv_cache, - context.slot_mapping.flatten(), - kv_cache_dtype=self.kv_cache_dtype, - scale=self._k_scale, - ) - - output = self._forward_prefill( - prefill_q, k_c_normed, k_pe, kv_cache, context - ) - else: - decode_ql_nope, decode_q_pe = self._q_proj_and_k_up_proj(q) - self.rotary_emb(positions, decode_q_pe, k_pe) - - if kv_cache.numel() > 0: - concat_and_cache_mla( - k_c_normed, - k_pe.squeeze(1), - kv_cache, - context.slot_mapping.flatten(), - kv_cache_dtype=self.kv_cache_dtype, - scale=self._k_scale, - ) - - output = self._forward_decode( - decode_ql_nope, decode_q_pe, kv_cache, context - ) - - return output + return torch.ops.aiter.mla_attention(q, k_c_normed, k_pe, positions, self.W_K, self.W_V, self.W_K_scale, self.W_V_scale, + self.rotary_emb.cos_cache, self.rotary_emb.sin_cache, self.rotary_emb.is_neox_style, self.rotary_emb.head_size, self.rotary_emb.rotary_dim, + self.q_proj.weight.data, self.q_proj.weight_scale.data, self.q_proj.tp_dim, self.q_proj.tp_size, self.q_proj.reduce_results, + self.kv_b_proj.weight.data, self.kv_b_proj.weight_scale.data, self.kv_b_proj.tp_dim, self.kv_b_proj.tp_size, self.kv_b_proj.reduce_results, + self.o_proj.weight.data, self.o_proj.weight_scale.data, self.o_proj.tp_dim, self.o_proj.tp_size, self.o_proj.reduce_results, + self.kv_lora_rank, self.num_heads, + self.qk_nope_head_dim, self.qk_rope_head_dim, self.qk_head_dim, self.v_head_dim, self.kv_cache_dtype, + self.scale, self.layer_num) diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index 64d76048ec..fcb2bef63e 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -33,7 +33,7 @@ def init_aiter_topK_meta_data( dtype=torch.int32, device="cuda", ) - ns_topk_ids, s_topk_ids = total_topk_ids.split( + ns_topk_ids, s_topk_ids = torch.split(total_topk_ids, [top_k, n_shared_experts + is_EP], dim=1 ) shared_expert_ids = [n_routed_experts + i for i in range(n_shared_experts + is_EP)] @@ -52,7 +52,7 @@ def init_aiter_topK_meta_data( dtype=torch.float32, device="cuda", ) - ns_topk_weights, s_topk_weights = total_topk_weights.split( + ns_topk_weights, s_topk_weights = torch.split(total_topk_weights, [top_k, n_shared_experts + is_EP], dim=1 ) s_topk_weights.fill_(shared_experts_score) @@ -81,10 +81,10 @@ def rocm_aiter_topk_softmax_impl( ) total_topk_weights = total_topk_weights[:token] total_topk_ids = total_topk_ids[:token] - topk_weights, _ = total_topk_weights.split( + topk_weights, _ = torch.split(total_topk_weights, [topk, total_topk_weights.shape[1] - topk], dim=1 ) - topk_ids, _ = total_topk_ids.split( + topk_ids, _ = torch.split(total_topk_ids, [topk, total_topk_ids.shape[1] - topk], dim=1 ) else: @@ -141,10 +141,10 @@ def rocm_aiter_biased_grouped_topk_impl( ) total_topk_weights = total_topk_weights[:token] total_topk_ids = total_topk_ids[:token] - topk_weights, _ = total_topk_weights.split( + topk_weights, _ = torch.split(total_topk_weights, [topk, total_topk_weights.shape[1] - topk], dim=1 ) - topk_ids, _ = total_topk_ids.split( + topk_ids, _ = torch.split(total_topk_ids, [topk, total_topk_ids.shape[1] - topk], dim=1 ) else: @@ -199,10 +199,10 @@ def rocm_aiter_biased_grouped_topk_fake( ) total_topk_weights = total_topk_weights[:token] total_topk_ids = total_topk_ids[:token] - topk_weights, _ = total_topk_weights.split( + topk_weights, _ = torch.split(total_topk_weights, [topk, total_topk_weights.shape[1] - topk], dim=1 ) - topk_ids, _ = total_topk_ids.split( + topk_ids, _ = torch.split(total_topk_ids, [topk, total_topk_ids.shape[1] - topk], dim=1 ) else: @@ -242,10 +242,10 @@ def rocm_aiter_grouped_topk_impl( ) total_topk_weights = total_topk_weights[:token] total_topk_ids = total_topk_ids[:token] - topk_weights, _ = total_topk_weights.split( + topk_weights, _ = torch.split(total_topk_weights, [topk, total_topk_weights.shape[1] - topk], dim=1 ) - topk_ids, _ = total_topk_ids.split( + topk_ids, _ = torch.split(total_topk_ids, [topk, total_topk_ids.shape[1] - topk], dim=1 ) else: @@ -302,10 +302,10 @@ def rocm_aiter_grouped_topk_fake( ) total_topk_weights = total_topk_weights[:token] total_topk_ids = total_topk_ids[:token] - topk_weights, _ = total_topk_weights.split( + topk_weights, _ = torch.split(total_topk_weights, [topk, total_topk_weights.shape[1] - topk], dim=1 ) - topk_ids, _ = total_topk_ids.split( + topk_ids, _ = torch.split(total_topk_ids, [topk, total_topk_ids.shape[1] - topk], dim=1 ) else: diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index 41d8dcc241..b353e7cf51 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -189,152 +189,6 @@ def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: return 0.1 * mscale * math.log(scale) + 1.0 -class DeepseekV2Attention(nn.Module): - - def __init__( - self, - config: PretrainedConfig, - hidden_size: int, - num_heads: int, - qk_nope_head_dim: int, - qk_rope_head_dim: int, - v_head_dim: int, - q_lora_rank: int, - kv_lora_rank: int, - rope_theta: float = 10000, - rope_scaling: Optional[Dict[str, Any]] = None, - max_position_embeddings: int = 8192, - cache_config: str = "bf16", - quant_config: Optional[QuantizationConfig] = None, - prefix: str = "", - ) -> None: - super().__init__() - self.hidden_size = hidden_size - self.qk_nope_head_dim = qk_nope_head_dim - self.qk_rope_head_dim = qk_rope_head_dim - self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim - self.v_head_dim = v_head_dim - self.q_lora_rank = q_lora_rank - self.kv_lora_rank = kv_lora_rank - self.num_heads = num_heads - tp_size = get_tensor_model_parallel_world_size() - assert num_heads % tp_size == 0 - self.num_local_heads = num_heads // tp_size - self.scaling = self.qk_head_dim**-0.5 - self.rope_theta = rope_theta - self.max_position_embeddings = max_position_embeddings - - if self.q_lora_rank is not None: - self.q_a_proj = ReplicatedLinear(self.hidden_size, - self.q_lora_rank, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.q_a_proj") - self.q_a_layernorm = RMSNorm(self.q_lora_rank, - eps=config.rms_norm_eps) - self.q_b_proj = ColumnParallelLinear(q_lora_rank, - self.num_heads * - self.qk_head_dim, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.q_b_proj") - else: - self.q_proj = ColumnParallelLinear(self.hidden_size, - self.num_heads * - self.qk_head_dim, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.q_proj") - - self.kv_a_proj_with_mqa = ReplicatedLinear( - self.hidden_size, - self.kv_lora_rank + self.qk_rope_head_dim, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.kv_a_proj_with_mqa") - self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, - eps=config.rms_norm_eps) - self.kv_b_proj = ColumnParallelLinear( - self.kv_lora_rank, - self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.kv_b_proj") - # O projection. - self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim, - self.hidden_size, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.o_proj") - if rope_scaling: - rope_scaling["rope_type"] = 'deepseek_yarn' - - self.rotary_emb = get_rope(qk_rope_head_dim, - rotary_dim=qk_rope_head_dim, - max_position=max_position_embeddings, - base=rope_theta, - rope_scaling=rope_scaling, - is_neox_style=False) - - if rope_scaling: - mscale_all_dim = rope_scaling.get("mscale_all_dim", False) - scaling_factor = rope_scaling["factor"] - mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) - self.scaling = self.scaling * mscale * mscale - - self.attn = Attention(self.num_local_heads, - self.qk_head_dim, - self.scaling, - num_kv_heads=self.num_local_heads, - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.attn") - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - ) -> torch.Tensor: - if self.q_lora_rank is not None: - q = self.q_a_proj(hidden_states)[0] - q = self.q_a_layernorm(q) - q = self.q_b_proj(q)[0].view(-1, self.num_local_heads, - self.qk_head_dim) - else: - q = self.q_proj(hidden_states)[0].view(-1, self.num_local_heads, - self.qk_head_dim) - q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], - dim=-1) - latent_cache = self.kv_a_proj_with_mqa(hidden_states)[0] - kv_a, _ = latent_cache.split( - [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) - latent_cache = latent_cache.unsqueeze(1) - kv_a = self.kv_a_layernorm(kv_a.contiguous()) - kv = self.kv_b_proj(kv_a)[0] - kv = kv.view(-1, self.num_local_heads, - self.qk_nope_head_dim + self.v_head_dim) - k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) - k_pe = latent_cache[:, :, self.kv_lora_rank:] - - q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe) - - q[..., self.qk_nope_head_dim:] = q_pe - k = torch.empty_like(q) - k[..., :self.qk_nope_head_dim] = k_nope - k[..., self.qk_nope_head_dim:] = k_pe - # padding value to qk_head_dim for alignment - v = torch.nn.functional.pad( - v, [0, self.qk_head_dim - self.v_head_dim], - value=0).view(-1, self.num_local_heads * self.qk_head_dim) - attn_output = self.attn(q, k, v) - attn_output = attn_output.view( - -1, self.num_local_heads, - self.qk_head_dim)[..., :self.v_head_dim].reshape( - -1, self.num_local_heads * self.v_head_dim) - output, _ = self.o_proj(attn_output) - return output - - class DeepseekV2MLAAttention(nn.Module): """ Main reference: DeepseekV2 paper, and FlashInfer Implementation @@ -359,6 +213,7 @@ def __init__( cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + layer_num: int = 0, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -378,6 +233,7 @@ def __init__( self.scaling = self.qk_head_dim**-0.5 self.rope_theta = rope_theta self.max_position_embeddings = max_position_embeddings + self.layer_num = layer_num if self.q_lora_rank is not None: self.q_a_proj = ReplicatedLinear(self.hidden_size, @@ -458,10 +314,10 @@ def __init__( q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj, kv_b_proj=self.kv_b_proj, o_proj=self.o_proj, + layer_num=layer_num, ) self.prefix = prefix - self.debug_layer_idx = int(self.prefix.split(".")[-2]) def forward( self, @@ -473,7 +329,7 @@ def forward( hidden_states_or_q_c = self.q_a_layernorm(ckq) else: hidden_states_or_q_c = hidden_states - kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states).split( + kv_c, k_pe = torch.split(self.kv_a_proj_with_mqa(hidden_states), [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) kv_c_normed = self.kv_a_layernorm(kv_c) return self.mla_attn(hidden_states_or_q_c, @@ -519,6 +375,7 @@ def __init__( cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", + layer_num=layer_num, ) if (config.n_routed_experts is not None From 84c11644f2095107ae63e7a46bb6ad679774767b Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Tue, 21 Oct 2025 22:03:44 +0800 Subject: [PATCH 075/119] fix model_runner (#17) Co-authored-by: amd-ruitang3 --- atom/model_engine/model_runner.py | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 41c30a7c41..3df5a2bafd 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -597,16 +597,17 @@ def prepare_decode(self, batch: ScheduledBatch): self.kv_indptr, dtype=np.int64 ) var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[bs] - var["kv_last_page_lens"].np[:bs] = np.array( + var["kv_last_page_lens"].np[:scheduled_bs] = np.array( self.kv_last_page_lens, dtype=np.int64 ) + var["kv_last_page_lens"].np[scheduled_bs:bs] = 0 vars_used = [ ("slot_mapping", bs), # TODO: MTP support ("context_lens", bs), ("block_tables", bs), ("cu_seqlens_q", bs + 1), - ("kv_indices", bs), - ("kv_indptr", bs), + ("kv_indices", sum(context_lens)), + ("kv_indptr", bs + 1), ("kv_last_page_lens", bs), ] @@ -709,17 +710,6 @@ def forward(self, batch: ScheduledBatch) -> dict[int, int]: @torch.inference_mode() def capture_cudagraph(self): start_time = time.time() - input_ids = self.forward_vars["input_ids"].gpu - positions = self.forward_vars["positions"].gpu - slot_mapping = self.forward_vars["slot_mapping"].gpu - context_lens = self.forward_vars["context_lens"].gpu - block_tables = self.forward_vars["block_tables"].gpu - cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu - kv_indptr = self.forward_vars["kv_indptr"].gpu - kv_indices = self.forward_vars["kv_indices"].gpu - kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu - outputs = self.forward_vars["outputs"] - # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) if self.config.compilation_config.cudagraph_capture_sizes: self.graph_bs = self.config.compilation_config.cudagraph_capture_sizes @@ -733,6 +723,21 @@ def capture_cudagraph(self): self.graph_bs = cuda_graph_sizes self.graph_bs.sort(reverse=True) + self.forward_vars["cu_seqlens_q"].np[: self.graph_bs[0] + 1] = np.arange( + 0, self.graph_bs[0] + 1 + ) + self.forward_vars["cu_seqlens_q"].copy_to_gpu(self.graph_bs[0] + 1) + input_ids = self.forward_vars["input_ids"].gpu + positions = self.forward_vars["positions"].gpu + slot_mapping = self.forward_vars["slot_mapping"].gpu + context_lens = self.forward_vars["context_lens"].gpu + block_tables = self.forward_vars["block_tables"].gpu + cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu + kv_indptr = self.forward_vars["kv_indptr"].gpu + kv_indices = self.forward_vars["kv_indices"].gpu + kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu + outputs = self.forward_vars["outputs"] + self.graphs: dict[int, torch.cuda.CUDAGraph] = dict() self.graph_pool = None From b8fc20752a620370cb40643cb1265ebdef849d81 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 23 Oct 2025 17:50:08 +0800 Subject: [PATCH 076/119] support level 1 and 3 --- atom/model_ops/attention_mla.py | 26 +++++++++++- atom/model_ops/embed_head.py | 3 +- atom/model_ops/moe.py | 73 +++++++++++++++++++++++++++------ atom/model_ops/topK.py | 20 ++++++++- atom/models/deepseek_v2.py | 3 +- 5 files changed, 108 insertions(+), 17 deletions(-) diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index 78a9f4ca80..a683c9d818 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -19,6 +19,7 @@ ) from aiter.mla import mla_decode_fwd +from aiter.jit.utils.torch_guard import torch_compile_guard from typing import Optional, Tuple from tqdm import tqdm @@ -94,7 +95,30 @@ def aiter_mla_decode_fwd( logit_cap=logit_cap, ) +def _forward_rope_fake( + positions: torch.Tensor, + # if is_nope_first + # [[batch_size, seq_len, num_heads, nope_size+rope_size] + # if NOT is_nope_first + # [[batch_size, seq_len, num_heads, rope_size+nope_size], + query: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, + head_size: int, + rotary_dim: int, + key: Optional[torch.Tensor] = None, + offsets: Optional[torch.Tensor] = None, + is_nope_first: bool=False, +) -> Tuple[torch.Tensor, torch.Tensor]: + query_shape = query.shape + if key is not None: + key_shape = key.shape + return query.view(query_shape), key.view(key_shape) + else: + return query.view(query_shape) +@torch_compile_guard(gen_fake=_forward_rope_fake) def _forward_rope( positions: torch.Tensor, # if is_nope_first @@ -109,7 +133,7 @@ def _forward_rope( rotary_dim: int, key: Optional[torch.Tensor] = None, offsets: Optional[torch.Tensor] = None, - is_nope_first=False, + is_nope_first: bool=False, ) -> Tuple[torch.Tensor, torch.Tensor]: import aiter diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 1a6e359d3d..7b8534a63f 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -40,7 +40,8 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): def forward(self, x: torch.Tensor): if self.tp_size > 1: - mask = (x >= self.vocab_start_idx) & (x < self.vocab_end_idx) + mask = torch.logical_and(x >= self.vocab_start_idx, x < self.vocab_end_idx) + # mask = (x >= self.vocab_start_idx) & (x < self.vocab_end_idx) x = mask * (x - self.vocab_start_idx) y = F.embedding(x, self.weight) if self.tp_size > 1: diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 3559957d62..5bcfc9cbd6 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -24,6 +24,7 @@ per_tensor_dequantize, shuffle_weights, ) +from atom.utils.custom_register import direct_register_custom_op class FusedMoEMethodBase(QuantizeMethodBase): @@ -169,6 +170,56 @@ def apply( ) +def rocm_aiter_fused_moe_impl( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + expert_mask: Optional[torch.Tensor] = None, + activation: int = ActivationType.Silu.value, + quant_type: int = QuantType.No.value, + doweight_stage1: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, +) -> torch.Tensor: + from aiter import ActivationType, QuantType + from aiter.fused_moe import fused_moe + + activation_ = ActivationType(activation) + quant_type_ = QuantType(quant_type) + + return fused_moe(hidden_states, w1, w2, topk_weight, topk_ids, expert_mask, + activation_, quant_type_, doweight_stage1, w1_scale, + w2_scale, a1_scale, a2_scale) + + +def rocm_aiter_fused_moe_fake( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + expert_mask: Optional[torch.Tensor] = None, + activation: int = ActivationType.Silu.value, + quant_type: int = QuantType.No.value, + doweight_stage1: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, +) -> torch.Tensor: + return torch.empty_like(hidden_states) + +direct_register_custom_op( + op_name="rocm_aiter_fused_moe", + op_func=rocm_aiter_fused_moe_impl, + mutates_args=[], + fake_impl=rocm_aiter_fused_moe_fake, +) + class Fp8MoEMethod(FusedMoEMethodBase): """MoE method for FP8. Supports loading FP8 checkpoints with static weight scale and @@ -477,24 +528,22 @@ def apply( num_fused_shared_experts=layer.num_fused_shared_experts, routed_scaling_factor=layer.routed_scaling_factor, ) - return fused_moe( + + return torch.ops.aiter.rocm_aiter_fused_moe( x, layer.w13_weight, layer.w2_weight, - topk_weight=topk_weights, - topk_ids=topk_ids, + topk_weights, + topk_ids, expert_mask=expert_map, - activation=activation, - quant_type=self.quant_type, - w1_scale=( - layer.w13_weight_scale if self.block_quant else layer.w13_weight_scale - ), - w2_scale=( - layer.w2_weight_scale if self.block_quant else layer.w2_weight_scale - ), + activation=activation.value, + quant_type=self.quant_type.value, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, a1_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, - ) + doweight_stage1=apply_router_weight_on_input) + def determine_expert_map( diff --git a/atom/model_ops/topK.py b/atom/model_ops/topK.py index fcb2bef63e..b231412dde 100644 --- a/atom/model_ops/topK.py +++ b/atom/model_ops/topK.py @@ -2,6 +2,7 @@ from typing import Optional import torch +from atom.utils.custom_register import direct_register_custom_op def is_rocm_aiter_fusion_shared_expert_enabled(): @@ -326,6 +327,21 @@ def rocm_aiter_topk_softmax( gating_output, topk, renormalize, num_fused_shared_experts ) +direct_register_custom_op( + op_name="rocm_aiter_biased_grouped_topk_impl", + op_func=rocm_aiter_biased_grouped_topk_impl, + mutates_args=[], + fake_impl=rocm_aiter_biased_grouped_topk_fake, +) + +direct_register_custom_op( + op_name="rocm_aiter_grouped_topk_impl", + op_func=rocm_aiter_grouped_topk_impl, + mutates_args=[], + fake_impl=rocm_aiter_grouped_topk_fake, +) + + def rocm_aiter_grouped_topk( hidden_states: torch.Tensor, @@ -340,7 +356,7 @@ def rocm_aiter_grouped_topk( routed_scaling_factor: float = 1.0, ) -> tuple[torch.Tensor, torch.Tensor]: if e_score_correction_bias is not None: - return rocm_aiter_biased_grouped_topk_impl( + return torch.ops.aiter.rocm_aiter_biased_grouped_topk_impl( gating_output, e_score_correction_bias, num_expert_group, @@ -352,7 +368,7 @@ def rocm_aiter_grouped_topk( ) else: assert scoring_func == "softmax" or scoring_func == "sigmoid" - return rocm_aiter_grouped_topk_impl( + return torch.ops.aiter.rocm_aiter_grouped_topk_impl( gating_output, num_expert_group, topk_group, diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index b353e7cf51..5e00a1e052 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -55,6 +55,7 @@ is_rocm_aiter_fusion_shared_expert_enabled, is_rocm_aiter_fuse_routed_scaling_factor ) +from atom.utils.decorators import support_torch_compile class DeepseekV2MLP(nn.Module): @@ -444,7 +445,7 @@ def forward( return hidden_states, residual - +@support_torch_compile class DeepseekV2Model(nn.Module): def __init__( self, From 2fec5bf2c5c07dc0864a0e3f43814d6afd450342 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Thu, 23 Oct 2025 08:52:46 -0500 Subject: [PATCH 077/119] fix for gfx950 --- atom/model_ops/attention_mla.py | 2 +- atom/model_ops/moe.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index a683c9d818..f9f5800292 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -343,7 +343,7 @@ def _forward_linear_fp8_no_bias(x: torch.Tensor, weight: torch.Tensor, weight_sc quant_func = get_hip_quant(quant_type) x, x_scale = quant_func( x, - quant_dtype=torch.float8_e4m3fnuz, + quant_dtype=dtypes.fp8, scale=None, ) y = gemm_a8w8_blockscale( diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 5bcfc9cbd6..38fea53ea1 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -353,9 +353,8 @@ def create_weights( # If loading fp8 checkpoint, pass the weight loaders. # If loading an fp16 checkpoint, do not (we will quantize in # process_weights_after_loading() - if self.quant_config["quant_dtype"] == torch.float8_e4m3fnuz: - set_weight_attrs(w13_weight_scale, extra_weight_attrs) - set_weight_attrs(w2_weight_scale, extra_weight_attrs) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) # INPUT_SCALES if not self.quant_config["is_dynamic"]: From b57255cd3288f7124e11d1bc4b10d726efee7732 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 15 Oct 2025 14:05:24 +0800 Subject: [PATCH 078/119] draft deffer output --- atom/model_engine/engine_core.py | 5 +- atom/model_engine/model_runner.py | 330 ++++++++++++++++++++++++++++-- atom/model_engine/scheduler.py | 174 +++++++++++++++- 3 files changed, 475 insertions(+), 34 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index a4c7d0b68c..e3264b6080 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -138,8 +138,9 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() - out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) - self.scheduler.postprocess(scheduled_batchs.seqs, out) + out, prev_seqs = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) + # scheduled_batchs.prev_seqs = prev_seqs + self.scheduler.postprocess(scheduled_batchs.seqs, out, prev_seqs) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] ) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index dd3e8b6d91..52cda159e6 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -3,7 +3,9 @@ import signal import time import weakref -from typing import Optional +from typing import Optional, Union, Any +import queue +from collections import deque import numpy as np import torch @@ -13,7 +15,7 @@ from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture from atom.config import Config, set_current_atom_config -from atom.model_engine.scheduler import ScheduledBatchs +from atom.model_engine.scheduler import ScheduledBatchs, InputBatch from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler @@ -26,6 +28,7 @@ logger = logging.getLogger("atom") from atom.utils.forward_context import AttentionMetadata, set_forward_context +from atom.model_engine.outputs import ModelRunnerOutput, AsyncModelRunnerOutput suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -34,6 +37,54 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } +# Wrapper for ModelRunnerOutput to support overlapped execution. +class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): + + def __init__( + self, + # model_runner_output: ModelRunnerOutput, + sampled_token_ids: torch.Tensor, + # invalid_req_indices: list[int], + async_output_copy_stream: torch.cuda.Stream, + ): + # self._model_runner_output = model_runner_output + # self._invalid_req_indices = invalid_req_indices + + # Event on the copy stream so we can synchronize the non-blocking copy. + # self._async_copy_ready_event = torch.cuda.Event() + + # Keep a reference to the device tensor to avoid it being + # deallocated until we finish copying it to the host. + self._sampled_token_ids = sampled_token_ids + + # Initiate the copy on a separate stream, but do not synchronize it. + default_stream = torch.cuda.current_stream() + with torch.cuda.stream(async_output_copy_stream): + async_output_copy_stream.wait_stream(default_stream) + self._sampled_token_ids_cpu = self._sampled_token_ids.to( + 'cpu', non_blocking=True) + # self._async_copy_ready_event.record() + + def get_output(self) -> ModelRunnerOutput: + """Copy the device tensors to the host and return a ModelRunnerOutput. + + This function blocks until the copy is finished. + """ + # self._async_copy_ready_event.synchronize() + + # Release the device tensor once the copy has completed + del self._sampled_token_ids + + valid_sampled_token_ids = self._sampled_token_ids_cpu.tolist() + # for i in self._invalid_req_indices: + # valid_sampled_token_ids[i].clear() + + return valid_sampled_token_ids + # output = self._model_runner_output + # output.sampled_token_ids = valid_sampled_token_ids + # return output + + class OutputProcessor: @@ -47,34 +98,59 @@ def __init__(self): self.token_ids_gpu: list[torch.Tensor] = [] self.token_ids_cpu: list[torch.Tensor] = [] - def send_to_cpu_async(self, gpu_tensor: torch.Tensor): + self.token_req_ids: list[list[int]] = [] + + self.pending_outputs: deque = deque() # saving (cpu_tensor, seqs) + + + def send_to_cpu_async(self, batch: ScheduledBatchs, gpu_tensor: torch.Tensor): default_stream = torch.cuda.current_stream() with torch.cuda.stream(self.async_copy_stream): self.async_copy_stream.wait_stream(default_stream) cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) self.async_copy_event.record(self.async_copy_stream) - self.token_ids_gpu.append(gpu_tensor) + # self.token_ids_gpu.append(gpu_tensor) self.token_ids_cpu.append(cpu_tensor) + self.pending_outputs.append((cpu_tensor, batch.seqs)) + + def revc_async_output_tuple(self) -> tuple[list[int], list[Sequence]]: + if not self.pending_outputs: + return [], [] + + self.async_copy_event.synchronize() + if self.pending_outputs: + cpu_tensor, seqs = self.pending_outputs.popleft() + token_ids = cpu_tensor.tolist() + return token_ids, seqs + return [], [] def recv_async_output(self) -> list[int]: self.async_copy_event.synchronize() - for _ in self.token_ids_gpu: + for _ in self.token_ids_cpu: token_ids = self.token_ids_cpu.pop(0).tolist() return token_ids return [] + def clean_token(self): + self.token_ids_cpu: list[torch.Tensor] = [] + self.pending_outputs.clear() + def update_and_ret( self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor ) -> list[int]: - token_ids = self.recv_async_output() - self.send_to_cpu_async(sampled_token_ids) + # token_ids = self.recv_async_output() + prev_token_ids, prev_seqs = self.revc_async_output_tuple() + self.send_to_cpu_async(batch, sampled_token_ids) + # return token_ids + # return prev_token_ids, prev_seqs + deferred_reqIDs = [] returned_reqIDs = [] for req in batch.seqs: if req.id in self.deferred_request_id: returned_reqIDs.append(req.id) else: - deferred_reqIDs.append(req.id) + self.deferred_request_id.append(req.id) if returned_reqIDs: for ids in self.token_ids_cpu[:-1]: @@ -84,11 +160,9 @@ def update_and_ret( req_index = batch.unfinished_prev_req.index(req_id) batch.unfinished_prev_req.remove(req_id) req = batch.seqs[req_index] - req.append_token(ids_list[req_index]) - - self.async_copy_event.synchronize() - - return [] + req.append_token(prev_token_ids[req_index]) + return prev_token_ids, prev_seqs + # self.async_copy_event.synchronize() class ModelRunner: @@ -125,6 +199,22 @@ def __init__(self, rank: int, config: Config): torch.set_default_device("cuda") self.out_processor = OutputProcessor() self.sampler = Sampler() + + self.input_batch = InputBatch( + max_num_reqs=self.config.max_num_seqs, + max_model_len=self.config.max_model_len) + + self.input_ids = self._make_buffer(self.config.max_num_batched_tokens, + dtype=torch.int32) + self.positions = self._make_buffer(self.config.max_num_batched_tokens, + dtype=torch.int64) + self.use_async_scheduling = True + self.async_output_copy_stream = torch.cuda.Stream() if \ + self.use_async_scheduling else None + + self.out_processor = OutputProcessor() + + self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) torch.set_default_device(None) load_model(self.model, config.model, config.hf_config, config.load_dummy) @@ -138,6 +228,21 @@ def __init__(self, rank: int, config: Config): if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") + + def _make_buffer(self, + *size: Union[int, torch.SymInt], + dtype: torch.dtype, + numpy: bool = True) -> CpuGpuBuffer: + # Bfloat16 torch tensors cannot be directly cast to a numpy array, so + # if a bfloat16 buffer is needed without a corresponding numpy array, + # don't bother instantiating the numpy array. + return CpuGpuBuffer(*size, + dtype=dtype, + device=self.device, + pin_memory=True, + with_numpy=numpy) + + def exit(self): if not self.still_running: return @@ -184,8 +289,17 @@ def warmup_model(self): Sequence([0] * max_model_len, block_size=self.block_size) for _ in range(num_seqs) ] - dummy_batch = ScheduledBatchs(seqs, True, False) + + num_scheduled_tokens: dict[str, int] = {} + + for seq in seqs: + num_scheduled_tokens[seq.id] = seq.num_tokens + + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) + dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) self.forward(dummy_batch) + # self.out_processor.clean_token() torch.cuda.empty_cache() def allocate_decode_vars(self): @@ -351,6 +465,79 @@ def prepare_block_tables(self, seqs: list[Sequence]): block_tables[i, : lens[i]] = seq.block_table return block_tables + def _prepare_input_ids(self, total_num_scheduled_tokens: int, + cu_num_tokens: np.ndarray) -> None: + """Prepare the input IDs for the current batch. + + Carefully handles the `prev_sampled_token_ids` which can be cached + from the previous engine iteration, in which case those tokens on the + GPU need to be copied into the corresponding slots into input_ids.""" + + if self.input_batch.prev_sampled_token_ids is None: + # Normal scheduling case + self.input_ids.copy_to_gpu(total_num_scheduled_tokens) + return + + # Async scheduling case, where some decode requests from the previous + # iteration won't have entries in input_ids_cpu and need to be copied + # on the GPU from prev_sampled_token_ids. + prev_req_id_to_index = self.input_batch.prev_req_id_to_index + assert prev_req_id_to_index is not None + flattened_indices = [] + prev_common_req_indices = [] + indices_match = True + max_flattened_index = -1 + for req_id, cur_index in self.input_batch.req_id_to_index.items(): + if (prev_index := prev_req_id_to_index.get(req_id)) is not None: + prev_common_req_indices.append(prev_index) + # We need to compute the flattened input_ids index of the + # last token in each common request. + flattened_index = cu_num_tokens[cur_index].item() - 1 + # flattened_indices.append(flattened_index) + flattened_indices.append(prev_index) + indices_match &= (prev_index == flattened_index) + max_flattened_index = max(max_flattened_index, flattened_index) + num_commmon_tokens = len(flattened_indices) + if num_commmon_tokens < total_num_scheduled_tokens: + # If not all requests are decodes from the last iteration, + # We need to copy the input_ids_cpu to the GPU first. + self.input_ids.copy_to_gpu(total_num_scheduled_tokens) + if num_commmon_tokens == 0: + # No requests in common with the previous iteration + # So input_ids_cpu will have all the input ids. + return + if indices_match and max_flattened_index == (num_commmon_tokens - 1): + # Common-case optimization: the batch is unchanged + # and no reordering happened. + # The indices are both the same permutation of 0..N-1 so + # we can copy directly using a single slice. + self.input_ids.gpu[:num_commmon_tokens].copy_( + self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, + 0], + non_blocking=True) + return + # Upload the index tensors asynchronously + # so the scatter can be non-blocking. + input_ids_index_tensor = torch.tensor(flattened_indices, + dtype=torch.int64, + pin_memory=True).to( + self.device, + non_blocking=True) + prev_common_req_indices_tensor = torch.tensor( + prev_common_req_indices, + dtype=torch.int64, + pin_memory=True).to(self.device, non_blocking=True) + # print('self.input_batch.prev_sampled_token_ids', self.input_batch.prev_sampled_token_ids) + self.input_ids.gpu.scatter_( + dim=0, + index=input_ids_index_tensor, + src=self.input_batch.prev_sampled_token_ids[ + prev_common_req_indices_tensor]) + # prev_common_req_indices_tensor, 0]) + + self.positions.copy_to_gpu(total_num_scheduled_tokens) + + def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): input_ids = [] positions = [] @@ -491,7 +678,11 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) - + num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens + num_input_tokens = num_scheduled_tokens + input_ids = self.input_ids.gpu[:num_input_tokens] + positions = self.positions.gpu[:num_input_tokens] + return input_ids, positions return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): @@ -521,7 +712,47 @@ def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: ).cuda(non_blocking=True) return temperatures + # def _get_cumsum_and_arange( + # self, + # num_tokens: np.ndarray, + # cumsum_dtype: Optional[np.dtype] = None, + # ) -> tuple[np.ndarray, np.ndarray]: + # """Get the cumulative sum and batched arange of the given array. + # # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) + # # Equivalent to but faster than: + # # np.concatenate([np.arange(n) for n in num_tokens]) + # """ + # # Step 1. [2, 5, 3] -> [2, 7, 10] + # cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) + # total_num_tokens = cu_num_tokens[-1] + # # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] + # cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) + # # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + # arange = self.arange_np[:total_num_tokens] - cumsums_offsets + + # return cu_num_tokens, arange + + def prepare_model(self, scheduled_batchs: ScheduledBatchs): + self._update_states(scheduled_batchs) + + total_num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens + assert total_num_scheduled_tokens > 0 + + _req_ids = self.input_batch.req_ids + req_ids = [req_id for req_id in _req_ids if req_id is not None] + + # print("scheduled_batchs.num_scheduled_tokens", scheduled_batchs.num_scheduled_tokens) + # print("req_ids", req_ids) + tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] + num_scheduled_tokens = np.array(tokens, dtype=np.int32) + # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] + # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + # cu_num_tokens, arange = self._get_cumsum_and_arange( + # num_scheduled_tokens) + cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) + + self._prepare_input_ids(total_num_scheduled_tokens, cu_num_tokens) seqs = scheduled_batchs.seqs is_prefill = scheduled_batchs.is_prefill prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode @@ -550,20 +781,75 @@ def postprocess( ) -> list[int]: if self.rank == 0: sampled_tokens = self.sampler(logits, temperatures) - token_ids = sampled_tokens.tolist() - # token_ids = self.prev_sampled.update_and_ret( - # batch, - # sampled_tokens, + + # num_sampled_tokens = sampled_tokens.shape[0] + sampled_token_ids = sampled_tokens + if self.use_async_scheduling: + # assert sampled_token_ids.shape[-1] == 1 + + # Cache the sampled tokens on the GPU and avoid CPU sync. + # These will be copied into input_ids in the next step + # when preparing inputs. + # if self.input_batch.prev_sampled_token_ids is None: + # output = None + # else: + # output = self.input_batch.prev_sampled_token_ids + + self.input_batch.prev_sampled_token_ids = \ + sampled_token_ids + # print("self.input_batch.prev_sampled_token_ids in postprocess", self.input_batch.prev_sampled_token_ids) + self.input_batch.prev_req_id_to_index = { + req_id: i + for i, req_id in enumerate(self.input_batch.req_ids) + } + + + # output = AsyncGPUModelRunnerOutput( + # sampled_token_ids=sampled_token_ids, + # # invalid_req_indices=invalid_req_indices, + # async_output_copy_stream=self.async_output_copy_stream, # ) + + # .get_output() + # self.handle_output(output) + + # token_ids = sampled_tokens.tolist() + output, prev_seqs = self.out_processor.update_and_ret( + batch, + sampled_tokens, + ) + print("sampled_token_ids", sampled_token_ids) + print("prev_token_ids", output) + print('----------') else: - token_ids = [] - return token_ids + output = None + prev_seqs = None + # self.worker_response_mq.enqueue(result) + return output, prev_seqs + + def _update_states(self, batch: ScheduledBatchs): + for req_id in self.input_batch.finished_req_ids: + self.input_batch.remove_request(req_id) + + for request in batch.seqs: + self.input_batch.add_request(request) + + def finish_req(self, batch: ScheduledBatchs): + request_id = self.input_batch.req_ids + for id in request_id: + self.input_batch.finished_req_ids.add(id) + # print("finish_req:", self.input_batch.finished_req_ids) + @torch.inference_mode() - def forward(self, batch: ScheduledBatchs) -> list[int]: + def forward(self, batch: ScheduledBatchs, is_wramup = False) -> list[int]: + # self._update_states(batch) input_ids, positions, temperatures = self.prepare_model(batch) logits = self.run_model(input_ids, positions, batch.is_prefill) reset_context() + self.finish_req(batch) + # self.postprocess(batch, logits, temperatures) + # return self.async_get_output() return self.postprocess(batch, logits, temperatures) @torch.inference_mode() diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index d72117b560..d978a8dce5 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -1,10 +1,13 @@ from collections import deque -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Optional, cast, Any from atom.config import Config from atom.model_engine.block_manager import BlockManager from atom.model_engine.sequence import Sequence, SequenceStatus +import torch +import numpy as np @dataclass class ScheduledBatchs: @@ -12,6 +15,109 @@ class ScheduledBatchs: is_prefill: bool deferred_reqID: list[int] + unfinished_prev_req: list[Sequence] + + # req_id -> num_scheduled_tokens + # Number of tokens scheduled for each request. + num_scheduled_tokens: dict[str, int] + # Total number of tokens scheduled for all requests. + # Equal to sum(num_scheduled_tokens.values()) + total_num_scheduled_tokens: int + # Cached reference to the GPU tensor of previously sampled tokens + # prev_sampled_token_ids: Optional[torch.Tensor] = None + # prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None + # prev_req_id_to_index: Optional[dict[str, int]] = None + + +class InputBatch: + + def __init__( + self, + max_num_reqs: int, + max_model_len: int + ): + self.max_num_reqs = max_num_reqs + self.max_model_len = max_model_len + self.token_ids_cpu_tensor = torch.zeros( + (max_num_reqs, max_model_len), + device="cpu", + dtype=torch.int32, + pin_memory=False, + ) + self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() + self.num_tokens = np.zeros(max_num_reqs, dtype=np.int32) + self.req_id_to_index: dict[str, int] = {} + + # Cached reference to the GPU tensor of previously sampled tokens + self.prev_sampled_token_ids: Optional[torch.Tensor] = None + self.prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None + self.prev_req_id_to_index: Optional[dict[str, int]] = None + + self._req_ids: list[Optional[str]] = [] + self.req_output_token_ids: list[Optional[list[int]]] = [] + + self.finished_req_ids: set[str] = set() + + @property + def num_reqs(self) -> int: + return len(self.req_id_to_index) + + def _register_add_request(self, request: "Sequence") -> int: + """Track add-request operations for logits processors. + Not applicable to pooling models. + """ + + new_req_index = self.num_reqs + + assert new_req_index < self.max_num_reqs + return new_req_index + + + def add_request( + self, + request: "Sequence", + ) -> int: + req_index = self._register_add_request(request) + + req_id = request.id + + if req_index == len(self._req_ids): + self._req_ids.append(req_id) + # self.req_output_token_ids.append(request.output_token_ids) + else: + self._req_ids[req_index] = req_id + # self.req_output_token_ids[req_index] = request.output_token_ids + self.req_id_to_index[req_id] = req_index + + # new_req_index = request.last_token + # self.req_id_to_index[req_id] = new_req_index + + # self._req_ids.append(req_id) + # print("req_id:", req_id) + + def remove_request(self, req_id: str) -> Optional[int]: + """This method must always be followed by a call to condense(). + + Args: + req_id: request to remove + + Returns: + Removed request index, or `None` if `req_id` not recognized + """ + req_index = self.req_id_to_index.pop(req_id, None) + if req_index is None: + return None + + self._req_ids[req_index] = None + # self.req_output_token_ids[req_index] = None + + + @property + def req_ids(self) -> list[str]: + # None elements should only be present transiently + # while performing state updates to the batch. + return cast(list[str], self._req_ids) + class Scheduler: @@ -39,6 +145,9 @@ def schedule(self) -> ScheduledBatchs: scheduled_seqs = [] num_seqs = 0 num_batched_tokens = 0 + + num_scheduled_tokens: dict[str, int] = {} + if not self.running and not self.waiting: self.block_manager.reset() @@ -55,9 +164,16 @@ def schedule(self) -> ScheduledBatchs: self.waiting.popleft() self.running.append(seq) scheduled_seqs.append(seq) + # in vllm, using num_new_tokens + num_scheduled_tokens[seq.id] = seq.num_tokens + + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + if scheduled_seqs: return ScheduledBatchs( - seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[] + seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[], unfinished_prev_req=scheduled_seqs, + num_scheduled_tokens=num_scheduled_tokens, + total_num_scheduled_tokens=total_num_scheduled_tokens, ) # decode @@ -73,12 +189,19 @@ def schedule(self) -> ScheduledBatchs: num_seqs += 1 self.block_manager.may_append(seq) scheduled_seqs.append(seq) + num_scheduled_tokens[seq.id] = seq.num_tokens + + total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + assert scheduled_seqs self.running.extendleft(reversed(scheduled_seqs)) return ScheduledBatchs( seqs=scheduled_seqs, is_prefill=False, deferred_reqID=self.deferred_reqID, + unfinished_prev_req=scheduled_seqs, + num_scheduled_tokens=num_scheduled_tokens, + total_num_scheduled_tokens=total_num_scheduled_tokens, ) def preempt(self, seq: Sequence): @@ -86,20 +209,51 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): + def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int], prev_seqs: list[Sequence]): # # placeholder for the each decode step - # token_ids = [self.eos_token_id] * len(seqs) - # for seq, token_id in zip(seqs, token_ids): - # seq.append_token(token_id) + token_ids = [self.eos_token_id] * len(seqs) + for seq, token_id in zip(seqs, prev_token_ids): + seq.append_token(token_id) + + if not prev_token_ids or not prev_seqs: + return + + # seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} + + # self.unfinished_req = [] + # for i, seq in enumerate(seqs): + # if seq.id in seq_id_to_index: + # token_idx = seq_id_to_index[seq.id] + # token_id = prev_token_ids[token_idx] + + # seq.append_token(token_id) + + # leave_reason = None + # if not seq.ignore_eos and token_id == self.eos_token_id: + # leave_reason = "eos" + # elif seq.num_completion_tokens == seq.max_tokens: + # leave_reason = "max_tokens" + + # if leave_reason is not None: + # seq.leave_reason = leave_reason + # seq.status = SequenceStatus.FINISHED + # self.block_manager.deallocate(seq) + # self.running.remove(seq) + # else: + # self.unfinished_req.append(i) + # else: + # print(f"New sequence {seq.id} found in current batch") + - # if not prev_token_ids: - # return # update token_ids with the actual sampled token ids self.unfinished_req = [] + # prev_token_ids = out.get_output() + # worker_response_mq.dequeue( token_ids = prev_token_ids for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): - seq.append_token(token_id) - # seq.token_ids[-2] = token_id + # if seq.id in prev_seqs: + # seq.append_token(token_id) + seq.token_ids[-2] = token_id leave_reason = None # Check if sequence ends with any stop sequence From 35c960c5473d3ce530499fb1251040a90361d3b9 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 15 Oct 2025 17:01:48 +0800 Subject: [PATCH 079/119] draft deffer output 2 --- atom/model_engine/model_runner.py | 98 +++++-------------------------- atom/model_engine/scheduler.py | 79 +++++++++++++++---------- 2 files changed, 63 insertions(+), 114 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 52cda159e6..102728edee 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -28,7 +28,6 @@ logger = logging.getLogger("atom") from atom.utils.forward_context import AttentionMetadata, set_forward_context -from atom.model_engine.outputs import ModelRunnerOutput, AsyncModelRunnerOutput suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -37,55 +36,6 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } -# Wrapper for ModelRunnerOutput to support overlapped execution. -class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput): - - def __init__( - self, - # model_runner_output: ModelRunnerOutput, - sampled_token_ids: torch.Tensor, - # invalid_req_indices: list[int], - async_output_copy_stream: torch.cuda.Stream, - ): - # self._model_runner_output = model_runner_output - # self._invalid_req_indices = invalid_req_indices - - # Event on the copy stream so we can synchronize the non-blocking copy. - # self._async_copy_ready_event = torch.cuda.Event() - - # Keep a reference to the device tensor to avoid it being - # deallocated until we finish copying it to the host. - self._sampled_token_ids = sampled_token_ids - - # Initiate the copy on a separate stream, but do not synchronize it. - default_stream = torch.cuda.current_stream() - with torch.cuda.stream(async_output_copy_stream): - async_output_copy_stream.wait_stream(default_stream) - self._sampled_token_ids_cpu = self._sampled_token_ids.to( - 'cpu', non_blocking=True) - # self._async_copy_ready_event.record() - - def get_output(self) -> ModelRunnerOutput: - """Copy the device tensors to the host and return a ModelRunnerOutput. - - This function blocks until the copy is finished. - """ - # self._async_copy_ready_event.synchronize() - - # Release the device tensor once the copy has completed - del self._sampled_token_ids - - valid_sampled_token_ids = self._sampled_token_ids_cpu.tolist() - # for i in self._invalid_req_indices: - # valid_sampled_token_ids[i].clear() - - return valid_sampled_token_ids - # output = self._model_runner_output - # output.sampled_token_ids = valid_sampled_token_ids - # return output - - - class OutputProcessor: def __init__(self): @@ -152,15 +102,20 @@ def update_and_ret( else: self.deferred_request_id.append(req.id) - if returned_reqIDs: - for ids in self.token_ids_cpu[:-1]: - ids_list = ids.tolist() - for req_id in returned_reqIDs: - self.deferred_request_id.remove(req_id) - req_index = batch.unfinished_prev_req.index(req_id) - batch.unfinished_prev_req.remove(req_id) - req = batch.seqs[req_index] - req.append_token(prev_token_ids[req_index]) + # prev_seqs = [] + # if returned_reqIDs: + # # for ids in self.token_ids_cpu[:-1]: + # # ids_list = ids.tolist() + # for req_id in returned_reqIDs: + # self.deferred_request_id.remove(req_id) + # for i, seq in enumerate(batch.seqs): + # if seq.id in returned_reqIDs: + # # batch.unfinished_prev_req.remove(i) + # prev_seqs.append(seq) + # # req_index = batch.unfinished_prev_req.index(req_id) + # # batch.unfinished_prev_req.remove(req_id) + # # req = batch.seqs[req_index] + # # req.append_token(prev_token_ids[req_index]) return prev_token_ids, prev_seqs # self.async_copy_event.synchronize() @@ -712,26 +667,6 @@ def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: ).cuda(non_blocking=True) return temperatures - # def _get_cumsum_and_arange( - # self, - # num_tokens: np.ndarray, - # cumsum_dtype: Optional[np.dtype] = None, - # ) -> tuple[np.ndarray, np.ndarray]: - # """Get the cumulative sum and batched arange of the given array. - # # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) - # # Equivalent to but faster than: - # # np.concatenate([np.arange(n) for n in num_tokens]) - # """ - # # Step 1. [2, 5, 3] -> [2, 7, 10] - # cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) - # total_num_tokens = cu_num_tokens[-1] - # # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] - # cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) - # # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - # arange = self.arange_np[:total_num_tokens] - cumsums_offsets - - # return cu_num_tokens, arange - def prepare_model(self, scheduled_batchs: ScheduledBatchs): self._update_states(scheduled_batchs) @@ -742,8 +677,6 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): _req_ids = self.input_batch.req_ids req_ids = [req_id for req_id in _req_ids if req_id is not None] - # print("scheduled_batchs.num_scheduled_tokens", scheduled_batchs.num_scheduled_tokens) - # print("req_ids", req_ids) tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] num_scheduled_tokens = np.array(tokens, dtype=np.int32) # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] @@ -818,9 +751,6 @@ def postprocess( batch, sampled_tokens, ) - print("sampled_token_ids", sampled_token_ids) - print("prev_token_ids", output) - print('----------') else: output = None prev_seqs = None diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index d978a8dce5..fbea489c5d 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -211,41 +211,60 @@ def preempt(self, seq: Sequence): def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int], prev_seqs: list[Sequence]): # # placeholder for the each decode step - token_ids = [self.eos_token_id] * len(seqs) - for seq, token_id in zip(seqs, prev_token_ids): - seq.append_token(token_id) + # token_ids = [self.eos_token_id] * len(seqs) + # for seq, token_id in zip(seqs, prev_token_ids): + # seq.append_token(token_id) + + # print('--------------------') + # for i, seq in enumerate(prev_seqs): + # print("prev_seqs", seq.id) + # print(seq.token_ids) + # for i, seq in enumerate(seqs): + # print("seqs", seq.id) + # print(seq.token_ids) + # print("prev_token_ids", prev_token_ids) if not prev_token_ids or not prev_seqs: return - # seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} - # self.unfinished_req = [] - # for i, seq in enumerate(seqs): - # if seq.id in seq_id_to_index: - # token_idx = seq_id_to_index[seq.id] - # token_id = prev_token_ids[token_idx] - - # seq.append_token(token_id) - - # leave_reason = None - # if not seq.ignore_eos and token_id == self.eos_token_id: - # leave_reason = "eos" - # elif seq.num_completion_tokens == seq.max_tokens: - # leave_reason = "max_tokens" - - # if leave_reason is not None: - # seq.leave_reason = leave_reason - # seq.status = SequenceStatus.FINISHED - # self.block_manager.deallocate(seq) - # self.running.remove(seq) - # else: - # self.unfinished_req.append(i) - # else: - # print(f"New sequence {seq.id} found in current batch") - - - # update token_ids with the actual sampled token ids + # processed_seqs = set() + + # for i, prev_seq in enumerate(prev_seqs): + # token_id = prev_token_ids[i] + + # current_seq = None + # for seq in seqs: + # if seq.id == prev_seq.id: + # current_seq = seq + + # if current_seq and current_seq.id not in processed_seqs: + # processed_seqs.add(current_seq.id) + + # # Check if this token has been used before (to avoid outputting a word repeatedly) + # if (len(current_seq.token_ids) > prev_seq.num_tokens and + # current_seq.token_ids[-1] == token_id): + # print(f"Token {token_id} already applied to sequence {current_seq.id}") + # continue + + # current_seq.append_token(token_id) + # print(f"Applied token {token_id} to sequence {current_seq.id}") + + # leave_reason = None + # if not current_seq.ignore_eos and token_id == self.eos_token_id: + # leave_reason = "eos" + # elif current_seq.num_completion_tokens == current_seq.max_tokens: + # leave_reason = "max_tokens" + + # if leave_reason is not None: + # current_seq.leave_reason = leave_reason + # current_seq.status = SequenceStatus.FINISHED + # self.block_manager.deallocate(current_seq) + # self.running.remove(current_seq) + # else: + # self.unfinished_req.append(i) + + seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} self.unfinished_req = [] # prev_token_ids = out.get_output() # worker_response_mq.dequeue( From 70b80e8d7998acd066cf827782e3b6330b0d5025 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 11:01:55 +0800 Subject: [PATCH 080/119] fix acc issue --- atom/model_engine/engine_core.py | 4 +- atom/model_engine/model_runner.py | 71 +++++++++++++++++-------------- atom/model_engine/scheduler.py | 64 +++++----------------------- 3 files changed, 50 insertions(+), 89 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index e3264b6080..6300e32225 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -138,9 +138,9 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() - out, prev_seqs = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) + out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) # scheduled_batchs.prev_seqs = prev_seqs - self.scheduler.postprocess(scheduled_batchs.seqs, out, prev_seqs) + self.scheduler.postprocess(scheduled_batchs.seqs, out) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] ) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 102728edee..057a1f362f 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -88,13 +88,13 @@ def clean_token(self): def update_and_ret( self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor ) -> list[int]: - # token_ids = self.recv_async_output() - prev_token_ids, prev_seqs = self.revc_async_output_tuple() + token_ids = self.recv_async_output() + # prev_token_ids, prev_seqs = self.revc_async_output_tuple() self.send_to_cpu_async(batch, sampled_token_ids) # return token_ids # return prev_token_ids, prev_seqs - deferred_reqIDs = [] + # deferred_reqIDs = [] returned_reqIDs = [] for req in batch.seqs: if req.id in self.deferred_request_id: @@ -102,23 +102,19 @@ def update_and_ret( else: self.deferred_request_id.append(req.id) - # prev_seqs = [] - # if returned_reqIDs: - # # for ids in self.token_ids_cpu[:-1]: - # # ids_list = ids.tolist() - # for req_id in returned_reqIDs: - # self.deferred_request_id.remove(req_id) - # for i, seq in enumerate(batch.seqs): - # if seq.id in returned_reqIDs: - # # batch.unfinished_prev_req.remove(i) - # prev_seqs.append(seq) - # # req_index = batch.unfinished_prev_req.index(req_id) - # # batch.unfinished_prev_req.remove(req_id) - # # req = batch.seqs[req_index] - # # req.append_token(prev_token_ids[req_index]) - return prev_token_ids, prev_seqs + if returned_reqIDs: + # for ids in self.token_ids_cpu[:-1]: + # ids_list = ids.tolist() + for req_id in returned_reqIDs: + self.deferred_request_id.remove(req_id) + # req_index = batch.unfinished_prev_req.index(req_id) + # batch.unfinished_prev_req.remove(req_id) + # req = batch.seqs[req_index] + # req.append_token(ids_list[req_index]) + # self.async_copy_event.synchronize() + return token_ids class ModelRunner: @@ -254,7 +250,7 @@ def warmup_model(self): # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) self.forward(dummy_batch) - # self.out_processor.clean_token() + self.out_processor.clean_token() torch.cuda.empty_cache() def allocate_decode_vars(self): @@ -490,7 +486,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, prev_common_req_indices_tensor]) # prev_common_req_indices_tensor, 0]) - self.positions.copy_to_gpu(total_num_scheduled_tokens) + # self.positions.gpu[:num_commmon_tokens].copy_() def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): @@ -552,7 +548,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): ) assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" input_ids = [] - positions = [] + # positions = [] slot_mapping = [] context_lens = [] dropout_p = 0.0 @@ -560,7 +556,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self.total_blocks = 0 context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] - positions = np.array(context_lens, dtype=np.int64) + # positions = np.array(context_lens, dtype=np.int64) context_lens = np.array(context_lens, dtype=np.int32) input_ids = [seq.last_token for seq in scheduled_batchs.seqs] input_ids = np.array(input_ids, dtype=np.int64) @@ -586,11 +582,20 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self._update_paged_kv_tensors(seq.block_table, current_seq_len) block_tables = self.prepare_block_tables(scheduled_batchs.seqs) + # cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + # cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) + num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens + num_input_tokens = num_scheduled_tokens + prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] + prev_positions_ = self.input_batch.prev_position_ids var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping - var["input_ids"].np[:bs] = input_ids - var["positions"].np[:bs] = positions + var["input_ids"].np[:bs] = prev_input_ids_[:bs].cpu() + var["positions"].np[:bs] = prev_positions_[:bs] + # var["input_ids"].np[:bs] = input_ids + # var["positions"].np[:bs] = positions + var["context_lens"].np[:bs] = context_lens var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables @@ -633,11 +638,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) - num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens - num_input_tokens = num_scheduled_tokens - input_ids = self.input_ids.gpu[:num_input_tokens] - positions = self.positions.gpu[:num_input_tokens] - return input_ids, positions + return prev_input_ids_, prev_positions_ return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): @@ -735,7 +736,12 @@ def postprocess( req_id: i for i, req_id in enumerate(self.input_batch.req_ids) } - + # plus 1 to fit next iter seq len + context_lens = [seq.num_tokens + 1 for seq in batch.seqs] + # positions = np.array(context_lens, dtype=np.int64) + + self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) + # output = AsyncGPUModelRunnerOutput( # sampled_token_ids=sampled_token_ids, @@ -747,15 +753,14 @@ def postprocess( # self.handle_output(output) # token_ids = sampled_tokens.tolist() - output, prev_seqs = self.out_processor.update_and_ret( + output = self.out_processor.update_and_ret( batch, sampled_tokens, ) else: output = None - prev_seqs = None # self.worker_response_mq.enqueue(result) - return output, prev_seqs + return output def _update_states(self, batch: ScheduledBatchs): for req_id in self.input_batch.finished_req_ids: diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index fbea489c5d..9bd2ff61a4 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -52,6 +52,7 @@ def __init__( self.prev_sampled_token_ids: Optional[torch.Tensor] = None self.prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None self.prev_req_id_to_index: Optional[dict[str, int]] = None + self.prev_position_ids: Optional[torch.Tensor] = None self._req_ids: list[Optional[str]] = [] self.req_output_token_ids: list[Optional[list[int]]] = [] @@ -209,62 +210,17 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int], prev_seqs: list[Sequence]): + def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): + # # placeholder for the each decode step - # token_ids = [self.eos_token_id] * len(seqs) - # for seq, token_id in zip(seqs, prev_token_ids): - # seq.append_token(token_id) - - # print('--------------------') - # for i, seq in enumerate(prev_seqs): - # print("prev_seqs", seq.id) - # print(seq.token_ids) - # for i, seq in enumerate(seqs): - # print("seqs", seq.id) - # print(seq.token_ids) - # print("prev_token_ids", prev_token_ids) - - if not prev_token_ids or not prev_seqs: + token_ids = [self.eos_token_id] * len(seqs) + for seq, token_id in zip(seqs, token_ids): + seq.append_token(token_id) + + if not prev_token_ids: return - - # self.unfinished_req = [] - # processed_seqs = set() - - # for i, prev_seq in enumerate(prev_seqs): - # token_id = prev_token_ids[i] - - # current_seq = None - # for seq in seqs: - # if seq.id == prev_seq.id: - # current_seq = seq - - # if current_seq and current_seq.id not in processed_seqs: - # processed_seqs.add(current_seq.id) - - # # Check if this token has been used before (to avoid outputting a word repeatedly) - # if (len(current_seq.token_ids) > prev_seq.num_tokens and - # current_seq.token_ids[-1] == token_id): - # print(f"Token {token_id} already applied to sequence {current_seq.id}") - # continue - - # current_seq.append_token(token_id) - # print(f"Applied token {token_id} to sequence {current_seq.id}") - - # leave_reason = None - # if not current_seq.ignore_eos and token_id == self.eos_token_id: - # leave_reason = "eos" - # elif current_seq.num_completion_tokens == current_seq.max_tokens: - # leave_reason = "max_tokens" - - # if leave_reason is not None: - # current_seq.leave_reason = leave_reason - # current_seq.status = SequenceStatus.FINISHED - # self.block_manager.deallocate(current_seq) - # self.running.remove(current_seq) - # else: - # self.unfinished_req.append(i) - - seq_id_to_index = {seq.id: i for i, seq in enumerate(prev_seqs)} + + # update token_ids with the actual sampled token ids self.unfinished_req = [] # prev_token_ids = out.get_output() # worker_response_mq.dequeue( From a6bf9524ffff732d776e05595160534a44d426d9 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 15:33:34 +0800 Subject: [PATCH 081/119] commit on opt deffer --- atom/model_engine/model_runner.py | 58 ++++++++++--------------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 057a1f362f..955ab53cd2 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -91,28 +91,6 @@ def update_and_ret( token_ids = self.recv_async_output() # prev_token_ids, prev_seqs = self.revc_async_output_tuple() self.send_to_cpu_async(batch, sampled_token_ids) - # return token_ids - # return prev_token_ids, prev_seqs - - # deferred_reqIDs = [] - returned_reqIDs = [] - for req in batch.seqs: - if req.id in self.deferred_request_id: - returned_reqIDs.append(req.id) - else: - self.deferred_request_id.append(req.id) - - if returned_reqIDs: - # for ids in self.token_ids_cpu[:-1]: - # ids_list = ids.tolist() - for req_id in returned_reqIDs: - self.deferred_request_id.remove(req_id) - # req_index = batch.unfinished_prev_req.index(req_id) - # batch.unfinished_prev_req.remove(req_id) - # req = batch.seqs[req_index] - # req.append_token(ids_list[req_index]) - - # self.async_copy_event.synchronize() return token_ids @@ -159,6 +137,10 @@ def __init__(self, rank: int, config: Config): dtype=torch.int32) self.positions = self._make_buffer(self.config.max_num_batched_tokens, dtype=torch.int64) + + self.input_ids_index_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) + self.prev_common_req_indices_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) + self.use_async_scheduling = True self.async_output_copy_stream = torch.cuda.Stream() if \ self.use_async_scheduling else None @@ -469,24 +451,22 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, return # Upload the index tensors asynchronously # so the scatter can be non-blocking. - input_ids_index_tensor = torch.tensor(flattened_indices, - dtype=torch.int64, - pin_memory=True).to( - self.device, - non_blocking=True) - prev_common_req_indices_tensor = torch.tensor( - prev_common_req_indices, - dtype=torch.int64, - pin_memory=True).to(self.device, non_blocking=True) - # print('self.input_batch.prev_sampled_token_ids', self.input_batch.prev_sampled_token_ids) + + self.input_ids_index_tensor.cpu[:num_commmon_tokens] = torch.tensor( + flattened_indices, dtype=torch.int64, device="cpu") + self.prev_common_req_indices_tensor.cpu[:num_commmon_tokens] = torch.tensor( + prev_common_req_indices, dtype=torch.int64, device="cpu") + + self.input_ids_index_tensor.copy_to_gpu(num_commmon_tokens) + self.prev_common_req_indices_tensor.copy_to_gpu(num_commmon_tokens) + self.input_ids.gpu.scatter_( dim=0, - index=input_ids_index_tensor, + index=self.input_ids_index_tensor.gpu[:num_commmon_tokens], src=self.input_batch.prev_sampled_token_ids[ - prev_common_req_indices_tensor]) + self.prev_common_req_indices_tensor.gpu[:num_commmon_tokens]]) # prev_common_req_indices_tensor, 0]) - # self.positions.gpu[:num_commmon_tokens].copy_() def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): @@ -556,7 +536,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): self.total_blocks = 0 context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] - # positions = np.array(context_lens, dtype=np.int64) + positions = np.array(context_lens, dtype=np.int64) context_lens = np.array(context_lens, dtype=np.int32) input_ids = [seq.last_token for seq in scheduled_batchs.seqs] input_ids = np.array(input_ids, dtype=np.int64) @@ -592,7 +572,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping var["input_ids"].np[:bs] = prev_input_ids_[:bs].cpu() - var["positions"].np[:bs] = prev_positions_[:bs] + var["positions"].np[:bs] = positions # var["input_ids"].np[:bs] = input_ids # var["positions"].np[:bs] = positions @@ -638,7 +618,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): kv_indices=var["kv_indices"].gpu, kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], ) - return prev_input_ids_, prev_positions_ + return prev_input_ids_, positions return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): @@ -731,7 +711,6 @@ def postprocess( self.input_batch.prev_sampled_token_ids = \ sampled_token_ids - # print("self.input_batch.prev_sampled_token_ids in postprocess", self.input_batch.prev_sampled_token_ids) self.input_batch.prev_req_id_to_index = { req_id: i for i, req_id in enumerate(self.input_batch.req_ids) @@ -773,7 +752,6 @@ def finish_req(self, batch: ScheduledBatchs): request_id = self.input_batch.req_ids for id in request_id: self.input_batch.finished_req_ids.add(id) - # print("finish_req:", self.input_batch.finished_req_ids) @torch.inference_mode() From f9df1dc5ae7d0e4a5f479b54764c3e29ff80aadb Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 17:16:39 +0800 Subject: [PATCH 082/119] remove d2h in prepare decode --- atom/model_engine/model_runner.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 955ab53cd2..36be10382c 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -567,11 +567,12 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens num_input_tokens = num_scheduled_tokens prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] - prev_positions_ = self.input_batch.prev_position_ids + # prev_positions_ = self.input_batch.prev_position_ids var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping - var["input_ids"].np[:bs] = prev_input_ids_[:bs].cpu() + # var["input_ids"].np[:bs] = prev_input_ids_[:bs] + var["input_ids"].gpu[:bs] = prev_input_ids_[:bs] var["positions"].np[:bs] = positions # var["input_ids"].np[:bs] = input_ids # var["positions"].np[:bs] = positions @@ -594,7 +595,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): for el in [ "slot_mapping", - "input_ids", + # "input_ids", "positions", "context_lens", "block_tables", @@ -704,11 +705,6 @@ def postprocess( # Cache the sampled tokens on the GPU and avoid CPU sync. # These will be copied into input_ids in the next step # when preparing inputs. - # if self.input_batch.prev_sampled_token_ids is None: - # output = None - # else: - # output = self.input_batch.prev_sampled_token_ids - self.input_batch.prev_sampled_token_ids = \ sampled_token_ids self.input_batch.prev_req_id_to_index = { @@ -716,20 +712,10 @@ def postprocess( for i, req_id in enumerate(self.input_batch.req_ids) } # plus 1 to fit next iter seq len - context_lens = [seq.num_tokens + 1 for seq in batch.seqs] + # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] # positions = np.array(context_lens, dtype=np.int64) - self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) - - - # output = AsyncGPUModelRunnerOutput( - # sampled_token_ids=sampled_token_ids, - # # invalid_req_indices=invalid_req_indices, - # async_output_copy_stream=self.async_output_copy_stream, - # ) - - # .get_output() - # self.handle_output(output) + # self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) # token_ids = sampled_tokens.tolist() output = self.out_processor.update_and_ret( From d37d75d1114bd3d35fcb1a24a1bbcd6023169153 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 16 Oct 2025 18:05:27 +0800 Subject: [PATCH 083/119] use np --- atom/model_engine/model_runner.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 36be10382c..062b90495e 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -452,10 +452,8 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # Upload the index tensors asynchronously # so the scatter can be non-blocking. - self.input_ids_index_tensor.cpu[:num_commmon_tokens] = torch.tensor( - flattened_indices, dtype=torch.int64, device="cpu") - self.prev_common_req_indices_tensor.cpu[:num_commmon_tokens] = torch.tensor( - prev_common_req_indices, dtype=torch.int64, device="cpu") + self.input_ids_index_tensor.np[:num_commmon_tokens] = np.array(flattened_indices, dtype=np.int64) + self.prev_common_req_indices_tensor.np[:num_commmon_tokens] = np.array(prev_common_req_indices, dtype=np.int64) self.input_ids_index_tensor.copy_to_gpu(num_commmon_tokens) self.prev_common_req_indices_tensor.copy_to_gpu(num_commmon_tokens) From cf53ec89433c24a67a9204f00703e8845d0f15b4 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 17 Oct 2025 16:50:06 +0800 Subject: [PATCH 084/119] reduce prepare_input_ids overhead in prefill --- atom/model_engine/model_runner.py | 41 +++++++++++++++++++++++++------ atom/model_engine/scheduler.py | 6 +++-- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 062b90495e..e84ed2a51f 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -138,6 +138,12 @@ def __init__(self, rank: int, config: Config): self.positions = self._make_buffer(self.config.max_num_batched_tokens, dtype=torch.int64) + self.arange_np = np.arange(max(self.config.max_num_seqs + 1, + self.config.max_model_len, + self.config.max_num_batched_tokens), + dtype=np.int64) + + self.input_ids_index_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) self.prev_common_req_indices_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) @@ -226,7 +232,7 @@ def warmup_model(self): num_scheduled_tokens: dict[str, int] = {} for seq in seqs: - num_scheduled_tokens[seq.id] = seq.num_tokens + num_scheduled_tokens[seq.id] = 1 total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) @@ -445,8 +451,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # The indices are both the same permutation of 0..N-1 so # we can copy directly using a single slice. self.input_ids.gpu[:num_commmon_tokens].copy_( - self.input_batch.prev_sampled_token_ids[:num_commmon_tokens, - 0], + self.input_batch.prev_sampled_token_ids[:num_commmon_tokens], non_blocking=True) return # Upload the index tensors asynchronously @@ -647,6 +652,26 @@ def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: ).cuda(non_blocking=True) return temperatures + def _get_cumsum_and_arange( + self, + num_tokens: np.ndarray, + cumsum_dtype: Optional[np.dtype] = None, + ) -> tuple[np.ndarray, np.ndarray]: + """Get the cumulative sum and batched arange of the given array. + # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) + # Equivalent to but faster than: + # np.concatenate([np.arange(n) for n in num_tokens]) + """ + # Step 1. [2, 5, 3] -> [2, 7, 10] + cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) + total_num_tokens = cu_num_tokens[-1] + # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] + cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) + # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + arange = self.arange_np[:total_num_tokens] - cumsums_offsets + + return cu_num_tokens, arange + def prepare_model(self, scheduled_batchs: ScheduledBatchs): self._update_states(scheduled_batchs) @@ -661,9 +686,9 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): num_scheduled_tokens = np.array(tokens, dtype=np.int32) # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - # cu_num_tokens, arange = self._get_cumsum_and_arange( - # num_scheduled_tokens) - cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) + cu_num_tokens, arange = self._get_cumsum_and_arange( + num_scheduled_tokens) + # cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) self._prepare_input_ids(total_num_scheduled_tokens, cu_num_tokens) seqs = scheduled_batchs.seqs @@ -676,7 +701,7 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): def run_model( self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool ): - torch.cuda.empty_cache() + # torch.cuda.empty_cache() ctx = get_context() bs = ctx.batch_size if is_prefill or self.enforce_eager or bs > self.graph_bs[-1]: @@ -703,11 +728,13 @@ def postprocess( # Cache the sampled tokens on the GPU and avoid CPU sync. # These will be copied into input_ids in the next step # when preparing inputs. + invalid_req_indices_set = [None] self.input_batch.prev_sampled_token_ids = \ sampled_token_ids self.input_batch.prev_req_id_to_index = { req_id: i for i, req_id in enumerate(self.input_batch.req_ids) + if req_id not in invalid_req_indices_set } # plus 1 to fit next iter seq len # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 9bd2ff61a4..aff2a460c9 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -166,7 +166,8 @@ def schedule(self) -> ScheduledBatchs: self.running.append(seq) scheduled_seqs.append(seq) # in vllm, using num_new_tokens - num_scheduled_tokens[seq.id] = seq.num_tokens + num_new_tokens = 1 + num_scheduled_tokens[seq.id] = num_new_tokens total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) @@ -190,7 +191,8 @@ def schedule(self) -> ScheduledBatchs: num_seqs += 1 self.block_manager.may_append(seq) scheduled_seqs.append(seq) - num_scheduled_tokens[seq.id] = seq.num_tokens + num_new_tokens = 1 + num_scheduled_tokens[seq.id] = num_new_tokens total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) From 405985b1d5ccd5b635fffd67724c836367c04bc5 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 17 Oct 2025 23:43:30 +0800 Subject: [PATCH 085/119] rename inputbatch to prevbatch --- atom/model_engine/model_runner.py | 4 ++-- atom/model_engine/scheduler.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index e84ed2a51f..9eb6f96361 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -15,7 +15,7 @@ from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture from atom.config import Config, set_current_atom_config -from atom.model_engine.scheduler import ScheduledBatchs, InputBatch +from atom.model_engine.scheduler import ScheduledBatchs, PrevScheduledBatchs from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler @@ -129,7 +129,7 @@ def __init__(self, rank: int, config: Config): self.out_processor = OutputProcessor() self.sampler = Sampler() - self.input_batch = InputBatch( + self.input_batch = PrevScheduledBatchs( max_num_reqs=self.config.max_num_seqs, max_model_len=self.config.max_model_len) diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index aff2a460c9..39c22d34ef 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -29,7 +29,7 @@ class ScheduledBatchs: # prev_req_id_to_index: Optional[dict[str, int]] = None -class InputBatch: +class PrevScheduledBatchs: def __init__( self, From d3bae13c43668f003ddcaada59553de8b62f1802 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Fri, 17 Oct 2025 23:51:40 +0800 Subject: [PATCH 086/119] fix profile by push cover --- atom/model_engine/engine_core.py | 2 +- atom/model_engine/model_runner.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 6300e32225..60a258a8bc 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -220,5 +220,5 @@ def start_profiler(self): def stop_profiler(self): if self.profile_enbaled: print("Stopping profiler...") - self.runner_mgr.call_func("stop_profiler") + self.runner_mgr.call_func("stop_profiler", wait_out=True) print("Profiler stopped.") diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 9eb6f96361..7a7ff8b3c6 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -213,6 +213,7 @@ def stop_profiler(self): if self.profiler is not None: self.profiler.__exit__(None, None, None) self.profiler = None + return True def warmup_model(self): torch.cuda.empty_cache() From 6214e822dcb950a684fbedca4e2872c32b7e0e83 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sat, 18 Oct 2025 00:27:27 +0800 Subject: [PATCH 087/119] rename and remove some useless code --- atom/model_engine/engine_core.py | 1 - atom/model_engine/model_runner.py | 44 +++++++++++-------------------- atom/model_engine/scheduler.py | 10 +------ 3 files changed, 17 insertions(+), 38 deletions(-) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 60a258a8bc..d3e1c21885 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -139,7 +139,6 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batchs = self.scheduler.schedule() out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) - # scheduled_batchs.prev_seqs = prev_seqs self.scheduler.postprocess(scheduled_batchs.seqs, out) self.output_queue.put_nowait( [seq for seq in scheduled_batchs.seqs if seq.is_finished] diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 7a7ff8b3c6..75f83bf6ac 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -59,7 +59,6 @@ def send_to_cpu_async(self, batch: ScheduledBatchs, gpu_tensor: torch.Tensor): self.async_copy_stream.wait_stream(default_stream) cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) self.async_copy_event.record(self.async_copy_stream) - # self.token_ids_gpu.append(gpu_tensor) self.token_ids_cpu.append(cpu_tensor) self.pending_outputs.append((cpu_tensor, batch.seqs)) @@ -89,7 +88,6 @@ def update_and_ret( self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor ) -> list[int]: token_ids = self.recv_async_output() - # prev_token_ids, prev_seqs = self.revc_async_output_tuple() self.send_to_cpu_async(batch, sampled_token_ids) return token_ids @@ -129,7 +127,7 @@ def __init__(self, rank: int, config: Config): self.out_processor = OutputProcessor() self.sampler = Sampler() - self.input_batch = PrevScheduledBatchs( + self.prev_input_batch = PrevScheduledBatchs( max_num_reqs=self.config.max_num_seqs, max_model_len=self.config.max_model_len) @@ -236,7 +234,6 @@ def warmup_model(self): num_scheduled_tokens[seq.id] = 1 total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) - # print("total_num_scheduled_tokens dummy_batch", total_num_scheduled_tokens) dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) self.forward(dummy_batch) self.out_processor.clean_token() @@ -413,7 +410,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, from the previous engine iteration, in which case those tokens on the GPU need to be copied into the corresponding slots into input_ids.""" - if self.input_batch.prev_sampled_token_ids is None: + if self.prev_input_batch.prev_sampled_token_ids is None: # Normal scheduling case self.input_ids.copy_to_gpu(total_num_scheduled_tokens) return @@ -421,13 +418,13 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # Async scheduling case, where some decode requests from the previous # iteration won't have entries in input_ids_cpu and need to be copied # on the GPU from prev_sampled_token_ids. - prev_req_id_to_index = self.input_batch.prev_req_id_to_index + prev_req_id_to_index = self.prev_input_batch.prev_req_id_to_index assert prev_req_id_to_index is not None flattened_indices = [] prev_common_req_indices = [] indices_match = True max_flattened_index = -1 - for req_id, cur_index in self.input_batch.req_id_to_index.items(): + for req_id, cur_index in self.prev_input_batch.req_id_to_index.items(): if (prev_index := prev_req_id_to_index.get(req_id)) is not None: prev_common_req_indices.append(prev_index) # We need to compute the flattened input_ids index of the @@ -452,7 +449,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, # The indices are both the same permutation of 0..N-1 so # we can copy directly using a single slice. self.input_ids.gpu[:num_commmon_tokens].copy_( - self.input_batch.prev_sampled_token_ids[:num_commmon_tokens], + self.prev_input_batch.prev_sampled_token_ids[:num_commmon_tokens], non_blocking=True) return # Upload the index tensors asynchronously @@ -467,7 +464,7 @@ def _prepare_input_ids(self, total_num_scheduled_tokens: int, self.input_ids.gpu.scatter_( dim=0, index=self.input_ids_index_tensor.gpu[:num_commmon_tokens], - src=self.input_batch.prev_sampled_token_ids[ + src=self.prev_input_batch.prev_sampled_token_ids[ self.prev_common_req_indices_tensor.gpu[:num_commmon_tokens]]) # prev_common_req_indices_tensor, 0]) @@ -532,7 +529,7 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): ) assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" input_ids = [] - # positions = [] + positions = [] slot_mapping = [] context_lens = [] dropout_p = 0.0 @@ -571,15 +568,11 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens num_input_tokens = num_scheduled_tokens prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] - # prev_positions_ = self.input_batch.prev_position_ids var = self.decode_vars var["slot_mapping"].np[:graph_bs] = slot_mapping - # var["input_ids"].np[:bs] = prev_input_ids_[:bs] var["input_ids"].gpu[:bs] = prev_input_ids_[:bs] var["positions"].np[:bs] = positions - # var["input_ids"].np[:bs] = input_ids - # var["positions"].np[:bs] = positions var["context_lens"].np[:bs] = context_lens var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables @@ -680,7 +673,7 @@ def prepare_model(self, scheduled_batchs: ScheduledBatchs): total_num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens assert total_num_scheduled_tokens > 0 - _req_ids = self.input_batch.req_ids + _req_ids = self.prev_input_batch.req_ids req_ids = [req_id for req_id in _req_ids if req_id is not None] tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] @@ -730,19 +723,17 @@ def postprocess( # These will be copied into input_ids in the next step # when preparing inputs. invalid_req_indices_set = [None] - self.input_batch.prev_sampled_token_ids = \ + self.prev_input_batch.prev_sampled_token_ids = \ sampled_token_ids - self.input_batch.prev_req_id_to_index = { + self.prev_input_batch.prev_req_id_to_index = { req_id: i - for i, req_id in enumerate(self.input_batch.req_ids) + for i, req_id in enumerate(self.prev_input_batch.req_ids) if req_id not in invalid_req_indices_set } # plus 1 to fit next iter seq len # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] # positions = np.array(context_lens, dtype=np.int64) - # self.input_batch.prev_position_ids = np.array(context_lens, dtype=np.int64) - # token_ids = sampled_tokens.tolist() output = self.out_processor.update_and_ret( batch, @@ -750,20 +741,19 @@ def postprocess( ) else: output = None - # self.worker_response_mq.enqueue(result) return output def _update_states(self, batch: ScheduledBatchs): - for req_id in self.input_batch.finished_req_ids: - self.input_batch.remove_request(req_id) + for req_id in self.prev_input_batch.finished_req_ids: + self.prev_input_batch.remove_request(req_id) for request in batch.seqs: - self.input_batch.add_request(request) + self.prev_input_batch.add_request(request) def finish_req(self, batch: ScheduledBatchs): - request_id = self.input_batch.req_ids + request_id = self.prev_input_batch.req_ids for id in request_id: - self.input_batch.finished_req_ids.add(id) + self.prev_input_batch.finished_req_ids.add(id) @torch.inference_mode() @@ -773,8 +763,6 @@ def forward(self, batch: ScheduledBatchs, is_wramup = False) -> list[int]: logits = self.run_model(input_ids, positions, batch.is_prefill) reset_context() self.finish_req(batch) - # self.postprocess(batch, logits, temperatures) - # return self.async_get_output() return self.postprocess(batch, logits, temperatures) @torch.inference_mode() diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 39c22d34ef..2afa33b783 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -23,10 +23,6 @@ class ScheduledBatchs: # Total number of tokens scheduled for all requests. # Equal to sum(num_scheduled_tokens.values()) total_num_scheduled_tokens: int - # Cached reference to the GPU tensor of previously sampled tokens - # prev_sampled_token_ids: Optional[torch.Tensor] = None - # prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None - # prev_req_id_to_index: Optional[dict[str, int]] = None class PrevScheduledBatchs: @@ -94,7 +90,6 @@ def add_request( # self.req_id_to_index[req_id] = new_req_index # self._req_ids.append(req_id) - # print("req_id:", req_id) def remove_request(self, req_id: str) -> Optional[int]: """This method must always be followed by a call to condense(). @@ -224,12 +219,9 @@ def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): # update token_ids with the actual sampled token ids self.unfinished_req = [] - # prev_token_ids = out.get_output() - # worker_response_mq.dequeue( token_ids = prev_token_ids for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): - # if seq.id in prev_seqs: - # seq.append_token(token_id) + # seq.append_token(token_id) seq.token_ids[-2] = token_id leave_reason = None From 123569a449a5c9bb655a669020e27ac48a31f042 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 19 Oct 2025 15:47:04 +0000 Subject: [PATCH 088/119] finalize deferred_out --- atom/bench/example.py | 29 +- atom/model_engine/engine_core.py | 11 +- atom/model_engine/model_runner.py | 716 ++++++++++++++---------------- atom/model_engine/scheduler.py | 274 +++++------- atom/model_engine/sequence.py | 8 + atom/utils/context.py | 1 + 6 files changed, 474 insertions(+), 565 deletions(-) diff --git a/atom/bench/example.py b/atom/bench/example.py index 238d7aa7a2..aeeff6f577 100644 --- a/atom/bench/example.py +++ b/atom/bench/example.py @@ -40,34 +40,46 @@ parser.add_argument( - "--cudagraph-capture-sizes", type=str, default="[1,2,4,8,16]", help="Sizes to capture cudagraph." + "--cudagraph-capture-sizes", + type=str, + default="[1,2,4,8,16]", + help="Sizes to capture cudagraph.", ) parser.add_argument("--level", type=int, default=3, help="The level of compilation") parser.add_argument( - "--torch-profiler-dir", type=str, default=None, help="Directory to save torch profiler traces" + "--torch-profiler-dir", + type=str, + default=None, + help="Directory to save torch profiler traces", ) parser.add_argument( - "--enable-expert-parallel", action="store_true", help="Enable expert parallel(EP MoE)." + "--enable-expert-parallel", + action="store_true", + help="Enable expert parallel(EP MoE).", ) +parser.add_argument("--block-size", type=int, default=16, help="KV cache block size.") parser.add_argument( - "--block-size", type=int, default=16, help="KV cache block size." + "--temperature", type=float, default=0.6, help="temperature for sampling" ) parser.add_argument( "--load_dummy", action="store_true", help="Skip loading model weights." ) + def parse_size_list(size_str: str) -> List[int]: import ast + try: return ast.literal_eval(size_str) except (ValueError, SyntaxError) as e: raise ValueError(f"Error list size: {size_str}") from e + def main(): args = parser.parse_args() model_name_or_path = args.model @@ -84,12 +96,12 @@ def main(): enable_expert_parallel=args.enable_expert_parallel, torch_profiler_dir=args.torch_profiler_dir, compilation_config=CompilationConfig( - level = args.level, - cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes) - ) + level=args.level, + cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes), + ), ) - sampling_params = SamplingParams(temperature=0.6, max_tokens=256) + sampling_params = SamplingParams(temperature=args.temperature, max_tokens=256) prompts = [ "introduce yourself", "list all prime numbers within 100", @@ -122,6 +134,7 @@ def main(): # _ = llm.generate(warmup_prompts, sampling_params) # generate (with profiling) + # outputs_ = llm.generate(["Benchmark: "], SamplingParams()) outputs = llm.generate(prompts, sampling_params) for prompt, output in zip(prompts, outputs): diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index d3e1c21885..704e54c9ac 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -137,12 +137,11 @@ def busy_loop(self): break def _process_engine_step(self): - scheduled_batchs = self.scheduler.schedule() - out = self.runner_mgr.call_func("forward", scheduled_batchs, wait_out=True) - self.scheduler.postprocess(scheduled_batchs.seqs, out) - self.output_queue.put_nowait( - [seq for seq in scheduled_batchs.seqs if seq.is_finished] - ) + scheduled_batch = self.scheduler.schedule() + out = self.runner_mgr.call_func("forward", scheduled_batch, wait_out=True) + seqs = scheduled_batch.seqs.values() + self.scheduler.postprocess(seqs, out) + self.output_queue.put_nowait([seq for seq in seqs if seq.is_finished]) def pull_and_process_input_queue(self): recv_reqs = [] diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 75f83bf6ac..9a018944f3 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -1,21 +1,17 @@ import logging import os -import signal import time -import weakref -from typing import Optional, Union, Any -import queue -from collections import deque +from typing import Any, Optional, Union import numpy as np import torch import torch.profiler as torch_profiler import tqdm from aiter import destroy_dist_env, dtypes, init_dist_env -from aiter.dist.parallel_state import get_tensor_model_parallel_rank, graph_capture +from aiter.dist.parallel_state import graph_capture from atom.config import Config, set_current_atom_config -from atom.model_engine.scheduler import ScheduledBatchs, PrevScheduledBatchs +from atom.model_engine.scheduler import ScheduledBatch from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model from atom.model_ops.sampler import Sampler @@ -36,42 +32,31 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } -class OutputProcessor: - def __init__(self): +class tokenIDProcessor: + + def __init__(self, max_num_batched_tokens: int, device: torch.device): """Asynchronously copy the sampled_token_ids tensor to the host.""" + self.is_deferred_out = True + self.input_ids = CpuGpuBuffer( + max_num_batched_tokens, dtype=torch.int32, device=device + ) + self.input_ids_loc = CpuGpuBuffer( + max_num_batched_tokens, dtype=torch.int64, device=device + ) # Event on the copy stream so we can synchronize the non-blocking copy. self.async_copy_event = torch.cuda.Event() self.async_copy_stream = torch.cuda.Stream() - self.deferred_request_id: list[list[int]] = [] - self.deferred_token_id: list[list[int]] = [] - self.token_ids_gpu: list[torch.Tensor] = [] - self.token_ids_cpu: list[torch.Tensor] = [] - - self.token_req_ids: list[list[int]] = [] - - self.pending_outputs: deque = deque() # saving (cpu_tensor, seqs) - + self.clean() - def send_to_cpu_async(self, batch: ScheduledBatchs, gpu_tensor: torch.Tensor): + def send_to_cpu_async(self, gpu_tensor: torch.Tensor): default_stream = torch.cuda.current_stream() with torch.cuda.stream(self.async_copy_stream): self.async_copy_stream.wait_stream(default_stream) cpu_tensor = gpu_tensor.to("cpu", non_blocking=True) self.async_copy_event.record(self.async_copy_stream) self.token_ids_cpu.append(cpu_tensor) - self.pending_outputs.append((cpu_tensor, batch.seqs)) - - def revc_async_output_tuple(self) -> tuple[list[int], list[Sequence]]: - if not self.pending_outputs: - return [], [] - - self.async_copy_event.synchronize() - if self.pending_outputs: - cpu_tensor, seqs = self.pending_outputs.popleft() - token_ids = cpu_tensor.tolist() - return token_ids, seqs - return [], [] + self.token_ids_gpu.append(gpu_tensor) def recv_async_output(self) -> list[int]: self.async_copy_event.synchronize() @@ -80,18 +65,125 @@ def recv_async_output(self) -> list[int]: return token_ids return [] - def clean_token(self): + def clean(self): + self.token_ids_gpu: list[torch.Tensor] = [] self.token_ids_cpu: list[torch.Tensor] = [] - self.pending_outputs.clear() - def update_and_ret( - self, batch: ScheduledBatchs, sampled_token_ids: torch.Tensor - ) -> list[int]: + self.prev_batch: Optional[ScheduledBatch] = None + + def prepare_sampled_ids( + self, batch: ScheduledBatch, sampled_token_ids: torch.Tensor + ) -> dict[int, int]: + if not self.is_deferred_out: + token_ids = sampled_token_ids.tolist() + seq_ids = batch.seqs.keys() + return {seq_id: token_id for seq_id, token_id in zip(seq_ids, token_ids)} token_ids = self.recv_async_output() - self.send_to_cpu_async(batch, sampled_token_ids) + self.send_to_cpu_async(sampled_token_ids) + if self.prev_batch is not None: + seq_ids = self.prev_batch.seqs.keys() + token_ids = { + seq_id: token_id for seq_id, token_id in zip(seq_ids, token_ids) + } + else: + # first time, no previous tokens + token_ids = {} + + self.prev_batch = batch + self.prev_token_ids = sampled_token_ids return token_ids + def get_prev_alive_locations(self, batch: ScheduledBatch) -> tuple[list[int], bool]: + token_ids = self.prev_token_ids + deferred_prev_indices = [ + i + for i, seq_id in enumerate(self.prev_batch.seqs.keys()) + if seq_id in batch.seqs + ] + return ( + deferred_prev_indices, + len(deferred_prev_indices) == token_ids.size(0), + ) + + def prepare_input_ids( + self, + batch: ScheduledBatch, + ) -> torch.Tensor: + """Prepare the input IDs for the current batch. + + Carefully handles the `prev_sampled_token_ids` which can be cached + from the previous engine iteration, in which case those tokens on the + GPU need to be copied into the corresponding slots into input_ids.""" + seqs = list(batch.seqs.values()) + token_nums = batch.num_scheduled_tokens + total_tokens = batch.total_tokens_num + total_tokens_prefill = batch.total_tokens_num_prefill + total_tokens_decode = batch.total_tokens_num_decode + total_reqs_prefill = batch.total_seqs_num_prefill + total_reqs_decode = batch.total_seqs_num_decode + """for prefill: all input ids are new""" + start_loc = 0 + for seq, new_token_num in zip( + seqs[:total_reqs_prefill], token_nums[:total_reqs_prefill] + ): + self.input_ids.np[start_loc : start_loc + new_token_num] = seq[ + seq.num_cached_tokens : + ] + start_loc += new_token_num + self.input_ids.copy_to_gpu(total_tokens_prefill) + + # TODO: remove this when we support mixed prefill and decode in one batch + if total_reqs_prefill > 0: + return self.input_ids.gpu[:total_tokens_prefill] + + if not self.is_deferred_out: + token_ids = [ + seq.token_ids[-1] + for seq in seqs[ + total_reqs_prefill : total_reqs_prefill + total_reqs_decode + ] + ] + self.input_ids.np[:total_tokens_decode] = token_ids + self.input_ids.copy_to_gpu(total_tokens_decode) + return self.input_ids.gpu[:total_tokens_decode] + + """for decode: input ids are from prev_sampled_token_ids""" + locations, is_all_alive = self.get_prev_alive_locations(batch) + num_deferred_tokens = len(locations) + if is_all_alive: + num_norm_tokens = total_tokens_decode - num_deferred_tokens + if num_norm_tokens > 0: + token_ids = [ + seq.token_ids[-1] + for seq in seqs[ + total_reqs_prefill : total_reqs_prefill + num_norm_tokens + ] + ] + self.input_ids.np[:num_norm_tokens] = token_ids + self.input_ids.copy_to_gpu(num_norm_tokens) + # no new requests added and old requests finished + self.input_ids.gpu[ + num_norm_tokens : num_norm_tokens + num_deferred_tokens + ] = self.prev_token_ids + + elif num_deferred_tokens == total_tokens_decode: + # no new requests added but some old requests finished + self.input_ids_loc.np[:num_deferred_tokens] = locations + self.input_ids_loc.copy_to_gpu(num_deferred_tokens) + torch.gather( + self.prev_token_ids, + 0, + self.input_ids_loc.gpu[:num_deferred_tokens], + out=self.input_ids.gpu[:num_deferred_tokens], + ) + else: + # TODO: new requests' input_ids need to be filled in + assert False, "TODO new requests' input_ids need to be filled in" + # print(f"{self.input_ids.gpu[:total_tokens]=}") + return self.input_ids.gpu[:total_tokens] + + class ModelRunner: def __init__(self, rank: int, config: Config): @@ -124,61 +216,61 @@ def __init__(self, rank: int, config: Config): default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") - self.out_processor = OutputProcessor() + self.tokenID_processor = tokenIDProcessor( + self.config.max_num_batched_tokens, self.device + ) self.sampler = Sampler() - - self.prev_input_batch = PrevScheduledBatchs( - max_num_reqs=self.config.max_num_seqs, - max_model_len=self.config.max_model_len) - - self.input_ids = self._make_buffer(self.config.max_num_batched_tokens, - dtype=torch.int32) - self.positions = self._make_buffer(self.config.max_num_batched_tokens, - dtype=torch.int64) - - self.arange_np = np.arange(max(self.config.max_num_seqs + 1, - self.config.max_model_len, - self.config.max_num_batched_tokens), - dtype=np.int64) - - - self.input_ids_index_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) - self.prev_common_req_indices_tensor = CpuGpuBuffer(self.config.max_num_seqs, dtype=torch.int64, device=self.device) - - self.use_async_scheduling = True - self.async_output_copy_stream = torch.cuda.Stream() if \ - self.use_async_scheduling else None - - self.out_processor = OutputProcessor() - - + self.arange_np = np.arange( + max( + self.config.max_num_seqs + 1, + self.config.max_model_len, + self.config.max_num_batched_tokens, + ), + dtype=np.int64, + ) + self.async_output_copy_stream = torch.cuda.Stream() self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) torch.set_default_device(None) load_model(self.model, config.model, config.hf_config, config.load_dummy) torch.set_default_device("cuda") + self.allocate_forward_vars() self.warmup_model() logger.info(f"Model warmup done: {config.model}") - self.allocate_decode_vars() torch.set_default_device("cpu") torch.set_default_dtype(default_dtype) if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") - - def _make_buffer(self, - *size: Union[int, torch.SymInt], - dtype: torch.dtype, - numpy: bool = True) -> CpuGpuBuffer: + def _make_buffer( + self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True + ) -> CpuGpuBuffer: # Bfloat16 torch tensors cannot be directly cast to a numpy array, so # if a bfloat16 buffer is needed without a corresponding numpy array, # don't bother instantiating the numpy array. - return CpuGpuBuffer(*size, - dtype=dtype, - device=self.device, - pin_memory=True, - with_numpy=numpy) + return CpuGpuBuffer( + *size, dtype=dtype, device=self.device, pin_memory=True, with_numpy=numpy + ) + + def _get_cumsum_and_arange( + self, + num_tokens: np.ndarray, + cumsum_dtype: Optional[np.dtype] = None, + ) -> tuple[np.ndarray, np.ndarray]: + """Get the cumulative sum and batched arange of the given array. + # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) + # Equivalent to but faster than: + # np.concatenate([np.arange(n) for n in num_tokens]) + """ + # Step 1. [2, 5, 3] -> [2, 7, 10] + cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) + total_num_tokens = cu_num_tokens[-1] + # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] + cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) + # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] + arange = self.arange_np[:total_num_tokens] - cumsums_offsets + return cu_num_tokens, arange def exit(self): if not self.still_running: @@ -227,53 +319,53 @@ def warmup_model(self): Sequence([0] * max_model_len, block_size=self.block_size) for _ in range(num_seqs) ] - - num_scheduled_tokens: dict[str, int] = {} - - for seq in seqs: - num_scheduled_tokens[seq.id] = 1 - - total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) - dummy_batch = ScheduledBatchs(seqs, True, False, seqs, num_scheduled_tokens, total_num_scheduled_tokens) + seqs = {seq.id: seq for seq in seqs} + + num_scheduled_tokens = np.array([max_model_len] * num_seqs, dtype=np.int32) + total_tokens_num = num_scheduled_tokens.sum() + + dummy_batch = ScheduledBatch( + seqs=seqs, + num_scheduled_tokens=num_scheduled_tokens, + total_tokens_num=total_tokens_num, + total_tokens_num_prefill=total_tokens_num, + total_seqs_num=num_seqs, + total_seqs_num_prefill=num_seqs, + ) self.forward(dummy_batch) - self.out_processor.clean_token() + self.tokenID_processor.clean() torch.cuda.empty_cache() - def allocate_decode_vars(self): + def allocate_forward_vars(self): config = self.config hidden_size = config.hf_config.hidden_size hidden_type = config.hf_config.torch_dtype max_bs = min(self.config.max_num_seqs, 512) max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size - self.decode_vars = { - "input_ids": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), - "positions": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), - "slot_mapping": CpuGpuBuffer(max_bs, dtype=torch.int64, device=self.device), - "context_lens": CpuGpuBuffer(max_bs, dtype=torch.int32, device=self.device), - "block_tables": CpuGpuBuffer( - max_bs, max_num_blocks, dtype=torch.int32, device=self.device - ), - "cu_seqlens_q": CpuGpuBuffer( - max_bs + 1, dtype=torch.int32, device=self.device - ), - "kv_indptr": CpuGpuBuffer( - max_bs + 1, dtype=torch.int32, device=self.device - ), - "kv_indices": CpuGpuBuffer( - max_bs * max_num_blocks, dtype=torch.int32, device=self.device - ), - "kv_last_page_lens": CpuGpuBuffer( - max_bs, dtype=torch.int32, device=self.device - ), + max_num_batched_tokens = config.max_num_batched_tokens + i64_kwargs = {"dtype": torch.int64, "device": self.device} + i32_kwargs = {"dtype": torch.int32, "device": self.device} + f32_kwargs = {"dtype": torch.float, "device": self.device} + self.forward_vars = { + "input_ids": self.tokenID_processor.input_ids, + "positions": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), + "slot_mapping": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), + "context_lens": CpuGpuBuffer(max_bs, **i32_kwargs), + "block_tables": CpuGpuBuffer(max_bs, max_num_blocks, **i32_kwargs), + "temperatures": CpuGpuBuffer(max_bs, **f32_kwargs), + "cu_seqlens_q": CpuGpuBuffer(max_bs + 1, **i32_kwargs), + "kv_indptr": CpuGpuBuffer(max_bs + 1, **i32_kwargs), + "kv_indices": CpuGpuBuffer(max_bs * max_num_blocks, **i32_kwargs), + "kv_last_page_lens": CpuGpuBuffer(max_bs, **i32_kwargs), "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), } - self.decode_vars["cu_seqlens_q"].cpu.copy_( + self.forward_vars["cu_seqlens_q"].cpu.copy_( torch.arange(0, max_bs + 1, step=1, dtype=torch.int32) ) - self.decode_vars["cu_seqlens_q"].copy_to_gpu() + self.forward_vars["cu_seqlens_q"].copy_to_gpu() - self.decode_vars["kv_last_page_lens"].cpu.fill_(1) - self.decode_vars["kv_last_page_lens"].copy_to_gpu() + self.forward_vars["kv_last_page_lens"].cpu.fill_(1) + self.forward_vars["kv_last_page_lens"].copy_to_gpu() def get_num_blocks(self): torch.set_default_device("cuda") @@ -396,96 +488,27 @@ def allocate_kv_cache(self, num_kvcache_blocks): return True def prepare_block_tables(self, seqs: list[Sequence]): - lens = [len(seq.block_table) for seq in seqs] - block_tables = np.empty((len(seqs), max(lens)), dtype=np.int32) + block_tables = self.forward_vars["block_tables"].np for i, seq in enumerate(seqs): - block_tables[i, : lens[i]] = seq.block_table - return block_tables - - def _prepare_input_ids(self, total_num_scheduled_tokens: int, - cu_num_tokens: np.ndarray) -> None: - """Prepare the input IDs for the current batch. - - Carefully handles the `prev_sampled_token_ids` which can be cached - from the previous engine iteration, in which case those tokens on the - GPU need to be copied into the corresponding slots into input_ids.""" - - if self.prev_input_batch.prev_sampled_token_ids is None: - # Normal scheduling case - self.input_ids.copy_to_gpu(total_num_scheduled_tokens) - return - - # Async scheduling case, where some decode requests from the previous - # iteration won't have entries in input_ids_cpu and need to be copied - # on the GPU from prev_sampled_token_ids. - prev_req_id_to_index = self.prev_input_batch.prev_req_id_to_index - assert prev_req_id_to_index is not None - flattened_indices = [] - prev_common_req_indices = [] - indices_match = True - max_flattened_index = -1 - for req_id, cur_index in self.prev_input_batch.req_id_to_index.items(): - if (prev_index := prev_req_id_to_index.get(req_id)) is not None: - prev_common_req_indices.append(prev_index) - # We need to compute the flattened input_ids index of the - # last token in each common request. - flattened_index = cu_num_tokens[cur_index].item() - 1 - # flattened_indices.append(flattened_index) - flattened_indices.append(prev_index) - indices_match &= (prev_index == flattened_index) - max_flattened_index = max(max_flattened_index, flattened_index) - num_commmon_tokens = len(flattened_indices) - if num_commmon_tokens < total_num_scheduled_tokens: - # If not all requests are decodes from the last iteration, - # We need to copy the input_ids_cpu to the GPU first. - self.input_ids.copy_to_gpu(total_num_scheduled_tokens) - if num_commmon_tokens == 0: - # No requests in common with the previous iteration - # So input_ids_cpu will have all the input ids. - return - if indices_match and max_flattened_index == (num_commmon_tokens - 1): - # Common-case optimization: the batch is unchanged - # and no reordering happened. - # The indices are both the same permutation of 0..N-1 so - # we can copy directly using a single slice. - self.input_ids.gpu[:num_commmon_tokens].copy_( - self.prev_input_batch.prev_sampled_token_ids[:num_commmon_tokens], - non_blocking=True) - return - # Upload the index tensors asynchronously - # so the scatter can be non-blocking. - - self.input_ids_index_tensor.np[:num_commmon_tokens] = np.array(flattened_indices, dtype=np.int64) - self.prev_common_req_indices_tensor.np[:num_commmon_tokens] = np.array(prev_common_req_indices, dtype=np.int64) - - self.input_ids_index_tensor.copy_to_gpu(num_commmon_tokens) - self.prev_common_req_indices_tensor.copy_to_gpu(num_commmon_tokens) - - self.input_ids.gpu.scatter_( - dim=0, - index=self.input_ids_index_tensor.gpu[:num_commmon_tokens], - src=self.prev_input_batch.prev_sampled_token_ids[ - self.prev_common_req_indices_tensor.gpu[:num_commmon_tokens]]) - # prev_common_req_indices_tensor, 0]) - + block_tables[i] = 0 + block_tables[i, : seq.num_blocks] = seq.block_table - - def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): - input_ids = [] + def prepare_prefill(self, batch: ScheduledBatch): + bs = batch.total_seqs_num_prefill + sum_scheduled_tokens = batch.total_tokens_num_prefill + var = self.forward_vars positions = [] - cu_seqlens_q = [0] cu_seqlens_k = [0] max_seqlen_q = 0 max_seqlen_k = 0 slot_mapping = [] - block_tables = None - for seq in scheduled_batchs.seqs: - seqlen = len(seq) - input_ids.extend(seq[seq.num_cached_tokens :]) + seqs = list(batch.seqs.values()) + seqs = seqs[:bs] + for seq in seqs: + seqlen = seq.num_tokens positions.extend(list(range(seq.num_cached_tokens, seqlen))) seqlen_q = seqlen - seq.num_cached_tokens seqlen_k = seqlen - cu_seqlens_q.append(cu_seqlens_q[-1] + seqlen_q) cu_seqlens_k.append(cu_seqlens_k[-1] + seqlen_k) max_seqlen_q = max(seqlen_q, max_seqlen_q) max_seqlen_k = max(seqlen_k, max_seqlen_k) @@ -498,84 +521,78 @@ def prepare_prefill(self, scheduled_batchs: ScheduledBatchs): else: end = start + seq.last_block_num_tokens slot_mapping.extend(list(range(start, end))) - if cu_seqlens_k[-1] > cu_seqlens_q[-1]: # prefix cache - block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - block_tables = torch.from_numpy(block_tables).cuda(non_blocking=True) - input_ids = torch.tensor(input_ids, dtype=torch.int64, pin_memory=True) - positions = torch.tensor(positions, dtype=torch.int64, pin_memory=True) - cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) + if cu_seqlens_k[-1] > batch.total_tokens_num: # prefix cache + self.prepare_block_tables(seqs) + var["positions"].np[:sum_scheduled_tokens] = positions + var["slot_mapping"].np[: len(slot_mapping)] = slot_mapping cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int64, pin_memory=True) + min_seqlen_q = 0 dropout_p = 0.0 + vars_used = [ + ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("slot_mapping", len(slot_mapping)), + ] + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} set_context( True, - len(scheduled_batchs.seqs), - cu_seqlens_q=cu_seqlens_q.cuda(non_blocking=True), + batch_size=bs, cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), + # slot_mapping=slot_mapping.cuda(non_blocking=True), max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, - slot_mapping=slot_mapping.cuda(non_blocking=True), - block_tables=block_tables, dropout_p=dropout_p, + **ctx, ) - return input_ids.cuda(non_blocking=True), positions.cuda(non_blocking=True) - - def prepare_decode(self, scheduled_batchs: ScheduledBatchs): - bs = len(scheduled_batchs.seqs) - graph_bs = ( - bs if self.enforce_eager else next(x for x in self.graph_bs if x >= bs) + return var["positions"].copy_to_gpu(sum_scheduled_tokens) + + def prepare_decode(self, batch: ScheduledBatch): + scheduled_bs = batch.total_seqs_num_decode + seqs = list(batch.seqs.values()) + seqs = seqs[batch.total_seqs_num_prefill :] + assert len(seqs) == scheduled_bs + bs = ( + scheduled_bs + if self.enforce_eager + else next(x for x in self.graph_bs if x >= scheduled_bs) ) - assert graph_bs >= bs, f"current decode {bs=} > max graph_bs{graph_bs}" - input_ids = [] - positions = [] - slot_mapping = [] - context_lens = [] + assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" dropout_p = 0.0 max_q_len = 1 self.total_blocks = 0 - context_lens = [seq.num_tokens for seq in scheduled_batchs.seqs] - positions = np.array(context_lens, dtype=np.int64) - context_lens = np.array(context_lens, dtype=np.int32) - input_ids = [seq.last_token for seq in scheduled_batchs.seqs] - input_ids = np.array(input_ids, dtype=np.int64) + context_lens = [seq.num_tokens for seq in seqs] + positions = context_lens slot_mapping = [ seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 - for seq in scheduled_batchs.seqs + for seq in seqs ] - slot_mapping.extend([-1] * (graph_bs - bs)) - slot_mapping = np.array(slot_mapping, dtype=np.int64) + slot_mapping.extend([-1] * (bs - scheduled_bs)) if isinstance(self.model, DeepseekV2ForCausalLM): assert self.block_size == 1, "AITER MLA requires only block size 1." self.kv_indices: list[int] = [] self.kv_indptr: list[int] = [0] self.kv_last_page_lens: list[int] = [] - qlens = [1 for seq in scheduled_batchs.seqs] # TODO support mtp - num_scheduled_tokens = np.array(qlens, dtype=np.int32) - max_q_len = max(qlens) - cu_seqlens_q = np.cumsum(num_scheduled_tokens, dtype=np.int32) - for seq in scheduled_batchs.seqs: - current_seq_len = len(seq) + for seq in seqs: + current_seq_len = seq.num_tokens self._update_paged_kv_tensors(seq.block_table, current_seq_len) - block_tables = self.prepare_block_tables(scheduled_batchs.seqs) - # cu_seqlens_q = torch.tensor(cu_seqlens_q, dtype=torch.int32, pin_memory=True) - # cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens - num_input_tokens = num_scheduled_tokens - prev_input_ids_ = self.input_ids.gpu[:num_input_tokens] - - var = self.decode_vars - var["slot_mapping"].np[:graph_bs] = slot_mapping - var["input_ids"].gpu[:bs] = prev_input_ids_[:bs] - var["positions"].np[:bs] = positions - - var["context_lens"].np[:bs] = context_lens - var["block_tables"].np[:bs, : block_tables.shape[1]] = block_tables + self.prepare_block_tables(seqs) + + sum_scheduled_tokens = batch.total_tokens_num_decode + var = self.forward_vars + var["slot_mapping"].np[:bs] = slot_mapping + var["positions"].np[:sum_scheduled_tokens] = positions + var["context_lens"].np[:scheduled_bs] = context_lens + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + ("block_tables", bs), + ] if isinstance(self.model, DeepseekV2ForCausalLM) and len(self.kv_indptr) > 0: # extend to the maximum number of blocks as returned by the scheduler @@ -583,41 +600,37 @@ def prepare_decode(self, scheduled_batchs: ScheduledBatchs): var["kv_indices"].np[: self.total_blocks] = np.array( self.kv_indices, dtype=np.int64 ) - var["kv_indptr"].np[: bs + 1] = np.array(self.kv_indptr, dtype=np.int64) - var["kv_indptr"].np[bs + 1 : graph_bs + 1] = var["kv_indptr"].np[bs] + var["kv_indptr"].np[: scheduled_bs + 1] = np.array( + self.kv_indptr, dtype=np.int64 + ) + var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[bs] var["kv_last_page_lens"].np[:bs] = np.array( self.kv_last_page_lens, dtype=np.int64 ) - var["cu_seqlens_q"].np[1 : bs + 1] = np.array(cu_seqlens_q, dtype=np.int64) - - for el in [ - "slot_mapping", - # "input_ids", - "positions", - "context_lens", - "block_tables", - "cu_seqlens_q", - "kv_indices", - "kv_indptr", - "kv_last_page_lens", - ]: - var[el].copy_to_gpu() + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("kv_indices", bs), + ("kv_indptr", bs), + ("kv_last_page_lens", bs), + ] + + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} set_context( False, - batch_size=bs, - graph_bs=graph_bs, - cu_seqlens_q=var["cu_seqlens_q"].gpu[: bs + 1], - slot_mapping=var["slot_mapping"].gpu[:bs], - context_lens=var["context_lens"].gpu[:bs], - block_tables=var["block_tables"].gpu[:bs, : block_tables.shape[1]], + batch_size=scheduled_bs, + graph_bs=bs, dropout_p=dropout_p, max_q_len=max_q_len, - kv_indptr=var["kv_indptr"].gpu[: bs + 1], - kv_indices=var["kv_indices"].gpu, - kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], + **ctx, ) - return prev_input_ids_, positions - return var["input_ids"].gpu[:bs], var["positions"].gpu[:bs] + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) + # for el in ctx: + # print(f"{el}: {ctx[el]}") + # print(f"positions: {positions}") + return positions def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): # Get the number of valid blocks based on sequence length. @@ -639,145 +652,80 @@ def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): last_page_len = self.block_size self.kv_last_page_lens.append(last_page_len) - def prepare_sample(self, seqs: list[Sequence]) -> torch.Tensor: - temperatures = [seq.temperature for seq in seqs] - temperatures = torch.tensor( - temperatures, dtype=torch.float32, pin_memory=True - ).cuda(non_blocking=True) - return temperatures - - def _get_cumsum_and_arange( - self, - num_tokens: np.ndarray, - cumsum_dtype: Optional[np.dtype] = None, - ) -> tuple[np.ndarray, np.ndarray]: - """Get the cumulative sum and batched arange of the given array. - # E.g., [2, 5, 3] -> ([2, 7, 10], [0, 1, 0, 1, 2, 3, 4, 0, 1, 2]) - # Equivalent to but faster than: - # np.concatenate([np.arange(n) for n in num_tokens]) - """ - # Step 1. [2, 5, 3] -> [2, 7, 10] - cu_num_tokens = np.cumsum(num_tokens, dtype=cumsum_dtype) - total_num_tokens = cu_num_tokens[-1] - # Step 2. [2, 7, 10] -> [0, 0, 2, 2, 2, 2, 2, 7, 7, 7] - cumsums_offsets = np.repeat(cu_num_tokens - num_tokens, num_tokens) - # Step 3. [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - arange = self.arange_np[:total_num_tokens] - cumsums_offsets - - return cu_num_tokens, arange - + def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: + temperatures = [seq.temperature for seq in batch.seqs.values()] + bs = batch.total_seqs_num + buffer = self.forward_vars["temperatures"] + buffer.np[:bs] = temperatures + return buffer.copy_to_gpu(bs) - def prepare_model(self, scheduled_batchs: ScheduledBatchs): - self._update_states(scheduled_batchs) + def prepare_model(self, batch: ScheduledBatch): + total_tokens_num = batch.total_tokens_num + assert total_tokens_num > 0 + bs = batch.total_seqs_num + seqs = batch.seqs.values() - total_num_scheduled_tokens = scheduled_batchs.total_num_scheduled_tokens - assert total_num_scheduled_tokens > 0 + num_scheduled_tokens = batch.num_scheduled_tokens + cu_seqlens_q, arange = self._get_cumsum_and_arange(num_scheduled_tokens) + self.forward_vars["cu_seqlens_q"].np[1 : bs + 1] = cu_seqlens_q - _req_ids = self.prev_input_batch.req_ids - req_ids = [req_id for req_id in _req_ids if req_id is not None] + input_ids = self.tokenID_processor.prepare_input_ids(batch) - tokens = [scheduled_batchs.num_scheduled_tokens[i] for i in req_ids] - num_scheduled_tokens = np.array(tokens, dtype=np.int32) - # cu_num_tokens: [2, 5, 3] -> [2, 7, 10] - # arange: [0, 1, 0, 1, 2, 3, 4, 0, 1, 2] - cu_num_tokens, arange = self._get_cumsum_and_arange( - num_scheduled_tokens) - # cu_num_tokens = np.concatenate([np.arange(n) for n in num_scheduled_tokens]) - - self._prepare_input_ids(total_num_scheduled_tokens, cu_num_tokens) - seqs = scheduled_batchs.seqs - is_prefill = scheduled_batchs.is_prefill + is_prefill = batch.total_tokens_num_prefill > 0 prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode - input_ids, positions = prepare_func(scheduled_batchs) - temperatures = self.prepare_sample(seqs) if self.rank == 0 else None - return input_ids, positions, temperatures - - def run_model( - self, input_ids: torch.Tensor, positions: torch.Tensor, is_prefill: bool - ): - # torch.cuda.empty_cache() + positions = prepare_func(batch) + temperatures = self.prepare_sample(batch) + return ( + input_ids, + positions, + temperatures, + ) + + def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor): ctx = get_context() bs = ctx.batch_size + is_prefill = ctx.is_prefill if is_prefill or self.enforce_eager or bs > self.graph_bs[-1]: - return self.model.compute_logits(self.model(input_ids, positions)) + hidden_states = self.model(input_ids, positions) else: graph_bs = ctx.graph_bs self.graphs[graph_bs].replay() - return self.model.compute_logits(self.decode_vars["outputs"][:bs]) + hidden_states = self.forward_vars["outputs"][:bs] + return self.model.compute_logits(hidden_states) def postprocess( self, - batch: ScheduledBatchs, + batch: ScheduledBatch, logits: torch.Tensor, - temperatures: Optional[torch.Tensor], - ) -> list[int]: - if self.rank == 0: - sampled_tokens = self.sampler(logits, temperatures) - - # num_sampled_tokens = sampled_tokens.shape[0] - sampled_token_ids = sampled_tokens - if self.use_async_scheduling: - # assert sampled_token_ids.shape[-1] == 1 - - # Cache the sampled tokens on the GPU and avoid CPU sync. - # These will be copied into input_ids in the next step - # when preparing inputs. - invalid_req_indices_set = [None] - self.prev_input_batch.prev_sampled_token_ids = \ - sampled_token_ids - self.prev_input_batch.prev_req_id_to_index = { - req_id: i - for i, req_id in enumerate(self.prev_input_batch.req_ids) - if req_id not in invalid_req_indices_set - } - # plus 1 to fit next iter seq len - # context_lens = [seq.num_tokens + 1 for seq in batch.seqs] - # positions = np.array(context_lens, dtype=np.int64) - - # token_ids = sampled_tokens.tolist() - output = self.out_processor.update_and_ret( - batch, - sampled_tokens, - ) - else: - output = None - return output - - def _update_states(self, batch: ScheduledBatchs): - for req_id in self.prev_input_batch.finished_req_ids: - self.prev_input_batch.remove_request(req_id) - - for request in batch.seqs: - self.prev_input_batch.add_request(request) - - def finish_req(self, batch: ScheduledBatchs): - request_id = self.prev_input_batch.req_ids - for id in request_id: - self.prev_input_batch.finished_req_ids.add(id) - + temperatures: torch.Tensor, + ) -> dict[int, int]: + sampled_tokens = self.sampler(logits, temperatures) + token_ids = self.tokenID_processor.prepare_sampled_ids( + batch, + sampled_tokens, + ) + return token_ids @torch.inference_mode() - def forward(self, batch: ScheduledBatchs, is_wramup = False) -> list[int]: - # self._update_states(batch) + def forward(self, batch: ScheduledBatch) -> dict[int, int]: input_ids, positions, temperatures = self.prepare_model(batch) - logits = self.run_model(input_ids, positions, batch.is_prefill) + logits = self.run_model(input_ids, positions) reset_context() - self.finish_req(batch) return self.postprocess(batch, logits, temperatures) @torch.inference_mode() def capture_cudagraph(self): start_time = time.time() - input_ids = self.decode_vars["input_ids"].gpu - positions = self.decode_vars["positions"].gpu - slot_mapping = self.decode_vars["slot_mapping"].gpu - context_lens = self.decode_vars["context_lens"].gpu - block_tables = self.decode_vars["block_tables"].gpu - cu_seqlens_q = self.decode_vars["cu_seqlens_q"].gpu - kv_indptr = self.decode_vars["kv_indptr"].gpu - kv_indices = self.decode_vars["kv_indices"].gpu - kv_last_page_lens = self.decode_vars["kv_last_page_lens"].gpu - outputs = self.decode_vars["outputs"] + input_ids = self.forward_vars["input_ids"].gpu + positions = self.forward_vars["positions"].gpu + slot_mapping = self.forward_vars["slot_mapping"].gpu + context_lens = self.forward_vars["context_lens"].gpu + block_tables = self.forward_vars["block_tables"].gpu + cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu + kv_indptr = self.forward_vars["kv_indptr"].gpu + kv_indices = self.forward_vars["kv_indices"].gpu + kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu + outputs = self.forward_vars["outputs"] # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) if self.config.compilation_config.cudagraph_capture_sizes: @@ -797,12 +745,10 @@ def capture_cudagraph(self): with graph_capture() as gc: capture_range = ( - tqdm.tqdm(self.graph_bs) - if get_tensor_model_parallel_rank() == 0 - else self.graph_bs + tqdm.tqdm(self.graph_bs) if self.rank == 0 else self.graph_bs ) for bs in capture_range: - if get_tensor_model_parallel_rank() == 0: + if self.rank == 0: capture_range.set_description(f"Capturing {bs=}") graph = torch.cuda.CUDAGraph() diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 2afa33b783..146422ab63 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -1,118 +1,45 @@ +import logging from collections import deque from dataclasses import dataclass, field -from typing import Optional, cast, Any +from typing import Any, Optional, cast -from atom.config import Config -from atom.model_engine.block_manager import BlockManager -from atom.model_engine.sequence import Sequence, SequenceStatus - -import torch import numpy as np +import torch -@dataclass -class ScheduledBatchs: - seqs: list[Sequence] - is_prefill: bool - deferred_reqID: list[int] - - unfinished_prev_req: list[Sequence] - - # req_id -> num_scheduled_tokens - # Number of tokens scheduled for each request. - num_scheduled_tokens: dict[str, int] - # Total number of tokens scheduled for all requests. - # Equal to sum(num_scheduled_tokens.values()) - total_num_scheduled_tokens: int +from atom.config import Config +from atom.model_engine.block_manager import BlockManager +from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType +logger = logging.getLogger("atom") -class PrevScheduledBatchs: +class ScheduledBatch: def __init__( self, - max_num_reqs: int, - max_model_len: int + seqs: dict[int, Sequence], + num_scheduled_tokens: np.ndarray, + total_tokens_num: int, + total_tokens_num_prefill: int = 0, + total_tokens_num_decode: int = 0, + total_seqs_num: int = 0, + total_seqs_num_prefill: int = 0, + total_seqs_num_decode: int = 0, ): - self.max_num_reqs = max_num_reqs - self.max_model_len = max_model_len - self.token_ids_cpu_tensor = torch.zeros( - (max_num_reqs, max_model_len), - device="cpu", - dtype=torch.int32, - pin_memory=False, - ) - self.token_ids_cpu = self.token_ids_cpu_tensor.numpy() - self.num_tokens = np.zeros(max_num_reqs, dtype=np.int32) - self.req_id_to_index: dict[str, int] = {} - - # Cached reference to the GPU tensor of previously sampled tokens - self.prev_sampled_token_ids: Optional[torch.Tensor] = None - self.prev_sampled_token_ids_invalid_indices: Optional[set[int]] = None - self.prev_req_id_to_index: Optional[dict[str, int]] = None - self.prev_position_ids: Optional[torch.Tensor] = None - - self._req_ids: list[Optional[str]] = [] - self.req_output_token_ids: list[Optional[list[int]]] = [] - - self.finished_req_ids: set[str] = set() - - @property - def num_reqs(self) -> int: - return len(self.req_id_to_index) - - def _register_add_request(self, request: "Sequence") -> int: - """Track add-request operations for logits processors. - Not applicable to pooling models. - """ - - new_req_index = self.num_reqs - - assert new_req_index < self.max_num_reqs - return new_req_index - - - def add_request( - self, - request: "Sequence", - ) -> int: - req_index = self._register_add_request(request) - - req_id = request.id - - if req_index == len(self._req_ids): - self._req_ids.append(req_id) - # self.req_output_token_ids.append(request.output_token_ids) - else: - self._req_ids[req_index] = req_id - # self.req_output_token_ids[req_index] = request.output_token_ids - self.req_id_to_index[req_id] = req_index - - # new_req_index = request.last_token - # self.req_id_to_index[req_id] = new_req_index - - # self._req_ids.append(req_id) + # len(seqs) == total_seqs_num == total_seqs_num_prefill + total_seqs_num_decode + self.seqs = seqs - def remove_request(self, req_id: str) -> Optional[int]: - """This method must always be followed by a call to condense(). + # num_scheduled_tokens for each sequence in the batch + self.num_scheduled_tokens = num_scheduled_tokens - Args: - req_id: request to remove + # Total number of tokens scheduled for all requests. + self.total_tokens_num = total_tokens_num + self.total_tokens_num_prefill = total_tokens_num_prefill + self.total_tokens_num_decode = total_tokens_num_decode - Returns: - Removed request index, or `None` if `req_id` not recognized - """ - req_index = self.req_id_to_index.pop(req_id, None) - if req_index is None: - return None - - self._req_ids[req_index] = None - # self.req_output_token_ids[req_index] = None - - - @property - def req_ids(self) -> list[str]: - # None elements should only be present transiently - # while performing state updates to the batch. - return cast(list[str], self._req_ids) + # Total number of reqs scheduled for all requests. + self.total_seqs_num = total_seqs_num + self.total_seqs_num_prefill = total_seqs_num_prefill + self.total_seqs_num_decode = total_seqs_num_decode class Scheduler: @@ -125,7 +52,6 @@ def __init__(self, config: Config): self.block_manager = BlockManager(config) self.waiting: deque[Sequence] = deque() self.running: deque[Sequence] = deque() - self.deferred_reqID = [] def is_finished(self): return not self.waiting and not self.running @@ -136,45 +62,55 @@ def add(self, seq: Sequence): def extend(self, seqs: list[Sequence]): self.waiting.extend(seqs) - def schedule(self) -> ScheduledBatchs: + def schedule(self) -> ScheduledBatch: # prefill - scheduled_seqs = [] - num_seqs = 0 + scheduled_seqs = {} + num_seqs_prefill = 0 num_batched_tokens = 0 - num_scheduled_tokens: dict[str, int] = {} + num_scheduled_tokens: list[int] = [] if not self.running and not self.waiting: self.block_manager.reset() - while self.waiting and num_seqs < self.max_num_seqs: + while self.waiting and num_seqs_prefill < self.max_num_seqs: seq = self.waiting[0] - if num_batched_tokens + len( - seq - ) > self.max_num_batched_tokens or not self.block_manager.can_allocate(seq): + num_new_tokens = seq.num_tokens - seq.num_cached_tokens + if ( + num_batched_tokens + num_new_tokens > self.max_num_batched_tokens + or not self.block_manager.can_allocate(seq) + ): break - num_seqs += 1 + num_seqs_prefill += 1 self.block_manager.allocate(seq) - num_batched_tokens += len(seq) - seq.num_cached_tokens + num_batched_tokens += num_new_tokens seq.status = SequenceStatus.RUNNING + seq.type = SequenceType.PREFILL self.waiting.popleft() self.running.append(seq) - scheduled_seqs.append(seq) - # in vllm, using num_new_tokens - num_new_tokens = 1 - num_scheduled_tokens[seq.id] = num_new_tokens - - total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) - - if scheduled_seqs: - return ScheduledBatchs( - seqs=scheduled_seqs, is_prefill=True, deferred_reqID=[], unfinished_prev_req=scheduled_seqs, - num_scheduled_tokens=num_scheduled_tokens, - total_num_scheduled_tokens=total_num_scheduled_tokens, + scheduled_seqs[seq.id] = seq + num_scheduled_tokens.append(num_new_tokens) + + num_scheduled_tokens_np = np.array(num_scheduled_tokens, dtype=np.int32) + total_tokens_num_prefill = num_scheduled_tokens_np.sum() + + if num_seqs_prefill > 0: + logger.info( + f"scheduled prefill batch: {num_seqs_prefill} reqs, {total_tokens_num_prefill} tokens" + ) + # lip: TODO for prefill/decode mixed batch + return ScheduledBatch( + seqs=scheduled_seqs, + num_scheduled_tokens=num_scheduled_tokens_np, + total_tokens_num=total_tokens_num_prefill, + total_tokens_num_prefill=total_tokens_num_prefill, + total_seqs_num=num_seqs_prefill, + total_seqs_num_prefill=num_seqs_prefill, ) # decode - while self.running and num_seqs < self.max_num_seqs: + num_seqs_decode = 0 + while self.running and num_seqs_decode < self.max_num_seqs: seq = self.running.popleft() while not self.block_manager.can_append(seq): if self.running: @@ -183,23 +119,29 @@ def schedule(self) -> ScheduledBatchs: self.preempt(seq) break else: - num_seqs += 1 + num_seqs_decode += 1 self.block_manager.may_append(seq) - scheduled_seqs.append(seq) - num_new_tokens = 1 - num_scheduled_tokens[seq.id] = num_new_tokens - - total_num_scheduled_tokens = sum(num_scheduled_tokens.values()) + scheduled_seqs[seq.id] = seq + seq.type = SequenceType.DECODE + num_new_tokens = 1 + num_scheduled_tokens.append(num_new_tokens) + + num_scheduled_tokens_np = np.array(num_scheduled_tokens, dtype=np.int32) + total_tokens_num_decode = num_scheduled_tokens_np.sum() assert scheduled_seqs - self.running.extendleft(reversed(scheduled_seqs)) - return ScheduledBatchs( + self.running.extendleft(reversed(scheduled_seqs.values())) + # logger.info( + # f"Scheduled decode batch: {num_seqs_decode} reqs, {total_tokens_num_decode} tokens" + # ) + return ScheduledBatch( seqs=scheduled_seqs, - is_prefill=False, - deferred_reqID=self.deferred_reqID, - unfinished_prev_req=scheduled_seqs, - num_scheduled_tokens=num_scheduled_tokens, - total_num_scheduled_tokens=total_num_scheduled_tokens, + num_scheduled_tokens=num_scheduled_tokens_np, + total_tokens_num=total_tokens_num_decode, + total_tokens_num_decode=total_tokens_num_decode, + total_seqs_num=num_seqs_prefill + num_seqs_decode, + total_seqs_num_prefill=num_seqs_prefill, + total_seqs_num_decode=num_seqs_decode, ) def preempt(self, seq: Sequence): @@ -207,40 +149,40 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: list[int]): - - # # placeholder for the each decode step - token_ids = [self.eos_token_id] * len(seqs) - for seq, token_id in zip(seqs, token_ids): - seq.append_token(token_id) - - if not prev_token_ids: - return - + def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): + is_deferred_out = True # update token_ids with the actual sampled token ids - self.unfinished_req = [] - token_ids = prev_token_ids - for i, (seq, token_id) in enumerate(zip(seqs, token_ids)): - # seq.append_token(token_id) - seq.token_ids[-2] = token_id + finished_seqs = [] + for seq in self.running: + if seq.id not in prev_token_ids: + continue + token_id = prev_token_ids[seq.id] + if is_deferred_out: + seq.token_ids[-1] = token_id + else: + seq.append_token(token_id) + leave_reason = None - # Check if sequence ends with any stop sequence - stop_matched = False for stop_seq in seq.stop_token_sequences: if len(seq.token_ids) >= len(stop_seq): - if seq.token_ids[-len(stop_seq):] == stop_seq: - stop_matched = True + if seq.token_ids[-len(stop_seq) :] == stop_seq: + leave_reason = "stop_sequence" break - - if (not seq.ignore_eos and token_id == self.eos_token_id) or stop_matched: - leave_reason = "eos" - elif seq.num_completion_tokens == seq.max_tokens: - leave_reason = "max_tokens" + else: + if not seq.ignore_eos and token_id == self.eos_token_id: + leave_reason = "eos" + elif seq.num_completion_tokens == seq.max_tokens: + leave_reason = "max_tokens" if leave_reason is not None: seq.leave_reason = leave_reason seq.status = SequenceStatus.FINISHED - self.block_manager.deallocate(seq) - self.running.remove(seq) - else: - self.unfinished_req.append(i) + finished_seqs.append(seq) + if is_deferred_out: + # placeholder for the each decode step + for seq in seqs: + if seq.status == SequenceStatus.RUNNING: + seq.append_token(self.eos_token_id) + for seq in finished_seqs: + self.block_manager.deallocate(seq) + self.running.remove(seq) diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index da5abdcc0b..64ab59b5ab 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -12,6 +12,12 @@ class SequenceStatus(Enum): EXIT_ENGINE = auto() +class SequenceType(Enum): + DUMMY = auto() + PREFILL = auto() + DECODE = auto() + + def get_exit_sequence(): exit_seq = Sequence([-1], 1) exit_seq.status = SequenceStatus.EXIT_ENGINE @@ -27,6 +33,7 @@ def __init__( self.block_size = block_size self.id = next(Sequence.counter) self.status = SequenceStatus.WAITING + self.type = SequenceType.DUMMY self.token_ids = copy(token_ids) self.last_token = token_ids[-1] self.num_tokens = len(self.token_ids) @@ -52,6 +59,7 @@ def __getitem__(self, key): @property def num_tokens(self): + '''The total number of tokens in the sequence. i.e. prompt + completion''' return self._num_tokens @num_tokens.setter diff --git a/atom/utils/context.py b/atom/utils/context.py index 84e15c3089..2038ebcdaf 100644 --- a/atom/utils/context.py +++ b/atom/utils/context.py @@ -24,6 +24,7 @@ class Context: kv_indices: Optional[torch.Tensor] = None kv_last_page_lens: Optional[torch.Tensor] = None + _CONTEXT = Context() From 0ab11a6cdc3507c3c157fe816eac75dffe784c41 Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Tue, 21 Oct 2025 22:03:44 +0800 Subject: [PATCH 089/119] fix model_runner (#17) Co-authored-by: amd-ruitang3 --- atom/model_engine/model_runner.py | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 9a018944f3..57d17e4b3d 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -604,16 +604,17 @@ def prepare_decode(self, batch: ScheduledBatch): self.kv_indptr, dtype=np.int64 ) var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[bs] - var["kv_last_page_lens"].np[:bs] = np.array( + var["kv_last_page_lens"].np[:scheduled_bs] = np.array( self.kv_last_page_lens, dtype=np.int64 ) + var["kv_last_page_lens"].np[scheduled_bs:bs] = 0 vars_used = [ ("slot_mapping", bs), # TODO: MTP support ("context_lens", bs), ("block_tables", bs), ("cu_seqlens_q", bs + 1), - ("kv_indices", bs), - ("kv_indptr", bs), + ("kv_indices", sum(context_lens)), + ("kv_indptr", bs + 1), ("kv_last_page_lens", bs), ] @@ -716,17 +717,6 @@ def forward(self, batch: ScheduledBatch) -> dict[int, int]: @torch.inference_mode() def capture_cudagraph(self): start_time = time.time() - input_ids = self.forward_vars["input_ids"].gpu - positions = self.forward_vars["positions"].gpu - slot_mapping = self.forward_vars["slot_mapping"].gpu - context_lens = self.forward_vars["context_lens"].gpu - block_tables = self.forward_vars["block_tables"].gpu - cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu - kv_indptr = self.forward_vars["kv_indptr"].gpu - kv_indices = self.forward_vars["kv_indices"].gpu - kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu - outputs = self.forward_vars["outputs"] - # self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16)) if self.config.compilation_config.cudagraph_capture_sizes: self.graph_bs = self.config.compilation_config.cudagraph_capture_sizes @@ -740,6 +730,21 @@ def capture_cudagraph(self): self.graph_bs = cuda_graph_sizes self.graph_bs.sort(reverse=True) + self.forward_vars["cu_seqlens_q"].np[: self.graph_bs[0] + 1] = np.arange( + 0, self.graph_bs[0] + 1 + ) + self.forward_vars["cu_seqlens_q"].copy_to_gpu(self.graph_bs[0] + 1) + input_ids = self.forward_vars["input_ids"].gpu + positions = self.forward_vars["positions"].gpu + slot_mapping = self.forward_vars["slot_mapping"].gpu + context_lens = self.forward_vars["context_lens"].gpu + block_tables = self.forward_vars["block_tables"].gpu + cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu + kv_indptr = self.forward_vars["kv_indptr"].gpu + kv_indices = self.forward_vars["kv_indices"].gpu + kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu + outputs = self.forward_vars["outputs"] + self.graphs: dict[int, torch.cuda.CUDAGraph] = dict() self.graph_pool = None From 608216b4d149a7e8d115463a35cba0cecbdd0aac Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Thu, 23 Oct 2025 22:11:45 +0800 Subject: [PATCH 090/119] add barrier to avoid get_context thread error --- atom/model_engine/model_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 57d17e4b3d..d08337ed39 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -485,6 +485,8 @@ def allocate_kv_cache(self, num_kvcache_blocks): ) set_forward_context(module.layer_num, attention_metadata) layer_id += 1 + if torch.distributed.is_initialized(): + torch.distributed.barrier() return True def prepare_block_tables(self, seqs: list[Sequence]): From 84fe02b7b06cb280a5d874e34f821dc582d63aa6 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Sat, 25 Oct 2025 19:04:04 +0800 Subject: [PATCH 091/119] fix level 1 acc issue --- atom/model_engine/model_runner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index d08337ed39..11c199ebbf 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -230,8 +230,11 @@ def __init__(self, rank: int, config: Config): ) self.async_output_copy_stream = torch.cuda.Stream() self.model = suppot_model_arch_dict[hf_config.architectures[0]](config) + self.use_kv_indptr = False torch.set_default_device(None) load_model(self.model, config.model, config.hf_config, config.load_dummy) + if isinstance(self.model, DeepseekV2ForCausalLM): + self.use_kv_indptr = True torch.set_default_device("cuda") self.allocate_forward_vars() self.warmup_model() @@ -573,7 +576,7 @@ def prepare_decode(self, batch: ScheduledBatch): ] slot_mapping.extend([-1] * (bs - scheduled_bs)) - if isinstance(self.model, DeepseekV2ForCausalLM): + if self.use_kv_indptr: assert self.block_size == 1, "AITER MLA requires only block size 1." self.kv_indices: list[int] = [] self.kv_indptr: list[int] = [0] @@ -596,7 +599,7 @@ def prepare_decode(self, batch: ScheduledBatch): ("block_tables", bs), ] - if isinstance(self.model, DeepseekV2ForCausalLM) and len(self.kv_indptr) > 0: + if self.use_kv_indptr and len(self.kv_indptr) > 0: # extend to the maximum number of blocks as returned by the scheduler self.kv_indices.extend([0] * (self.total_blocks - len(self.kv_indices))) var["kv_indices"].np[: self.total_blocks] = np.array( From 1d09c8e79fbaf290cd5218bfeec52c16807b22b4 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 26 Oct 2025 15:21:37 +0800 Subject: [PATCH 092/119] replace SharedMemory by MessageQueue --- atom/model_engine/async_proc.py | 64 +++++++++++++-------------------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/atom/model_engine/async_proc.py b/atom/model_engine/async_proc.py index f61b34f6cd..77f400d29e 100644 --- a/atom/model_engine/async_proc.py +++ b/atom/model_engine/async_proc.py @@ -6,14 +6,12 @@ import threading import weakref from contextlib import ExitStack -from multiprocessing.shared_memory import SharedMemory -from multiprocessing.synchronize import Event from threading import Thread from typing import List import zmq import zmq.asyncio - +from aiter.dist.shm_broadcast import MessageQueue from atom.utils import ( get_mp_context, get_open_zmq_ipc_path, @@ -31,26 +29,21 @@ class AsyncIOProc: def __init__( self, label: str, - label_shm: str, io_addrs: tuple[str, str], - sync_event: Event, + input_shm_handle: int, # runner class and its args runner_qualname: str, # atom.model_engine.model_runner.ModelRunner + rank: int, *args, **kwargs, ): self.label = f"AsyncIOProc({label})" - self.sync_event = sync_event self.io_addrs = io_addrs self.io_queues = queue.Queue(), queue.Queue() self.io_threads = [] - self.shm = SharedMemory(name=f"atom_{label_shm}_shm") - init_exit_handler(self) - + self.rpc_broadcast_mq = MessageQueue.create_from_handle(input_shm_handle, rank) # make sure exit handler is set before runner is created - runner_class = resolve_obj_by_qualname(runner_qualname) # type: ignore - self.runners = [] - self.runners.append(runner_class(*args, **kwargs)) + init_exit_handler(self) for addr, q, func in zip( self.io_addrs, self.io_queues, @@ -61,6 +54,10 @@ def __init__( t = threading.Thread(target=func, args=(addr, q), daemon=True) t.start() self.io_threads.append(t) + + runner_class = resolve_obj_by_qualname(runner_qualname) # type: ignore + self.runners = [] + self.runners.append(runner_class(rank, *args, **kwargs)) self.busy_loop() def exit(self): @@ -70,7 +67,6 @@ def exit(self): logger.debug(f"{self.label}: Shutting down runner...") for el in self.runners: el.exit() - self.shm.close() for t in self.io_threads: t.join(timeout=0.5) @@ -120,10 +116,7 @@ def busy_loop(self): logger.debug(f"{self.label}: exit busy_loop...") def get_func(self): - self.sync_event.wait() - n = int.from_bytes(self.shm.buf[0:4], "little") - method_name, *args = pickle.loads(self.shm.buf[4 : n + 4]) - self.sync_event.clear() + method_name, *args = self.rpc_broadcast_mq.dequeue() return method_name, args @@ -133,28 +126,35 @@ def __init__(self, finalizer, proc_num: int, runner: str, *args): self.parent_finalizer = finalizer io_addrs = [get_open_zmq_ipc_path(), get_open_zmq_ipc_path()] self.procs = [] - self.sync_events = [] ctx = get_mp_context() self.mp_ctx = ctx self.runner_label = runner.split(".")[-1] self.label = f"AsyncIOProcManager({self.runner_label})" - self.create_shared_memory() + self.rpc_broadcast_mq = MessageQueue( + proc_num, proc_num, max_chunk_bytes=16 * 1024 * 1024 + ) + scheduler_output_handle = self.rpc_broadcast_mq.export_handle() self.still_running = True init_exit_handler(self) for i in range(proc_num): label = f"ModelRunner{i}/{proc_num}" addrs = ( - io_addrs if i == 0 else [io_addrs[0], None] + [None, io_addrs[1]] if i == 0 else [None, None] ) # only get output from rank 0 - sync_event = ctx.Event() process = ctx.Process( target=AsyncIOProc, name=label, - args=(label, self.runner_label, addrs, sync_event, runner, i, *args), + args=( + label, + addrs, + scheduler_output_handle, + runner, + i, + *args, + ), ) process.start() self.procs.append(process) - self.sync_events.append(sync_event) self.zmq_ctx = zmq.Context(io_threads=2) self.outputs_queue = queue.Queue() @@ -167,16 +167,6 @@ def __init__(self, finalizer, proc_num: int, runner: str, *args): self.output_thread.start() self.monitor_procs() - def create_shared_memory(self): - name = f"atom_{self.runner_label}_shm" - try: - self.shm = SharedMemory(name=name, create=True, size=2**20) - except FileExistsError: - existing_shm = SharedMemory(name=name) - existing_shm.close() - existing_shm.unlink() - self.shm = SharedMemory(name=name, create=True, size=2**20) - def exit(self): if not self.still_running: return @@ -184,7 +174,6 @@ def exit(self): # 1. kill all runners logger.info(f"{self.label}: shutdown all runners...") shutdown_all_processes(self.procs, allowed_seconds=1) - self.shm.unlink() self.procs = [] self.output_thread.join() logger.info(f"{self.label}: All runners are shutdown.") @@ -213,11 +202,8 @@ def process_output_sockets(self, output_address: str): def call_func(self, func_name: str, *args, wait_out: bool = False): logger.debug(f"{self.label}: call_func {func_name} {args}") - n = pickle.dumps((func_name, *args)) - self.shm.buf[0:4] = len(n).to_bytes(4, "little") - self.shm.buf[4 : len(n) + 4] = n - for ev in self.sync_events: - ev.set() + msg = (func_name, *args) + self.rpc_broadcast_mq.enqueue(msg) if wait_out: ret = self.outputs_queue.get() if isinstance(ret, SystemExit): From db4ecdf65b54d583f773ac8fb4b29e5099554ef0 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 26 Oct 2025 02:27:56 -0500 Subject: [PATCH 093/119] fix kv_indptr --- atom/model_engine/model_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 11c199ebbf..157091f37d 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -608,7 +608,9 @@ def prepare_decode(self, batch: ScheduledBatch): var["kv_indptr"].np[: scheduled_bs + 1] = np.array( self.kv_indptr, dtype=np.int64 ) - var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[bs] + var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[ + scheduled_bs + ] var["kv_last_page_lens"].np[:scheduled_bs] = np.array( self.kv_last_page_lens, dtype=np.int64 ) From 73e51504977dbe2378efea23f4c588c1bb1653cd Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 26 Oct 2025 15:34:52 +0800 Subject: [PATCH 094/119] minor update --- atom/model_engine/block_manager.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/block_manager.py b/atom/model_engine/block_manager.py index 6df703bedb..50081bc204 100644 --- a/atom/model_engine/block_manager.py +++ b/atom/model_engine/block_manager.py @@ -2,7 +2,6 @@ import numpy as np import xxhash - from atom.config import Config from atom.model_engine.sequence import Sequence @@ -54,10 +53,11 @@ def _allocate_block(self, block_id: int) -> Block: self.used_block_ids.add(block_id) return self.blocks[block_id] - def _deallocate_block(self, block_id: int) -> Block: + def _deallocate_block(self, block_id: int): assert self.blocks[block_id].ref_count == 0 self.used_block_ids.remove(block_id) self.free_block_ids.append(block_id) + # self.free_block_ids.appendleft(block_id) def can_allocate(self, seq: Sequence) -> bool: return len(self.free_block_ids) >= seq.num_blocks @@ -115,7 +115,9 @@ def may_append(self, seq: Sequence): block_table.append(block_id) if self.block_size == 1: token_ids = [seq[-1]] - prefix = self.blocks[block_table[-2]].hash if len(block_table) > 1 else -1 + prefix = ( + self.blocks[block_table[-2]].hash if len(block_table) > 1 else -1 + ) h = self.compute_hash(token_ids, prefix) block.update(h, token_ids) self.hash_to_block_id[h] = block_id From 1301219d2d2ed30decfd25a1ef3fafbf7d48d484 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sun, 26 Oct 2025 21:10:56 +0800 Subject: [PATCH 095/119] refine --- atom/model_engine/model_runner.py | 14 +++++++++++--- atom/model_engine/scheduler.py | 3 +-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 157091f37d..e10123d9a9 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -9,7 +9,6 @@ import tqdm from aiter import destroy_dist_env, dtypes, init_dist_env from aiter.dist.parallel_state import graph_capture - from atom.config import Config, set_current_atom_config from atom.model_engine.scheduler import ScheduledBatch from atom.model_engine.sequence import Sequence @@ -31,12 +30,16 @@ "MixtralForCausalLM": MixtralForCausalLM, "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, } +# seed=1 +# np.random.seed(seed) +# torch.cuda.manual_seed_all(seed) class tokenIDProcessor: def __init__(self, max_num_batched_tokens: int, device: torch.device): """Asynchronously copy the sampled_token_ids tensor to the host.""" + # self.is_deferred_out = False self.is_deferred_out = True self.input_ids = CpuGpuBuffer( max_num_batched_tokens, dtype=torch.int32, device=device @@ -77,7 +80,9 @@ def prepare_sampled_ids( if not self.is_deferred_out: token_ids = sampled_token_ids.tolist() seq_ids = batch.seqs.keys() - return {seq_id: token_id for seq_id, token_id in zip(seq_ids, token_ids)} + ret = {seq_id: token_id for seq_id, token_id in zip(seq_ids, token_ids)} + ret[-1] = 0 + return ret token_ids = self.recv_async_output() self.send_to_cpu_async(sampled_token_ids) @@ -92,6 +97,7 @@ def prepare_sampled_ids( self.prev_batch = batch self.prev_token_ids = sampled_token_ids + token_ids[-1] = 1 return token_ids def get_prev_alive_locations(self, batch: ScheduledBatch) -> tuple[list[int], bool]: @@ -180,7 +186,6 @@ def prepare_input_ids( else: # TODO: new requests' input_ids need to be filled in assert False, "TODO new requests' input_ids need to be filled in" - # print(f"{self.input_ids.gpu[:total_tokens]=}") return self.input_ids.gpu[:total_tokens] @@ -678,6 +683,7 @@ def prepare_model(self, batch: ScheduledBatch): self.forward_vars["cu_seqlens_q"].np[1 : bs + 1] = cu_seqlens_q input_ids = self.tokenID_processor.prepare_input_ids(batch) + # print(f"{input_ids=}") is_prefill = batch.total_tokens_num_prefill > 0 prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode @@ -708,6 +714,8 @@ def postprocess( temperatures: torch.Tensor, ) -> dict[int, int]: sampled_tokens = self.sampler(logits, temperatures) + # print(f"{logits=}") + # print(f"{sampled_tokens=}") token_ids = self.tokenID_processor.prepare_sampled_ids( batch, sampled_tokens, diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 146422ab63..ad6bdfc5c1 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -5,7 +5,6 @@ import numpy as np import torch - from atom.config import Config from atom.model_engine.block_manager import BlockManager from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType @@ -150,7 +149,7 @@ def preempt(self, seq: Sequence): self.waiting.appendleft(seq) def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): - is_deferred_out = True + is_deferred_out = prev_token_ids[-1] # update token_ids with the actual sampled token ids finished_seqs = [] for seq in self.running: From f8f2049fc588523bdd83c54406aa7979e39056d1 Mon Sep 17 00:00:00 2001 From: ZhangLirong Date: Mon, 27 Oct 2025 22:44:34 +0800 Subject: [PATCH 096/119] Support piecewise deepseek and different attn backend (#19) * support piecewise deepseek in torch compile * support mla and attn backend for prefill and decode * rename common attn * remove _attention_ops in split * add decorator for split op * refine * remove block table in mla --- atom/config.py | 34 +--- atom/model_engine/model_runner.py | 173 +++---------------- atom/model_ops/attention.py | 35 ++-- atom/model_ops/attention_mla.py | 107 ++++++------ atom/model_ops/attentions/aiter_attention.py | 68 ++++++++ atom/model_ops/attentions/aiter_mla.py | 122 +++++++++++++ atom/model_ops/attentions/backends.py | 137 +++++++++++++++ atom/utils/__init__.py | 38 ++++ atom/utils/backends.py | 3 +- atom/utils/decorators.py | 3 - atom/utils/selector.py | 74 ++++++++ 11 files changed, 544 insertions(+), 250 deletions(-) create mode 100644 atom/model_ops/attentions/aiter_attention.py create mode 100644 atom/model_ops/attentions/aiter_mla.py create mode 100644 atom/model_ops/attentions/backends.py create mode 100644 atom/utils/selector.py diff --git a/atom/config.py b/atom/config.py index a451b63813..3083e0fa2c 100644 --- a/atom/config.py +++ b/atom/config.py @@ -10,17 +10,6 @@ from aiter.utility.dtypes import d_dtypes from transformers import AutoConfig, PretrainedConfig -CUDA_CAPTURE = True - - -def get_capture_status() -> bool: - return CUDA_CAPTURE - - -def set_capture_status(capture): - global CUDA_CAPTURE - CUDA_CAPTURE = capture - class CUDAGraphMode(enum.Enum): """Constants for the cudagraph mode in CompilationConfig. @@ -154,16 +143,6 @@ class CompilationConfig: Note that this flag is only effective when cudagraph_mode is PIECEWISE. """ - _attention_ops: ClassVar[list[str]] = [ - "aiter.unified_attention_with_output" - # "aiter.wrapper_fmha_v3_varlen_fwd", - # "vllm.unified_attention", - # "vllm.unified_attention_with_output", - # "vllm.mamba_mixer2", - # "vllm.mamba_mixer", - # "vllm.short_conv", - # "vllm.linear_attention", - ] inductor_compile_config: dict = field(default_factory=dict) """Additional configurations for inductor. @@ -217,6 +196,7 @@ def compute_hash(self) -> str: factors.append(self.local_cache_dir) factors.append(self.cudagraph_capture_sizes) factors.append(self.cuda_graph_sizes) + return hashlib.sha256(str(factors).encode()).hexdigest() def set_splitting_ops_for_v1(self): @@ -228,13 +208,10 @@ def set_splitting_ops_for_v1(self): ) if self.splitting_ops is None: - # NOTE: When using full cudagraph, instead of setting an empty - # list and capture the full cudagraph inside the flattened fx - # graph, we keep the piecewise fx graph structure but capture the - # full cudagraph outside the fx graph. This reduces some cpu - # overhead when the runtime batch_size is not cudagraph captured. - # see https://github.com/vllm-project/vllm/pull/20059 for details. - self.splitting_ops = self._attention_ops + self.splitting_ops = [ + "aiter.unified_attention_with_output", + "aiter.mla_attention", + ] class QuantizationConfig(dict): @@ -420,6 +397,7 @@ def compute_hash(self) -> str: vllm_factors.append(self.compilation_config.compute_hash()) factors.append(vllm_factors) + factors.append(self.tensor_parallel_size) hash_str = hashlib.md5( str(factors).encode(), usedforsecurity=False diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index e10123d9a9..a7b4e029e5 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -18,8 +18,9 @@ from atom.models.llama import LlamaForCausalLM from atom.models.mixtral import MixtralForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM -from atom.utils import CpuGpuBuffer, init_exit_handler +from atom.utils import CpuGpuBuffer, init_exit_handler, get_hf_text_config from atom.utils.context import get_context, reset_context, set_context +from atom.utils.selector import get_attn_backend logger = logging.getLogger("atom") from atom.utils.forward_context import AttentionMetadata, set_forward_context @@ -200,6 +201,13 @@ def __init__(self, rank: int, config: Config): self.world_size = config.tensor_parallel_size self.rank = rank self.label = f"Model Runner{rank}/{self.world_size}" + self.hf_text_config = get_hf_text_config(hf_config) + self.use_mla = self.is_deepseek_mla() + self.attn_backend = get_attn_backend( + self.block_size, + use_mla=self.use_mla, + ) + self.attn_metadata_builder = self.attn_backend.get_builder_cls()(self.block_size) # Initialize profiler for this rank self.profiler = None @@ -250,6 +258,21 @@ def __init__(self, rank: int, config: Config): if self.config.compilation_config.level == 1: self.model = torch.compile(self.model, fullgraph=True, backend="eager") + def is_deepseek_mla(self) -> bool: + if not hasattr(self.hf_text_config, "model_type"): + return False + elif self.hf_text_config.model_type in \ + ('deepseek_v2', 'deepseek_v3', 'deepseek_mtp'): + return self.hf_text_config.kv_lora_rank is not None + elif self.hf_text_config.model_type == 'eagle': + # if the model is an EAGLE module, check for the + # underlying architecture + return self.hf_text_config.model.model_type in \ + ('deepseek_v2', 'deepseek_v3') \ + and self.hf_text_config.kv_lora_rank is not None + return False + + def _make_buffer( self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True ) -> CpuGpuBuffer: @@ -504,59 +527,8 @@ def prepare_block_tables(self, seqs: list[Sequence]): block_tables[i, : seq.num_blocks] = seq.block_table def prepare_prefill(self, batch: ScheduledBatch): - bs = batch.total_seqs_num_prefill - sum_scheduled_tokens = batch.total_tokens_num_prefill - var = self.forward_vars - positions = [] - cu_seqlens_k = [0] - max_seqlen_q = 0 - max_seqlen_k = 0 - slot_mapping = [] - seqs = list(batch.seqs.values()) - seqs = seqs[:bs] - for seq in seqs: - seqlen = seq.num_tokens - positions.extend(list(range(seq.num_cached_tokens, seqlen))) - seqlen_q = seqlen - seq.num_cached_tokens - seqlen_k = seqlen - cu_seqlens_k.append(cu_seqlens_k[-1] + seqlen_k) - max_seqlen_q = max(seqlen_q, max_seqlen_q) - max_seqlen_k = max(seqlen_k, max_seqlen_k) - if not seq.block_table: - continue - for i in range(seq.num_cached_blocks, seq.num_blocks): - start = seq.block_table[i] * self.block_size - if i != seq.num_blocks - 1: - end = start + self.block_size - else: - end = start + seq.last_block_num_tokens - slot_mapping.extend(list(range(start, end))) - if cu_seqlens_k[-1] > batch.total_tokens_num: # prefix cache - self.prepare_block_tables(seqs) - var["positions"].np[:sum_scheduled_tokens] = positions - var["slot_mapping"].np[: len(slot_mapping)] = slot_mapping - cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) - - min_seqlen_q = 0 - dropout_p = 0.0 - vars_used = [ - ("block_tables", bs), - ("cu_seqlens_q", bs + 1), - ("slot_mapping", len(slot_mapping)), - ] - ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} - set_context( - True, - batch_size=bs, - cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), - # slot_mapping=slot_mapping.cuda(non_blocking=True), - max_seqlen_q=max_seqlen_q, - max_seqlen_k=max_seqlen_k, - min_seqlen_q=min_seqlen_q, - dropout_p=dropout_p, - **ctx, - ) - return var["positions"].copy_to_gpu(sum_scheduled_tokens) + return self.attn_metadata_builder.prepare_prefill(batch, self.forward_vars) + def prepare_decode(self, batch: ScheduledBatch): scheduled_bs = batch.total_seqs_num_decode @@ -569,101 +541,10 @@ def prepare_decode(self, batch: ScheduledBatch): else next(x for x in self.graph_bs if x >= scheduled_bs) ) assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" - dropout_p = 0.0 - max_q_len = 1 - self.total_blocks = 0 - - context_lens = [seq.num_tokens for seq in seqs] - positions = context_lens - slot_mapping = [ - seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 - for seq in seqs - ] - slot_mapping.extend([-1] * (bs - scheduled_bs)) - - if self.use_kv_indptr: - assert self.block_size == 1, "AITER MLA requires only block size 1." - self.kv_indices: list[int] = [] - self.kv_indptr: list[int] = [0] - self.kv_last_page_lens: list[int] = [] - - for seq in seqs: - current_seq_len = seq.num_tokens - self._update_paged_kv_tensors(seq.block_table, current_seq_len) - - self.prepare_block_tables(seqs) - - sum_scheduled_tokens = batch.total_tokens_num_decode - var = self.forward_vars - var["slot_mapping"].np[:bs] = slot_mapping - var["positions"].np[:sum_scheduled_tokens] = positions - var["context_lens"].np[:scheduled_bs] = context_lens - vars_used = [ - ("slot_mapping", bs), # TODO: MTP support - ("context_lens", bs), - ("block_tables", bs), - ] - if self.use_kv_indptr and len(self.kv_indptr) > 0: - # extend to the maximum number of blocks as returned by the scheduler - self.kv_indices.extend([0] * (self.total_blocks - len(self.kv_indices))) - var["kv_indices"].np[: self.total_blocks] = np.array( - self.kv_indices, dtype=np.int64 - ) - var["kv_indptr"].np[: scheduled_bs + 1] = np.array( - self.kv_indptr, dtype=np.int64 - ) - var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[ - scheduled_bs - ] - var["kv_last_page_lens"].np[:scheduled_bs] = np.array( - self.kv_last_page_lens, dtype=np.int64 - ) - var["kv_last_page_lens"].np[scheduled_bs:bs] = 0 - vars_used = [ - ("slot_mapping", bs), # TODO: MTP support - ("context_lens", bs), - ("block_tables", bs), - ("cu_seqlens_q", bs + 1), - ("kv_indices", sum(context_lens)), - ("kv_indptr", bs + 1), - ("kv_last_page_lens", bs), - ] + return self.attn_metadata_builder.prepare_decode(batch, bs, self.forward_vars) - ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} - set_context( - False, - batch_size=scheduled_bs, - graph_bs=bs, - dropout_p=dropout_p, - max_q_len=max_q_len, - **ctx, - ) - positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) - # for el in ctx: - # print(f"{el}: {ctx[el]}") - # print(f"positions: {positions}") - return positions - - def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): - # Get the number of valid blocks based on sequence length. - # If seq_len = 16, block_size = 16, - # block_table_bound is 1 with 1 valid block. - # If seq_len = 15, block_size = 16, - # block_table_bound is 0 + 1 with 1 valid block. - self.total_blocks += len(block_table) - block_table_bound = ( - seq_len // self.block_size + 1 - if seq_len % self.block_size != 0 - else seq_len // self.block_size - ) - self.kv_indices.extend(block_table[:block_table_bound]) - self.kv_indptr.append(self.kv_indptr[-1] + block_table_bound) - last_page_len = seq_len % self.block_size - if last_page_len == 0: - last_page_len = self.block_size - self.kv_last_page_lens.append(last_page_len) def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: temperatures = [seq.temperature for seq in batch.seqs.values()] diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py index 391d1d9eac..fc296e6aee 100644 --- a/atom/model_ops/attention.py +++ b/atom/model_ops/attention.py @@ -16,10 +16,21 @@ get_forward_context, set_forward_context, ) +from atom.utils import mark_spliting_op +def fake_( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, + scale: float, kv_cache_dtype: str, layer_num: int +)-> torch.Tensor: + output_shape = q.shape + output = torch.zeros(output_shape, + dtype=q.dtype, + device=q.device) + return output # Dynamo will not try to inspect any of the internal operations for prefill or decode # @torch.library.custom_op("aiter::unified_attention_with_output", mutates_args=["q", "k", "v", "k_cache", "v_cache", "k_scale", "v_scale"]) +@mark_spliting_op(is_custom=True, gen_fake=fake_) def unified_attention_with_output( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, scale: float, kv_cache_dtype: str, layer_num: int @@ -96,24 +107,12 @@ def unified_attention_with_output( return o -# @unified_attention_with_output.register_fake -def _( - q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - scale: float, kv_cache_dtype: str, layer_num: int -)-> torch.Tensor: - output_shape = q.shape - output = torch.zeros(output_shape, - dtype=q.dtype, - device=q.device) - return output - - -direct_register_custom_op( - op_name="unified_attention_with_output", - op_func=unified_attention_with_output, - mutates_args=[], - fake_impl=_, -) +# direct_register_custom_op( +# op_name="unified_attention_with_output", +# op_func=unified_attention_with_output, +# mutates_args=[], +# fake_impl=_, +# ) class Attention(nn.Module): diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index f9f5800292..3346d32414 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -26,6 +26,7 @@ import logging from aiter.dist.parallel_state import get_tp_group +from atom.utils import mark_spliting_op logger = logging.getLogger("atom") @@ -353,7 +354,51 @@ def _forward_linear_fp8_no_bias(x: torch.Tensor, weight: torch.Tensor, weight_sc y = get_tp_group().all_reduce(y, ca_fp8_quant=False) return y +def fake_mla_attention( + q: torch.Tensor, # query in unified attn + k_c_normed: torch.Tensor, # key in unified attn + k_pe: torch.Tensor, # value in unified attn + positions: torch.Tensor, + W_K: torch.Tensor, + W_V: torch.Tensor, + W_K_scale: torch.Tensor, + W_V_scale: torch.Tensor, + rotary_emb_cos_cache: torch.Tensor, + rotary_emb_sin_cache: torch.Tensor, + rotary_emb_is_neox_style: bool, + rotary_emb_head_size: int, + rotary_emb_rotary_dim: int, + q_proj_weight_data: torch.Tensor, + q_proj_weight_scale_data: torch.Tensor, + q_proj_tp_dim: int, + q_proj_tp_size: int, + q_proj_reduce_results: bool, + kv_b_proj_weight_data: torch.Tensor, + kv_b_proj_weight_scale_data: torch.Tensor, + kv_b_proj_tp_dim: int, + kv_b_proj_tp_size: int, + kv_b_proj_reduce_results: bool, + o_proj_weight_data: torch.Tensor, + o_proj_weight_scale_data: torch.Tensor, + o_proj_tp_dim: int, + o_proj_tp_size: int, + o_proj_reduce_results: bool, + kv_lora_rank: int, + num_heads: int, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + qk_head_dim: int, + v_head_dim: int, + kv_cache_dtype: str, + scale: float, + layer_num: int +) -> torch.Tensor: + output_shape = list(q.shape) + output_shape[-1] = 7168 # TODO: read from hidden_size + return torch.zeros(output_shape, dtype=q.dtype, device=q.device) + +@mark_spliting_op(is_custom=True, gen_fake=fake_mla_attention) def mla_attention( q: torch.Tensor, # query in unified attn k_c_normed: torch.Tensor, # key in unified attn @@ -447,56 +492,12 @@ def mla_attention( return output -def fake_mla_attention( - q: torch.Tensor, # query in unified attn - k_c_normed: torch.Tensor, # key in unified attn - k_pe: torch.Tensor, # value in unified attn - positions: torch.Tensor, - W_K: torch.Tensor, - W_V: torch.Tensor, - W_K_scale: torch.Tensor, - W_V_scale: torch.Tensor, - rotary_emb_cos_cache: torch.Tensor, - rotary_emb_sin_cache: torch.Tensor, - rotary_emb_is_neox_style: bool, - rotary_emb_head_size: int, - rotary_emb_rotary_dim: int, - q_proj_weight_data: torch.Tensor, - q_proj_weight_scale_data: torch.Tensor, - q_proj_tp_dim: int, - q_proj_tp_size: int, - q_proj_reduce_results: bool, - kv_b_proj_weight_data: torch.Tensor, - kv_b_proj_weight_scale_data: torch.Tensor, - kv_b_proj_tp_dim: int, - kv_b_proj_tp_size: int, - kv_b_proj_reduce_results: bool, - o_proj_weight_data: torch.Tensor, - o_proj_weight_scale_data: torch.Tensor, - o_proj_tp_dim: int, - o_proj_tp_size: int, - o_proj_reduce_results: bool, - kv_lora_rank: int, - num_heads: int, - qk_nope_head_dim: int, - qk_rope_head_dim: int, - qk_head_dim: int, - v_head_dim: int, - kv_cache_dtype: str, - scale: float, - layer_num: int -) -> torch.Tensor: - output_shape = list(q.shape) - output_shape[-1] = 7168 # TODO: read from hidden_size - return torch.zeros(output_shape, dtype=q.dtype, device=q.device) - - -direct_register_custom_op( - op_name="mla_attention", - op_func=mla_attention, - mutates_args=[], - fake_impl=fake_mla_attention, -) +# direct_register_custom_op( +# op_name="mla_attention", +# op_func=mla_attention, +# mutates_args=[], +# fake_impl=fake_mla_attention, +# ) class MLAAttention(nn.Module): @@ -618,9 +619,9 @@ def forward( ) -> torch.Tensor: return torch.ops.aiter.mla_attention(q, k_c_normed, k_pe, positions, self.W_K, self.W_V, self.W_K_scale, self.W_V_scale, self.rotary_emb.cos_cache, self.rotary_emb.sin_cache, self.rotary_emb.is_neox_style, self.rotary_emb.head_size, self.rotary_emb.rotary_dim, - self.q_proj.weight.data, self.q_proj.weight_scale.data, self.q_proj.tp_dim, self.q_proj.tp_size, self.q_proj.reduce_results, - self.kv_b_proj.weight.data, self.kv_b_proj.weight_scale.data, self.kv_b_proj.tp_dim, self.kv_b_proj.tp_size, self.kv_b_proj.reduce_results, - self.o_proj.weight.data, self.o_proj.weight_scale.data, self.o_proj.tp_dim, self.o_proj.tp_size, self.o_proj.reduce_results, + self.q_proj.weight, self.q_proj.weight_scale, self.q_proj.tp_dim, self.q_proj.tp_size, self.q_proj.reduce_results, + self.kv_b_proj.weight, self.kv_b_proj.weight_scale, self.kv_b_proj.tp_dim, self.kv_b_proj.tp_size, self.kv_b_proj.reduce_results, + self.o_proj.weight, self.o_proj.weight_scale, self.o_proj.tp_dim, self.o_proj.tp_size, self.o_proj.reduce_results, self.kv_lora_rank, self.num_heads, self.qk_nope_head_dim, self.qk_rope_head_dim, self.qk_head_dim, self.v_head_dim, self.kv_cache_dtype, self.scale, self.layer_num) diff --git a/atom/model_ops/attentions/aiter_attention.py b/atom/model_ops/attentions/aiter_attention.py new file mode 100644 index 0000000000..cfa8431941 --- /dev/null +++ b/atom/model_ops/attentions/aiter_attention.py @@ -0,0 +1,68 @@ +from dataclasses import dataclass +from typing import Type, Optional +from .backends import CommonAttentionBuilder, AttentionBackend +import torch +import numpy as np + +from atom.model_engine.scheduler import ScheduledBatch +from atom.utils.context import set_context + + +class AiterBackend(AttentionBackend): + @staticmethod + def get_name() -> str: + return "ROCM_AITER_ATTENTION" + + @staticmethod + def get_builder_cls() -> Type["AiterAttentionMetadataBuilder"]: + return AiterAttentionMetadataBuilder + +class AiterAttentionMetadataBuilder(CommonAttentionBuilder): + BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] + + def __init__(self, block_size: int): + self.block_size = block_size + + def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): + scheduled_bs = batch.total_seqs_num_decode + seqs = list(batch.seqs.values()) + self.total_blocks = 0 + dropout_p = 0.0 + max_q_len = 1 + + context_lens = [seq.num_tokens for seq in seqs] + positions = context_lens + slot_mapping = [ + seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 + for seq in seqs + ] + slot_mapping.extend([-1] * (bs - scheduled_bs)) + + block_tables = forward_vars["block_tables"].np + for i, seq in enumerate(seqs): + block_tables[i] = 0 + block_tables[i, : seq.num_blocks] = seq.block_table + + sum_scheduled_tokens = batch.total_tokens_num_decode + var = forward_vars + var["slot_mapping"].np[:bs] = slot_mapping + var["positions"].np[:sum_scheduled_tokens] = positions + var["context_lens"].np[:scheduled_bs] = context_lens + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + ("block_tables", bs), + ] + + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} + set_context( + False, + batch_size=scheduled_bs, + graph_bs=bs, + dropout_p=dropout_p, + max_q_len=max_q_len, + **ctx, + ) + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) + return positions + diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py new file mode 100644 index 0000000000..5eb46fff8b --- /dev/null +++ b/atom/model_ops/attentions/aiter_mla.py @@ -0,0 +1,122 @@ +from dataclasses import dataclass +from typing import Type, Optional +from .backends import CommonAttentionBuilder, AttentionBackend +import torch +import numpy as np + +from atom.model_engine.scheduler import ScheduledBatch +from atom.utils.context import set_context + + +class AiterMLABackend(AttentionBackend): + @staticmethod + def get_name() -> str: + return "ROCM_AITER_MLA" + + @staticmethod + def get_builder_cls() -> Type["AiterMLAMetadataBuilder"]: + return AiterMLAMetadataBuilder + +class AiterMLAMetadataBuilder(CommonAttentionBuilder): + BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] + + def __init__(self, block_size: int): + self.block_size = block_size + assert self.block_size == 1, "AITER MLA requires only block size 1." + + def prepare(self, batch: ScheduledBatch): + self.paged_kv_indices: list[int] = [] + self.paged_kv_indptr: list[int] = [0] + self.paged_kv_last_page_lens: list[int] = [] + self.total_blocks = 0 + + seqs = list(batch.seqs.values()) + for seq in seqs: + current_seq_len = seq.num_tokens + self._update_paged_kv_tensors(seq.block_table, current_seq_len) + + def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): + # Get the number of valid blocks based on sequence length. + # If seq_len = 16, block_size = 16, + # block_table_bound is 1 with 1 valid block. + # If seq_len = 15, block_size = 16, + # block_table_bound is 0 + 1 with 1 valid block. + self.total_blocks += len(block_table) + block_table_bound = ( + seq_len // self.block_size + 1 + if seq_len % self.block_size != 0 + else seq_len // self.block_size + ) + self.paged_kv_indices.extend(block_table[:block_table_bound]) + self.paged_kv_indptr.append(self.paged_kv_indptr[-1] + block_table_bound) + + last_page_len = seq_len % self.block_size + if last_page_len == 0: + last_page_len = self.block_size + self.paged_kv_last_page_lens.append(last_page_len) + + + def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): + scheduled_bs = batch.total_seqs_num_decode + seqs = list(batch.seqs.values()) + dropout_p = 0.0 + max_q_len = 1 + self.total_blocks = 0 + + context_lens = [seq.num_tokens for seq in seqs] + positions = context_lens + slot_mapping = [ + seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 + for seq in seqs + ] + slot_mapping.extend([-1] * (bs - scheduled_bs)) + + self.prepare(batch) + + + sum_scheduled_tokens = batch.total_tokens_num_decode + var = forward_vars + var["slot_mapping"].np[:bs] = slot_mapping + var["positions"].np[:sum_scheduled_tokens] = positions + var["context_lens"].np[:scheduled_bs] = context_lens + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + ] + + if len(self.paged_kv_indptr) > 0: + # extend to the maximum number of blocks as returned by the scheduler + self.paged_kv_indices.extend([0] * (self.total_blocks - len(self.paged_kv_indices))) + var["kv_indices"].np[: self.total_blocks] = np.array( + self.paged_kv_indices, dtype=np.int64 + ) + var["kv_indptr"].np[: scheduled_bs + 1] = np.array( + self.paged_kv_indptr, dtype=np.int64 + ) + var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[ + scheduled_bs + ] + var["kv_last_page_lens"].np[:scheduled_bs] = np.array( + self.paged_kv_last_page_lens, dtype=np.int64 + ) + var["kv_last_page_lens"].np[scheduled_bs:bs] = 0 + vars_used = [ + ("slot_mapping", bs), # TODO: MTP support + ("context_lens", bs), + # ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("kv_indices", sum(context_lens)), + ("kv_indptr", bs + 1), + ("kv_last_page_lens", bs), + ] + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} + set_context( + False, + batch_size=scheduled_bs, + graph_bs=bs, + dropout_p=dropout_p, + max_q_len=max_q_len, + **ctx, + ) + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) + return positions diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py new file mode 100644 index 0000000000..3d70a0a708 --- /dev/null +++ b/atom/model_ops/attentions/backends.py @@ -0,0 +1,137 @@ +from abc import ABC, abstractmethod +from typing import Type, Tuple, Generic, Optional, List, TypeVar, Dict, Any +from atom.model_engine.scheduler import ScheduledBatch +from atom.utils.context import set_context + +import torch + +T = TypeVar('T', bound="BroadcastableModelInput") + +class BroadcastableModelInput(ABC): + + @abstractmethod + def as_broadcastable_tensor_dict(self) -> Dict[str, Any]: + """ + Extract broadcastable fields. Override for fields that require some + custom deserialization. + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def from_broadcasted_tensor_dict( + cls: Type[T], + tensor_dict: Dict[str, Any], + attn_backend: Optional["AttentionBackend"] = None, + ) -> T: + """ + Pop fields from the given tensor_dict and populate a new instance of + BroadcastableModelInput. + """ + raise NotImplementedError + + +class AttentionBackend(ABC): + """Abstract class for attention backends.""" + # For some attention backends, we allocate an output tensor before + # calling the custom op. When piecewise cudagraph is enabled, this + # makes sure the output tensor is allocated inside the cudagraph. + accept_output_buffer: bool = False + + @staticmethod + @abstractmethod + def get_name() -> str: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_builder_cls() -> Type["AttentionMetadataBuilder"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + raise NotImplementedError + + + +class AttentionMetadataBuilder(ABC, Generic[T]): + """Abstract class for attention metadata builders.""" + + @abstractmethod + def __init__(self, inpblock_size: int) -> None: + """Create the builder, remember some configuration and parameters.""" + raise NotImplementedError + + @abstractmethod + def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): + raise NotImplementedError + + @abstractmethod + def prepare_prefill(self, batch: ScheduledBatch, forward_vars): + raise NotImplementedError + + +class CommonAttentionBuilder(AttentionMetadataBuilder[T], Generic[T]): + def __init__(self, block_size: int): + self.block_size = block_size + + def prepare_prefill(self, batch: ScheduledBatch, forward_vars): + bs = batch.total_seqs_num_prefill + sum_scheduled_tokens = batch.total_tokens_num_prefill + var = forward_vars + positions = [] + cu_seqlens_k = [0] + max_seqlen_q = 0 + max_seqlen_k = 0 + slot_mapping = [] + seqs = list(batch.seqs.values()) + seqs = seqs[:bs] + for seq in seqs: + seqlen = seq.num_tokens + positions.extend(list(range(seq.num_cached_tokens, seqlen))) + seqlen_q = seqlen - seq.num_cached_tokens + seqlen_k = seqlen + cu_seqlens_k.append(cu_seqlens_k[-1] + seqlen_k) + max_seqlen_q = max(seqlen_q, max_seqlen_q) + max_seqlen_k = max(seqlen_k, max_seqlen_k) + if not seq.block_table: + continue + for i in range(seq.num_cached_blocks, seq.num_blocks): + start = seq.block_table[i] * self.block_size + if i != seq.num_blocks - 1: + end = start + self.block_size + else: + end = start + seq.last_block_num_tokens + slot_mapping.extend(list(range(start, end))) + if cu_seqlens_k[-1] > batch.total_tokens_num: # prefix cache + self.prepare_block_tables(seqs) + var["positions"].np[:sum_scheduled_tokens] = positions + var["slot_mapping"].np[: len(slot_mapping)] = slot_mapping + cu_seqlens_k = torch.tensor(cu_seqlens_k, dtype=torch.int32, pin_memory=True) + + min_seqlen_q = 0 + dropout_p = 0.0 + vars_used = [ + ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("slot_mapping", len(slot_mapping)), + ] + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} + set_context( + True, + batch_size=bs, + cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), + # slot_mapping=slot_mapping.cuda(non_blocking=True), + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + min_seqlen_q=min_seqlen_q, + dropout_p=dropout_p, + **ctx, + ) + return var["positions"].copy_to_gpu(sum_scheduled_tokens) diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index 5a12da43da..e83cd5a867 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -13,15 +13,53 @@ from typing import Any, Iterator, Optional, Sequence, Union from urllib.parse import urlparse from uuid import uuid4 +from transformers import PretrainedConfig import numpy as np import psutil import torch import zmq import zmq.asyncio +from atom.utils.custom_register import direct_register_custom_op +from typing import Any, Callable, Optional logger = logging.getLogger("atom") +def mark_spliting_op( + is_custom: bool, + gen_fake: Optional[Callable[..., Any]] = None, + mutates_args: list[str] = [], +): + def decorator(func): + if not is_custom: + func.spliting_op = True + return func + + direct_register_custom_op( + op_name=func.__name__, + op_func=func, + mutates_args=mutates_args, + fake_impl=gen_fake, + ) + registered_op = getattr(torch.ops.aiter, func.__name__) + registered_op.spliting_op = True + return func + + return decorator + +def get_hf_text_config(config: PretrainedConfig): + """Get the "sub" config relevant to llm for multi modal models. + No op for pure text models. + """ + if hasattr(config, "text_config"): + # The code operates under the assumption that text_config should have + # `num_attention_heads` (among others). Assert here to fail early + # if transformers config doesn't align with this assumption. + assert hasattr(config.text_config, "num_attention_heads") + return config.text_config + else: + return config + def get_mp_context() -> Union[ForkContext, SpawnContext]: """Get a multiprocessing context with 'spawn' start method.""" diff --git a/atom/utils/backends.py b/atom/utils/backends.py index 087c1f18f9..66909c7390 100644 --- a/atom/utils/backends.py +++ b/atom/utils/backends.py @@ -247,8 +247,7 @@ def split_graph(graph: fx.GraphModule, for node in graph.graph.nodes: if node.op in ("output", "placeholder"): continue - # print(str(node.target)) - if node.op == 'call_function' and str(node.target) in ops: + if node.op == 'call_function' and (hasattr(node.target, 'spliting_op') and (node.target.spliting_op)): subgraph_id += 1 node_to_subgraph_id[node] = subgraph_id split_op_graphs.append(subgraph_id) diff --git a/atom/utils/decorators.py b/atom/utils/decorators.py index 7d97f6e05f..bc3f7e1914 100644 --- a/atom/utils/decorators.py +++ b/atom/utils/decorators.py @@ -285,9 +285,6 @@ def patched_inline_call(parent, func, args, kwargs): patched_inline_call): # print("self.compiled_callable to call torch compile") output = self.compiled_callable(*args, **kwargs) - from aiter import logger - logger.info("torch.compiler.is_compiling()") - logger.info(torch.compiler.is_compiling()) return output # usually, capturing the model once is enough, and then we can diff --git a/atom/utils/selector.py b/atom/utils/selector.py new file mode 100644 index 0000000000..d020485619 --- /dev/null +++ b/atom/utils/selector.py @@ -0,0 +1,74 @@ +import torch +from typing import Tuple, Optional, Type, List +from abc import ABC, abstractmethod +from functools import cache +from atom.utils import resolve_obj_by_qualname +from atom.model_ops.attentions.backends import AttentionMetadataBuilder + +class AttentionBackend(ABC): + """Abstract class for attention backends.""" + # For some attention backends, we allocate an output tensor before + # calling the custom op. When piecewise cudagraph is enabled, this + # makes sure the output tensor is allocated inside the cudagraph. + accept_output_buffer: bool = False + + @staticmethod + @abstractmethod + def get_name() -> str: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_builder_cls() -> Type["AttentionMetadataBuilder"]: + raise NotImplementedError + + @staticmethod + @abstractmethod + def get_kv_cache_shape( + num_blocks: int, + block_size: int, + num_kv_heads: int, + head_size: int, + ) -> Tuple[int, ...]: + raise NotImplementedError + + +def get_attn_backend( + block_size: int, + use_mla: bool = False, +) -> Type[AttentionBackend]: + """Selects which attention backend to use and lazily imports it.""" + # Accessing envs.* behind an @lru_cache decorator can cause the wrong + # value to be returned from the cache if the value changes between calls. + # To avoid this, we read envs.VLLM_USE_V1 here and pass it explicitly to the + # private function. + return _cached_get_attn_backend( + block_size=block_size, + use_mla=use_mla, + ) + + +@cache +def _cached_get_attn_backend( + block_size: int, + use_mla: bool = False, +) -> Type[AttentionBackend]: + + # get device-specific attn_backend + attention_cls = get_attn_backend_cls(block_size, use_mla) + if not attention_cls: + raise ValueError( + f"Invalid attention backend for {attention_cls}") + return resolve_obj_by_qualname(attention_cls) + +def get_attn_backend_cls(block_size, use_mla) -> str: + if use_mla: + if block_size == 1: + return "atom.model_ops.attentions.aiter_mla.AiterMLABackend" # noqa: E501 + else: + raise ValueError( + f" The selected backend" + f"does not support block size {block_size}." + "(currently only supports block size 1)") + + return "atom.model_ops.attentions.aiter_attention.AiterBackend" # noqa: E501 \ No newline at end of file From f58c89aa428c3f392768c23539c33f1190ecb4f2 Mon Sep 17 00:00:00 2001 From: ZhangLirong Date: Tue, 28 Oct 2025 17:49:15 +0800 Subject: [PATCH 097/119] Support parents attention for mla and mha (#22) * support parents attention for mla and mha * remove useless --- atom/config.py | 3 + atom/model_engine/model_runner.py | 86 +-- atom/model_ops/attention.py | 151 ---- atom/model_ops/attention_mha.py | 128 ++++ atom/model_ops/attention_mla.py | 692 ++++++------------- atom/model_ops/attentions/aiter_attention.py | 5 + atom/model_ops/attentions/aiter_mla.py | 6 + atom/model_ops/attentions/backends.py | 40 +- atom/model_ops/base_attention.py | 98 +++ atom/models/deepseek_v2.py | 22 +- atom/models/llama.py | 4 +- atom/models/mixtral.py | 3 +- atom/models/qwen3.py | 4 +- atom/utils/selector.py | 29 +- 14 files changed, 539 insertions(+), 732 deletions(-) delete mode 100644 atom/model_ops/attention.py create mode 100644 atom/model_ops/attention_mha.py create mode 100644 atom/model_ops/base_attention.py diff --git a/atom/config.py b/atom/config.py index 3083e0fa2c..d13846885a 100644 --- a/atom/config.py +++ b/atom/config.py @@ -153,6 +153,9 @@ class CompilationConfig: to integers, it also supports "cudagraph_capture_sizes" to specify the sizes for cudagraph capture.""" + static_forward_context: dict[str, Any] = field(default_factory=dict, + init=False) + def init_with_cudagraph_sizes(self) -> None: """To complete the initialization of config, we need to know the cudagraph sizes.""" diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index a7b4e029e5..6c9b897252 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -474,48 +474,50 @@ def allocate_kv_cache(self, num_kvcache_blocks): layer_id = 0 x = 16 // self.kv_cache.element_size() for module in self.model.modules(): - if hasattr(module, "k_cache") and hasattr(module, "v_cache"): - module.k_cache = self.kv_cache[0, layer_id].view( - config.num_kvcache_blocks, - num_kv_heads, - hf_config.head_dim // x, - self.block_size, - x, - ) - module.v_cache = self.kv_cache[1, layer_id].view( - config.num_kvcache_blocks, - num_kv_heads, - hf_config.head_dim, - self.block_size, - ) - module.max_model_len = self.config.max_model_len - if config.kv_cache_dtype == "fp8": - module.k_scale = self.kv_scale[0, layer_id] - module.v_scale = self.kv_scale[1, layer_id] - attention_metadata = AttentionMetadata( - k_cache=module.k_cache, - v_cache=module.v_cache, - k_scale=module.k_scale, - v_scale=module.v_scale, - ) - set_forward_context(module.layer_num, attention_metadata) - - layer_id += 1 - elif hasattr(module, "kv_cache") and isMLA: - module.kv_cache = self.kv_cache[layer_id].view( - config.num_kvcache_blocks * self.block_size, - 1, - 576, - ) - module.max_model_len = self.config.max_model_len - attention_metadata = AttentionMetadata( - k_cache=module.kv_cache, - v_cache=None, - k_scale=None, - v_scale=None, - ) - set_forward_context(module.layer_num, attention_metadata) - layer_id += 1 + # Since use attention base and there are child in attention, add base condition + if hasattr(module, "base_attention"): + if hasattr(module, "use_mla") and not module.use_mla: + module.k_cache = self.kv_cache[0, layer_id].view( + config.num_kvcache_blocks, + num_kv_heads, + hf_config.head_dim // x, + self.block_size, + x, + ) + module.v_cache = self.kv_cache[1, layer_id].view( + config.num_kvcache_blocks, + num_kv_heads, + hf_config.head_dim, + self.block_size, + ) + module.max_model_len = self.config.max_model_len + if config.kv_cache_dtype == "fp8": + module.k_scale = self.kv_scale[0, layer_id] + module.v_scale = self.kv_scale[1, layer_id] + attention_metadata = AttentionMetadata( + k_cache=module.k_cache, + v_cache=module.v_cache, + k_scale=module.k_scale, + v_scale=module.v_scale, + ) + set_forward_context(module.layer_num, attention_metadata) + + layer_id += 1 + elif hasattr(module, "use_mla") and module.use_mla: + module.kv_cache = self.kv_cache[layer_id].view( + config.num_kvcache_blocks * self.block_size, + 1, + 576, + ) + module.max_model_len = self.config.max_model_len + attention_metadata = AttentionMetadata( + k_cache=module.kv_cache, + v_cache=None, + k_scale=None, + v_scale=None, + ) + set_forward_context(module.layer_num, attention_metadata) + layer_id += 1 if torch.distributed.is_initialized(): torch.distributed.barrier() return True diff --git a/atom/model_ops/attention.py b/atom/model_ops/attention.py deleted file mode 100644 index fc296e6aee..0000000000 --- a/atom/model_ops/attention.py +++ /dev/null @@ -1,151 +0,0 @@ -# from flash_attn import flash_attn_with_kvcache -from dataclasses import dataclass - -import aiter -import torch -import triton -import triton.language as tl -from aiter.paged_attn import PagedAttention -from torch import nn - -from atom.utils.context import get_context -from atom.utils.custom_register import direct_register_custom_op -from atom.utils.forward_context import ( - AttentionMetadata, - ForwardContext, - get_forward_context, - set_forward_context, -) -from atom.utils import mark_spliting_op - -def fake_( - q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - scale: float, kv_cache_dtype: str, layer_num: int -)-> torch.Tensor: - output_shape = q.shape - output = torch.zeros(output_shape, - dtype=q.dtype, - device=q.device) - return output - -# Dynamo will not try to inspect any of the internal operations for prefill or decode -# @torch.library.custom_op("aiter::unified_attention_with_output", mutates_args=["q", "k", "v", "k_cache", "v_cache", "k_scale", "v_scale"]) -@mark_spliting_op(is_custom=True, gen_fake=fake_) -def unified_attention_with_output( - q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - scale: float, kv_cache_dtype: str, layer_num: int -)-> torch.Tensor: - context = get_context() - if context.slot_mapping.numel(): - # not dummy run - forward_context: ForwardContext = get_forward_context() - attn_metadata_ = forward_context.no_compile_layers[layer_num] - k_cache = attn_metadata_.k_cache - v_cache = attn_metadata_.v_cache - k_scale = attn_metadata_.k_scale - v_scale = attn_metadata_.v_scale - else: - # dummy run before allocate kv_cache, thus we create manually - k_cache = v_cache = torch.tensor([]) - k_scale = v_scale = None - - if k_cache.numel() and v_cache.numel(): - if kv_cache_dtype == "fp8": - aiter.reshape_and_cache_with_pertoken_quant( - k, - v, - k_cache, - v_cache, - k_scale, - v_scale, - context.slot_mapping, - asm_layout=True, - ) - else: - aiter.reshape_and_cache( - k, - v, - k_cache, - v_cache, - context.slot_mapping, - kv_cache_dtype="auto", - k_scale=None, - v_scale=None, - asm_layout=True, - ) - - - if context.is_prefill: - # if context.block_tables is not None: # prefix cache - # k, v = k_cache, v_cache - o = aiter.flash_attn_varlen_func( - q, - k, - v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q=context.min_seqlen_q, - dropout_p=context.dropout_p, - softmax_scale=scale, - causal=True, - ) - else: # decode - o = aiter.pa_fwd_asm( - q, - k_cache, - v_cache, - context.block_tables, - context.context_lens, - context.block_tables.stride(0), - K_QScale=k_scale, - V_QScale=v_scale, - out_=None, - high_precision=0, - ) - - return o - -# direct_register_custom_op( -# op_name="unified_attention_with_output", -# op_func=unified_attention_with_output, -# mutates_args=[], -# fake_impl=_, -# ) - - -class Attention(nn.Module): - - def __init__( - self, - num_heads, - head_dim, - scale, - num_kv_heads, - kv_cache_dtype="bf16", - layer_num=0, - **kwargs, - ): - super().__init__() - self.num_heads = num_heads - self.head_dim = head_dim - self.scale = scale - self.num_kv_heads = num_kv_heads - self.k_cache = self.v_cache = torch.tensor([]) - self.kv_cache_dtype = kv_cache_dtype - self.max_model_len = 0 - self.k_scale = self.v_scale = None - self.layer_num = layer_num - - def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor): - o: torch.Tensor - q = q.view(-1, self.num_heads, self.head_dim) - k = k.view(-1, self.num_kv_heads, self.head_dim) - v = v.view(-1, self.num_kv_heads, self.head_dim) - - - o = torch.ops.aiter.unified_attention_with_output(q, k, v, - self.scale, self.kv_cache_dtype, self.layer_num) - o = o.view(-1, self.num_heads * self.head_dim) - return o diff --git a/atom/model_ops/attention_mha.py b/atom/model_ops/attention_mha.py new file mode 100644 index 0000000000..71ab42596e --- /dev/null +++ b/atom/model_ops/attention_mha.py @@ -0,0 +1,128 @@ +# from flash_attn import flash_attn_with_kvcache +from dataclasses import dataclass + +import aiter +import torch +import triton +import triton.language as tl +from aiter.paged_attn import PagedAttention +from torch import nn + +from atom.utils.context import get_context +from atom.utils.custom_register import direct_register_custom_op +from atom.utils.forward_context import ( + AttentionMetadata, + ForwardContext, + get_forward_context, + set_forward_context, +) +from atom.utils import mark_spliting_op +from .attention_mla import MLAModules + + +class Attention(nn.Module): + + def __init__( + self, + num_heads, + head_dim, + scale, + num_kv_heads, + kv_cache_dtype="bf16", + layer_num=0, + mla_modules: MLAModules=None, + **kwargs, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = scale + self.num_kv_heads = num_kv_heads + self.k_cache = self.v_cache = torch.tensor([]) + self.kv_cache_dtype = kv_cache_dtype + self.max_model_len = 0 + self.k_scale = self.v_scale = None + self.layer_num = layer_num + + def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, position: torch.Tensor=None): + o: torch.Tensor + q = q.view(-1, self.num_heads, self.head_dim) + k = k.view(-1, self.num_kv_heads, self.head_dim) + v = v.view(-1, self.num_kv_heads, self.head_dim) + + + # o = torch.ops.aiter.unified_attention_with_output(q, k, v, + # self.scale, self.kv_cache_dtype, self.layer_num) + context = get_context() + if context.slot_mapping.numel(): + # not dummy run + forward_context: ForwardContext = get_forward_context() + attn_metadata_ = forward_context.no_compile_layers[self.layer_num] + k_cache = attn_metadata_.k_cache + v_cache = attn_metadata_.v_cache + k_scale = attn_metadata_.k_scale + v_scale = attn_metadata_.v_scale + else: + # dummy run before allocate kv_cache, thus we create manually + k_cache = v_cache = torch.tensor([]) + k_scale = v_scale = None + + if k_cache.numel() and v_cache.numel(): + if self.kv_cache_dtype == "fp8": + aiter.reshape_and_cache_with_pertoken_quant( + k, + v, + k_cache, + v_cache, + k_scale, + v_scale, + context.slot_mapping, + asm_layout=True, + ) + else: + aiter.reshape_and_cache( + k, + v, + k_cache, + v_cache, + context.slot_mapping, + kv_cache_dtype="auto", + k_scale=None, + v_scale=None, + asm_layout=True, + ) + + + if context.is_prefill: + # if context.block_tables is not None: # prefix cache + # k, v = k_cache, v_cache + o = aiter.flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=context.cu_seqlens_q, + cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q=context.min_seqlen_q, + dropout_p=context.dropout_p, + softmax_scale=self.scale, + causal=True, + ) + else: # decode + o = aiter.pa_fwd_asm( + q, + k_cache, + v_cache, + context.block_tables, + context.context_lens, + context.block_tables.stride(0), + K_QScale=k_scale, + V_QScale=v_scale, + out_=None, + high_precision=0, + ) + + + o = o.view(-1, self.num_heads * self.head_dim) + return o diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index 3346d32414..7c206fa8c4 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass import torch from torch import nn from atom.utils.context import get_context, Context @@ -31,6 +32,22 @@ logger = logging.getLogger("atom") +# MLA Specific Arguments +@dataclass +class MLAModules: + """Modules used in MLA. + """ + q_lora_rank: Optional[int] + kv_lora_rank: int + qk_nope_head_dim: int + qk_rope_head_dim: int + qk_head_dim: int + v_head_dim: int + rotary_emb: torch.nn.Module + q_proj: Optional[torch.nn.Module] + kv_b_proj: torch.nn.Module + o_proj: torch.nn.Module + def dynamic_per_batched_tensor_quant( x: torch.Tensor, dtype: torch.dtype = torch.float8_e4m3fn @@ -43,463 +60,6 @@ def dynamic_per_batched_tensor_quant( return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal() -def aiter_triton_fp8_bmm( - X: torch.Tensor, - WQ: torch.Tensor, - w_scale: torch.Tensor, - group_size: int = 128, - transpose_bm: Optional[bool] = False, -) -> torch.Tensor: - return _aiter_triton_fp8_bmm(X, WQ, w_scale, group_size=group_size, transpose_bm=transpose_bm) - - -def fake_bmm( - X: torch.Tensor, - WQ: torch.Tensor, - w_scale: torch.Tensor, - group_size: int = 128, - transpose_bm: Optional[bool] = False, -) -> torch.Tensor: - return torch.zeros([X.shape[0], X.shape[1], WQ.shape[1]], dtype=X.dtype, device=X.device) - - -direct_register_custom_op( - op_name="aiter_triton_fp8_bmm", - op_func=aiter_triton_fp8_bmm, - mutates_args=[], - fake_impl=fake_bmm, -) - - -def aiter_mla_decode_fwd( - q: torch.Tensor, - kv_buffer: torch.Tensor, - o: torch.Tensor, - qo_indptr: torch.Tensor, - max_seqlen_qo: int, - kv_indptr: Optional[torch.Tensor] = None, - kv_indices: Optional[torch.Tensor] = None, - kv_last_page_lens: Optional[torch.Tensor] = None, - sm_scale: float = 1.0, - logit_cap: float = 0.0, -) -> None: - mla_decode_fwd( - q, - kv_buffer.view(-1, 1, 1, q.shape[-1]), - o, - qo_indptr, - kv_indptr, - kv_indices, - kv_last_page_lens, - max_seqlen_qo, - sm_scale=sm_scale, - logit_cap=logit_cap, - ) - -def _forward_rope_fake( - positions: torch.Tensor, - # if is_nope_first - # [[batch_size, seq_len, num_heads, nope_size+rope_size] - # if NOT is_nope_first - # [[batch_size, seq_len, num_heads, rope_size+nope_size], - query: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, - is_neox_style: bool, - head_size: int, - rotary_dim: int, - key: Optional[torch.Tensor] = None, - offsets: Optional[torch.Tensor] = None, - is_nope_first: bool=False, -) -> Tuple[torch.Tensor, torch.Tensor]: - query_shape = query.shape - if key is not None: - key_shape = key.shape - return query.view(query_shape), key.view(key_shape) - else: - return query.view(query_shape) - -@torch_compile_guard(gen_fake=_forward_rope_fake) -def _forward_rope( - positions: torch.Tensor, - # if is_nope_first - # [[batch_size, seq_len, num_heads, nope_size+rope_size] - # if NOT is_nope_first - # [[batch_size, seq_len, num_heads, rope_size+nope_size], - query: torch.Tensor, - cos: torch.Tensor, - sin: torch.Tensor, - is_neox_style: bool, - head_size: int, - rotary_dim: int, - key: Optional[torch.Tensor] = None, - offsets: Optional[torch.Tensor] = None, - is_nope_first: bool=False, -) -> Tuple[torch.Tensor, torch.Tensor]: - import aiter - - assert ( - cos.dtype == query.dtype - ), f"cos_cache dtype ({cos.dtype}) does not match query dtype ({query.dtype})" - - rotate_style = 0 if is_neox_style else 1 - - num_tokens = positions.numel() - - query_shape = query.shape - query = query.view(1, num_tokens, -1, head_size) - if key is not None: - key_shape = key.shape - key = key.view(1, num_tokens, -1, head_size) - - positions = positions.view(*query.shape[:2]) - if offsets is not None: - offsets = offsets.view(*query.shape[:2]) - - if not is_nope_first: - query_ = query[..., : rotary_dim] - key_ = key[..., : rotary_dim] if key is not None else None - else: - query_ = query[..., -rotary_dim:] - key_ = key[..., -rotary_dim:] if key is not None else None - - if key_ is not None: - if offsets is None: - aiter.rope_cached_positions_2c_fwd_inplace( - query_, - key_, - cos, - sin, - positions, - rotate_style, - reuse_freqs_front_part=True, - nope_first=is_nope_first, - ) - else: - aiter.rope_cached_positions_offsets_2c_fwd_inplace( - query_, - key_, - cos, - sin, - positions, - offsets, - rotate_style, - reuse_freqs_front_part=True, - nope_first=is_nope_first, - ) - return query.view(query_shape), key.view(key_shape) - else: - if offsets is None: - aiter.rope_cached_positions_fwd_inplace( - query_, - cos, - sin, - positions, - rotate_style, - reuse_freqs_front_part=True, - nope_first=is_nope_first, - ) - else: - aiter.rope_cached_positions_offsets_fwd_inplace( - query_, - cos, - sin, - positions, - offsets, - rotate_style, - reuse_freqs_front_part=True, - nope_first=is_nope_first, - ) - return query.view(query_shape) - - -def _q_proj_and_k_up_proj(x, q_proj_weight_data, q_proj_weight_scale_data, q_proj_tp_dim, q_proj_tp_size, q_proj_reduce_results, W_K, W_K_scale, num_heads, qk_head_dim, qk_nope_head_dim, qk_rope_head_dim): - q_nope, q_pe = ( - torch.split(_forward_linear_fp8_no_bias(x, q_proj_weight_data, q_proj_weight_scale_data, q_proj_tp_dim, q_proj_tp_size, q_proj_reduce_results) - .view(-1, num_heads, qk_head_dim), - [qk_nope_head_dim, qk_rope_head_dim], dim=-1) - ) - - # Convert from (B, N, P) to (N, B, P) - q_nope = q_nope.transpose(0, 1) - # Multiply (N, B, P) x (N, P, L) -> (N, B, L), Convert from (N, B, L) to (B, N, L) - # ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) - ql_nope = torch.ops.aiter.aiter_triton_fp8_bmm( - q_nope, W_K, W_K_scale, group_size=128, transpose_bm=True - ) - return ql_nope, q_pe - - -def _forward_prefill( - q: torch.Tensor, - kv_c_normed: torch.Tensor, - k_pe: torch.Tensor, - kv_c_and_k_pe_cache: torch.Tensor, - kv_b_proj_weight_data, - kv_b_proj_weight_scale_data, - kv_b_proj_tp_dim, - kv_b_proj_tp_size, - kv_b_proj_reduce_results, - o_proj_weight_data, - o_proj_weight_scale_data, - o_proj_tp_dim, - o_proj_tp_size, - o_proj_reduce_results, - num_heads, - qk_nope_head_dim, - v_head_dim, - scale, - context: Context, -) -> torch.Tensor: - assert context is not None - - kv_nope = _forward_linear_fp8_no_bias(kv_c_normed, kv_b_proj_weight_data, kv_b_proj_weight_scale_data, kv_b_proj_tp_dim, kv_b_proj_tp_size, kv_b_proj_reduce_results).view( - -1, num_heads, qk_nope_head_dim + v_head_dim - ) - k_nope, v = torch.split(kv_nope, [qk_nope_head_dim, v_head_dim], dim=-1) - - if k_pe.dim() == 2: - k_pe = k_pe.unsqueeze(1) - k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1) - - output = flash_attn_varlen_func( - q=q, - k=k, - v=v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q=context.min_seqlen_q, - dropout_p=context.dropout_p, - softmax_scale=scale, - causal=True, - ) - - # return o_proj(output.flatten(start_dim=-2)) - return _forward_linear_fp8_no_bias(output.flatten(start_dim=-2), o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results) - - -def _v_up_proj_and_o_proj(x, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, W_V, W_V_scale, num_heads, kv_lora_rank, v_head_dim): - # Convert from (B, N, L) to (N, B, L) - x = x.view(-1, num_heads, kv_lora_rank).transpose(0, 1) - # Multiply (N, B, L) x (N, L, V) -> (N, B, V), Convert from (N, B, V) to (B, N, V) - # x = torch.bmm(x, self.W_UV).transpose(0, 1) - x = torch.ops.aiter.aiter_triton_fp8_bmm( - x, W_V, W_V_scale, group_size=128, transpose_bm=True - ) - # Convert from (B, N, V) to (B, N * V) - x = x.reshape(-1, num_heads * v_head_dim) - # return o_proj(x) - return _forward_linear_fp8_no_bias(x, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results) - - -def _forward_decode( - q_nope: torch.Tensor, - q_pe: torch.Tensor, - kv_c_and_k_pe_cache: torch.Tensor, - o_proj_weight_data, - o_proj_weight_scale_data, - o_proj_tp_dim, - o_proj_tp_size, - o_proj_reduce_results, - W_V, - W_V_scale, - num_heads, - kv_lora_rank, - v_head_dim, - scale, - context: Context, -) -> torch.Tensor: - assert kv_c_and_k_pe_cache.numel() > 0 - assert context is not None - B = q_nope.shape[0] - - q = torch.cat([q_nope, q_pe], dim=-1) - o = torch.empty( - B, num_heads, kv_lora_rank, dtype=q.dtype, device=q.device - ) - - kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2) - - aiter_mla_decode_fwd( - q, - kv_buffer, - o, - context.cu_seqlens_q, - context.max_q_len, - context.kv_indptr, - context.kv_indices, - context.kv_last_page_lens, - scale, - ) - - return _v_up_proj_and_o_proj(o, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, W_V, W_V_scale, num_heads, kv_lora_rank, v_head_dim) - - -def _forward_linear_fp8_no_bias(x: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, - tp_dim: int, tp_size: int, reduce_results: bool - ) -> torch.Tensor: - quant_type = QuantType.per_1x128 - quant_func = get_hip_quant(quant_type) - x, x_scale = quant_func( - x, - quant_dtype=dtypes.fp8, - scale=None, - ) - y = gemm_a8w8_blockscale( - x, weight, x_scale, weight_scale, dtype=dtypes.bf16 - ) - if tp_dim == 1 and tp_size > 1 and reduce_results: - y = get_tp_group().all_reduce(y, ca_fp8_quant=False) - return y - -def fake_mla_attention( - q: torch.Tensor, # query in unified attn - k_c_normed: torch.Tensor, # key in unified attn - k_pe: torch.Tensor, # value in unified attn - positions: torch.Tensor, - W_K: torch.Tensor, - W_V: torch.Tensor, - W_K_scale: torch.Tensor, - W_V_scale: torch.Tensor, - rotary_emb_cos_cache: torch.Tensor, - rotary_emb_sin_cache: torch.Tensor, - rotary_emb_is_neox_style: bool, - rotary_emb_head_size: int, - rotary_emb_rotary_dim: int, - q_proj_weight_data: torch.Tensor, - q_proj_weight_scale_data: torch.Tensor, - q_proj_tp_dim: int, - q_proj_tp_size: int, - q_proj_reduce_results: bool, - kv_b_proj_weight_data: torch.Tensor, - kv_b_proj_weight_scale_data: torch.Tensor, - kv_b_proj_tp_dim: int, - kv_b_proj_tp_size: int, - kv_b_proj_reduce_results: bool, - o_proj_weight_data: torch.Tensor, - o_proj_weight_scale_data: torch.Tensor, - o_proj_tp_dim: int, - o_proj_tp_size: int, - o_proj_reduce_results: bool, - kv_lora_rank: int, - num_heads: int, - qk_nope_head_dim: int, - qk_rope_head_dim: int, - qk_head_dim: int, - v_head_dim: int, - kv_cache_dtype: str, - scale: float, - layer_num: int -) -> torch.Tensor: - output_shape = list(q.shape) - output_shape[-1] = 7168 # TODO: read from hidden_size - return torch.zeros(output_shape, dtype=q.dtype, device=q.device) - - -@mark_spliting_op(is_custom=True, gen_fake=fake_mla_attention) -def mla_attention( - q: torch.Tensor, # query in unified attn - k_c_normed: torch.Tensor, # key in unified attn - k_pe: torch.Tensor, # value in unified attn - positions: torch.Tensor, - W_K: torch.Tensor, - W_V: torch.Tensor, - W_K_scale: torch.Tensor, - W_V_scale: torch.Tensor, - rotary_emb_cos_cache: torch.Tensor, - rotary_emb_sin_cache: torch.Tensor, - rotary_emb_is_neox_style: bool, - rotary_emb_head_size: int, - rotary_emb_rotary_dim: int, - q_proj_weight_data: torch.Tensor, - q_proj_weight_scale_data: torch.Tensor, - q_proj_tp_dim: int, - q_proj_tp_size: int, - q_proj_reduce_results: bool, - kv_b_proj_weight_data: torch.Tensor, - kv_b_proj_weight_scale_data: torch.Tensor, - kv_b_proj_tp_dim: int, - kv_b_proj_tp_size: int, - kv_b_proj_reduce_results: bool, - o_proj_weight_data: torch.Tensor, - o_proj_weight_scale_data: torch.Tensor, - o_proj_tp_dim: int, - o_proj_tp_size: int, - o_proj_reduce_results: bool, - kv_lora_rank: int, - num_heads: int, - qk_nope_head_dim: int, - qk_rope_head_dim: int, - qk_head_dim: int, - v_head_dim: int, - kv_cache_dtype: str, - scale: float, - layer_num: int -) -> torch.Tensor: - context = get_context() - if context.slot_mapping.numel(): - forward_context: ForwardContext = get_forward_context() - attn_metadata_ = forward_context.no_compile_layers[layer_num] - kv_cache = attn_metadata_.k_cache - else: - # dummy run before allocate kv_cache, thus we create manually - kv_cache = torch.tensor([]) - - _k_scale = torch.tensor(1.0, dtype=torch.float32) - - if context.is_prefill: - prefill_q = _forward_linear_fp8_no_bias(q, q_proj_weight_data, q_proj_weight_scale_data, - q_proj_tp_dim, q_proj_tp_size, q_proj_reduce_results).view(-1, num_heads, qk_head_dim) - prefill_q_pe = prefill_q[..., qk_nope_head_dim:] - _forward_rope(positions, prefill_q_pe, rotary_emb_cos_cache, rotary_emb_sin_cache, - rotary_emb_is_neox_style, rotary_emb_head_size, rotary_emb_rotary_dim, k_pe) - - if kv_cache.numel() > 0: - concat_and_cache_mla( - k_c_normed, - k_pe.squeeze(1), - kv_cache, - context.slot_mapping.flatten(), - kv_cache_dtype=kv_cache_dtype, - scale=_k_scale, - ) - - output = _forward_prefill( - prefill_q, k_c_normed, k_pe, kv_cache, kv_b_proj_weight_data, kv_b_proj_weight_scale_data, kv_b_proj_tp_dim, kv_b_proj_tp_size, kv_b_proj_reduce_results, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, num_heads, qk_nope_head_dim, v_head_dim, scale, context - ) - else: - decode_ql_nope, decode_q_pe = _q_proj_and_k_up_proj(q, q_proj_weight_data, q_proj_weight_scale_data, q_proj_tp_dim, - q_proj_tp_size, q_proj_reduce_results, W_K, W_K_scale, num_heads, qk_head_dim, qk_nope_head_dim, qk_rope_head_dim) - _forward_rope(positions, decode_q_pe, rotary_emb_cos_cache, rotary_emb_sin_cache, - rotary_emb_is_neox_style, rotary_emb_head_size, rotary_emb_rotary_dim, k_pe) - - if kv_cache.numel() > 0: - concat_and_cache_mla( - k_c_normed, - k_pe.squeeze(1), - kv_cache, - context.slot_mapping.flatten(), - kv_cache_dtype=kv_cache_dtype, - scale=_k_scale, - ) - - output = _forward_decode( - decode_ql_nope, decode_q_pe, kv_cache, o_proj_weight_data, o_proj_weight_scale_data, o_proj_tp_dim, o_proj_tp_size, o_proj_reduce_results, W_V, W_V_scale, num_heads, kv_lora_rank, v_head_dim, scale, context - ) - - return output - - -# direct_register_custom_op( -# op_name="mla_attention", -# op_func=mla_attention, -# mutates_args=[], -# fake_impl=fake_mla_attention, -# ) - - class MLAAttention(nn.Module): def __init__( self, @@ -508,18 +68,8 @@ def __init__( scale: float, num_kv_heads: int, kv_cache_dtype: str, - # MLA Specific Arguments - q_lora_rank: Optional[int], - kv_lora_rank: int, - qk_nope_head_dim: int, - qk_rope_head_dim: int, - qk_head_dim: int, - v_head_dim: int, - rotary_emb: RotaryEmbedding, - q_proj: ColumnParallelLinear, - kv_b_proj: ColumnParallelLinear, - o_proj: RowParallelLinear, layer_num: int = 0, + mla_modules: MLAModules=None, ) -> None: super().__init__() self.num_heads = num_heads @@ -528,16 +78,16 @@ def __init__( self.num_kv_heads = num_kv_heads self.kv_cache_dtype = kv_cache_dtype if kv_cache_dtype == "fp8" else "auto" - self.q_lora_rank = q_lora_rank - self.kv_lora_rank = kv_lora_rank - self.qk_nope_head_dim = qk_nope_head_dim - self.qk_rope_head_dim = qk_rope_head_dim - self.qk_head_dim = qk_head_dim - self.v_head_dim = v_head_dim - self.rotary_emb = rotary_emb - self.q_proj = q_proj - self.o_proj = o_proj - self.kv_b_proj = kv_b_proj + self.q_lora_rank = mla_modules.q_lora_rank + self.kv_lora_rank = mla_modules.kv_lora_rank + self.qk_nope_head_dim = mla_modules.qk_nope_head_dim + self.qk_rope_head_dim = mla_modules.qk_rope_head_dim + self.qk_head_dim = mla_modules.qk_head_dim + self.v_head_dim = mla_modules.v_head_dim + self.rotary_emb = mla_modules.rotary_emb + self.q_proj = mla_modules.q_proj + self.o_proj = mla_modules.o_proj + self.kv_b_proj = mla_modules.kv_b_proj self.kv_cache = torch.tensor([]) self._k_scale = torch.tensor(1.0, dtype=torch.float32) self.layer_num = layer_num @@ -559,9 +109,9 @@ def process_weights_after_loading(self): self.num_heads, self.qk_nope_head_dim + self.v_head_dim, ) - W_UK, W_UV = torch.split(kv_b_proj_weight, - [self.qk_nope_head_dim, self.v_head_dim], dim=-1 - ) + W_UK, W_UV = kv_b_proj_weight.split( + [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + ) if True: # is_rocm_aiter_fp8bmm_enabled(): W_K = W_UK.transpose(0, 1) # 16 512 128 @@ -592,7 +142,7 @@ def process_weights_after_loading(self): dtype=torch.bfloat16, device=self.W_K.device, ) - torch.ops.aiter.aiter_triton_fp8_bmm( + _aiter_triton_fp8_bmm( x, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True ) @@ -601,7 +151,7 @@ def process_weights_after_loading(self): dtype=torch.bfloat16, device=self.W_V.device, ) - torch.ops.aiter.aiter_triton_fp8_bmm( + _aiter_triton_fp8_bmm( x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True ) else: @@ -610,6 +160,127 @@ def process_weights_after_loading(self): # Convert from (L, N, P) to (N, P, L) self.W_UK_T = W_UK.permute(1, 2, 0) + def _aiter_mla_decode_fwd( + self, + q: torch.Tensor, + kv_buffer: torch.Tensor, + o: torch.Tensor, + qo_indptr: torch.Tensor, + max_seqlen_qo: int, + kv_indptr: Optional[torch.Tensor] = None, + kv_indices: Optional[torch.Tensor] = None, + kv_last_page_lens: Optional[torch.Tensor] = None, + sm_scale: float = 1.0, + logit_cap: float = 0.0, + ) -> None: + mla_decode_fwd( + q, + kv_buffer.view(-1, 1, 1, q.shape[-1]), + o, + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_lens, + max_seqlen_qo, + sm_scale=sm_scale, + logit_cap=logit_cap, + ) + + def _v_up_proj_and_o_proj(self, x): + # Convert from (B, N, L) to (N, B, L) + x = x.view(-1, self.num_heads, self.kv_lora_rank).transpose(0, 1) + # Multiply (N, B, L) x (N, L, V) -> (N, B, V), Convert from (N, B, V) to (B, N, V) + # x = torch.bmm(x, self.W_UV).transpose(0, 1) + x = _aiter_triton_fp8_bmm( + x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True + ) + # Convert from (B, N, V) to (B, N * V) + x = x.reshape(-1, self.num_heads * self.v_head_dim) + return self.o_proj(x) + + def _q_proj_and_k_up_proj(self, x): + q_nope, q_pe = ( + self.q_proj(x) + .view(-1, self.num_heads, self.qk_head_dim) + .split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + ) + + # Convert from (B, N, P) to (N, B, P) + q_nope = q_nope.transpose(0, 1) + # Multiply (N, B, P) x (N, P, L) -> (N, B, L), Convert from (N, B, L) to (B, N, L) + # ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1) + ql_nope = _aiter_triton_fp8_bmm( + q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True + ) + return ql_nope, q_pe + + def _forward_prefill( + self, + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + context: Context, + ) -> torch.Tensor: + assert context is not None + + kv_nope = self.kv_b_proj(kv_c_normed).view( + -1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim + ) + k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) + + if k_pe.dim() == 2: + k_pe = k_pe.unsqueeze(1) + k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1) + + output = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=context.cu_seqlens_q, + cu_seqlens_k=context.cu_seqlens_k, + max_seqlen_q=context.max_seqlen_q, + max_seqlen_k=context.max_seqlen_k, + min_seqlen_q=context.min_seqlen_q, + dropout_p=context.dropout_p, + softmax_scale=self.scale, + causal=True, + ) + + return self.o_proj(output.flatten(start_dim=-2)) + + def _forward_decode( + self, + q_nope: torch.Tensor, + q_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + context: Context, + ) -> torch.Tensor: + assert kv_c_and_k_pe_cache.numel() > 0 + assert context is not None + B = q_nope.shape[0] + + q = torch.cat([q_nope, q_pe], dim=-1) + o = torch.empty( + B, self.num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device + ) + + kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2) + + self._aiter_mla_decode_fwd( + q, + kv_buffer, + o, + context.cu_seqlens_q, + context.max_q_len, + context.kv_indptr, + context.kv_indices, + context.kv_last_page_lens, + self.scale, + ) + + return self._v_up_proj_and_o_proj(o) + def forward( self, q: torch.Tensor, # query in unified attn @@ -617,11 +288,50 @@ def forward( k_pe: torch.Tensor, # value in unified attn positions: torch.Tensor, ) -> torch.Tensor: - return torch.ops.aiter.mla_attention(q, k_c_normed, k_pe, positions, self.W_K, self.W_V, self.W_K_scale, self.W_V_scale, - self.rotary_emb.cos_cache, self.rotary_emb.sin_cache, self.rotary_emb.is_neox_style, self.rotary_emb.head_size, self.rotary_emb.rotary_dim, - self.q_proj.weight, self.q_proj.weight_scale, self.q_proj.tp_dim, self.q_proj.tp_size, self.q_proj.reduce_results, - self.kv_b_proj.weight, self.kv_b_proj.weight_scale, self.kv_b_proj.tp_dim, self.kv_b_proj.tp_size, self.kv_b_proj.reduce_results, - self.o_proj.weight, self.o_proj.weight_scale, self.o_proj.tp_dim, self.o_proj.tp_size, self.o_proj.reduce_results, - self.kv_lora_rank, self.num_heads, - self.qk_nope_head_dim, self.qk_rope_head_dim, self.qk_head_dim, self.v_head_dim, self.kv_cache_dtype, - self.scale, self.layer_num) + # kv_cache = self.kv_cache + context = get_context() + if context.slot_mapping.numel(): + forward_context: ForwardContext = get_forward_context() + attn_metadata_ = forward_context.no_compile_layers[self.layer_num] + kv_cache = attn_metadata_.k_cache + else: + # dummy run before allocate kv_cache, thus we create manually + kv_cache = torch.tensor([]) + + if context.is_prefill: + prefill_q = self.q_proj(q).view(-1, self.num_heads, self.qk_head_dim) + prefill_q_pe = prefill_q[..., self.qk_nope_head_dim :] + self.rotary_emb(positions, prefill_q_pe, k_pe) + + if kv_cache.numel() > 0: + concat_and_cache_mla( + k_c_normed, + k_pe.squeeze(1), + kv_cache, + context.slot_mapping.flatten(), + kv_cache_dtype=self.kv_cache_dtype, + scale=self._k_scale, + ) + + output = self._forward_prefill( + prefill_q, k_c_normed, k_pe, kv_cache, context + ) + else: + decode_ql_nope, decode_q_pe = self._q_proj_and_k_up_proj(q) + self.rotary_emb(positions, decode_q_pe, k_pe) + + if kv_cache.numel() > 0: + concat_and_cache_mla( + k_c_normed, + k_pe.squeeze(1), + kv_cache, + context.slot_mapping.flatten(), + kv_cache_dtype=self.kv_cache_dtype, + scale=self._k_scale, + ) + + output = self._forward_decode( + decode_ql_nope, decode_q_pe, kv_cache, context + ) + + return output diff --git a/atom/model_ops/attentions/aiter_attention.py b/atom/model_ops/attentions/aiter_attention.py index cfa8431941..ffbd291099 100644 --- a/atom/model_ops/attentions/aiter_attention.py +++ b/atom/model_ops/attentions/aiter_attention.py @@ -6,6 +6,7 @@ from atom.model_engine.scheduler import ScheduledBatch from atom.utils.context import set_context +from atom.model_ops.attention_mha import Attention class AiterBackend(AttentionBackend): @@ -17,6 +18,10 @@ def get_name() -> str: def get_builder_cls() -> Type["AiterAttentionMetadataBuilder"]: return AiterAttentionMetadataBuilder + @staticmethod + def get_impl_cls() -> Type["Attention"]: + return Attention + class AiterAttentionMetadataBuilder(CommonAttentionBuilder): BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index 5eb46fff8b..c54db5183b 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -6,6 +6,7 @@ from atom.model_engine.scheduler import ScheduledBatch from atom.utils.context import set_context +from atom.model_ops.attention_mla import MLAAttention class AiterMLABackend(AttentionBackend): @@ -17,6 +18,11 @@ def get_name() -> str: def get_builder_cls() -> Type["AiterMLAMetadataBuilder"]: return AiterMLAMetadataBuilder + @staticmethod + def get_impl_cls() -> Type["MLAAttention"]: + return MLAAttention + + class AiterMLAMetadataBuilder(CommonAttentionBuilder): BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py index 3d70a0a708..f11e3c92e3 100644 --- a/atom/model_ops/attentions/backends.py +++ b/atom/model_ops/attentions/backends.py @@ -2,8 +2,10 @@ from typing import Type, Tuple, Generic, Optional, List, TypeVar, Dict, Any from atom.model_engine.scheduler import ScheduledBatch from atom.utils.context import set_context +from atom.model_ops.attention_mla import MLAModules import torch +from torch import nn T = TypeVar('T', bound="BroadcastableModelInput") @@ -49,14 +51,8 @@ def get_builder_cls() -> Type["AttentionMetadataBuilder"]: raise NotImplementedError @staticmethod - @abstractmethod - def get_kv_cache_shape( - num_blocks: int, - block_size: int, - num_kv_heads: int, - head_size: int, - ) -> Tuple[int, ...]: - raise NotImplementedError + def get_impl_cls() -> Type["AttentionImpl"]: + return AttentionImpl @@ -64,7 +60,7 @@ class AttentionMetadataBuilder(ABC, Generic[T]): """Abstract class for attention metadata builders.""" @abstractmethod - def __init__(self, inpblock_size: int) -> None: + def __init__(self, block_size: int) -> None: """Create the builder, remember some configuration and parameters.""" raise NotImplementedError @@ -135,3 +131,29 @@ def prepare_prefill(self, batch: ScheduledBatch, forward_vars): **ctx, ) return var["positions"].copy_to_gpu(sum_scheduled_tokens) + + + +class AttentionImpl(nn.Module): + @abstractmethod + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: Optional[int] = None, + kv_cache_dtype: str = "auto", + layer_num: int = 0, + mla_modules: MLAModules=None, + ) -> None: + raise NotImplementedError + + @abstractmethod + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + position: torch.Tensor=None, + ) -> torch.Tensor: + raise NotImplementedError diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py new file mode 100644 index 0000000000..5eb8ac48da --- /dev/null +++ b/atom/model_ops/base_attention.py @@ -0,0 +1,98 @@ +# from flash_attn import flash_attn_with_kvcache +from dataclasses import dataclass + +import aiter +import torch +import triton +import triton.language as tl +from aiter.paged_attn import PagedAttention +from torch import nn + +from atom.utils.context import get_context +from atom.utils.custom_register import direct_register_custom_op +from atom.utils.forward_context import ( + AttentionMetadata, + ForwardContext, + get_forward_context, + set_forward_context, +) +from atom.utils import mark_spliting_op +from .attention_mla import MLAModules +from atom.config import get_current_atom_config +from atom.utils.selector import get_attn_backend + +def fake_( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, positions: torch.Tensor, + layer_name: str, use_mla: bool +) -> torch.Tensor: + output_shape = list(q.shape) + if use_mla: + output_shape[-1] = 7168 + output = torch.zeros(output_shape, + dtype=q.dtype, + device=q.device) + + return output + +# Dynamo will not try to inspect any of the internal operations for prefill or decode +# This way, although attention operation is complicated, +# we can still capture the model's computation graph as a full-graph +@mark_spliting_op(is_custom=True, gen_fake=fake_, mutates_args=[]) +def unified_attention_with_output_base( + q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, positions: torch.Tensor, + layer_name: str, use_mla: bool +) -> torch.Tensor: + atom_config = get_current_atom_config() + self = atom_config.compilation_config.static_forward_context[layer_name] + return self.impl.forward(q, k, v, positions) + + +class Attention(nn.Module): + + def __init__( + self, + num_heads, + head_dim, + scale, + num_kv_heads, + kv_cache_dtype="bf16", + layer_num=0, + use_mla: bool = False, + mla_modules: MLAModules=None, + **kwargs, + ): + super().__init__() + self.num_heads = num_heads + self.head_dim = head_dim + self.scale = scale + self.num_kv_heads = num_kv_heads + self.k_cache = self.v_cache = torch.tensor([]) + self.kv_cache_dtype = kv_cache_dtype + self.max_model_len = 0 + self.k_scale = self.v_scale = None + self.layer_num = layer_num + self.mla_modules = mla_modules + self.use_mla = use_mla + self.base_attention = None + self.kv_cache = torch.tensor([]) + + atom_config = get_current_atom_config() + block_size = atom_config.kv_cache_block_size + self.attn_backend = get_attn_backend( + block_size, + use_mla=self.use_mla, + ) + impl_cls = self.attn_backend.get_impl_cls() + self.impl = impl_cls(num_heads, head_dim, scale, num_kv_heads, + kv_cache_dtype, layer_num, mla_modules) + + compilation_config = atom_config.compilation_config + self.layer_name = f"MLA_{layer_num}" if self.use_mla else f"MHA_{layer_num}" + if self.layer_name in compilation_config.static_forward_context: + raise ValueError("Duplicate layer: {}".format(self.layer_name)) + compilation_config.static_forward_context[self.layer_name] = self + + + def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, positions: torch.Tensor=None): + output = torch.ops.aiter.unified_attention_with_output_base(q, k, v, positions, self.layer_name, self.use_mla) + return output diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index 5e00a1e052..424a68eece 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -28,7 +28,8 @@ from torch import nn from transformers import PretrainedConfig -from atom.model_ops.attention_mla import MLAAttention +from atom.model_ops.attention_mla import MLAModules +from atom.model_ops.base_attention import Attention from atom.config import QuantizationConfig, Config from aiter.dist.parallel_state import (get_pp_group, get_tensor_model_parallel_world_size) @@ -298,13 +299,8 @@ def __init__( # k_c.size(1) + k_pe.size(1) == kv_cache.size(2) # i.e. # kv_lora_rank + qk_rope_head_dim == head_size - self.mla_attn = MLAAttention( - num_heads=self.num_local_heads, - head_size=self.kv_lora_rank + self.qk_rope_head_dim, - scale=self.scaling, - num_kv_heads=1, - kv_cache_dtype=cache_config, - # MLA Args + + mla_modules = MLAModules( q_lora_rank=self.q_lora_rank, kv_lora_rank=self.kv_lora_rank, qk_nope_head_dim=self.qk_nope_head_dim, @@ -315,7 +311,17 @@ def __init__( q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj, kv_b_proj=self.kv_b_proj, o_proj=self.o_proj, + ) + + self.mla_attn = Attention( + num_heads=self.num_local_heads, + head_dim=self.kv_lora_rank + self.qk_rope_head_dim, + scale=self.scaling, + num_kv_heads=1, + kv_cache_dtype=cache_config, layer_num=layer_num, + use_mla=True, + mla_modules=mla_modules ) self.prefix = prefix diff --git a/atom/models/llama.py b/atom/models/llama.py index 7f4014ea2a..faf9cfb954 100644 --- a/atom/models/llama.py +++ b/atom/models/llama.py @@ -31,7 +31,9 @@ from transformers import LlamaConfig from aiter import QuantType -from atom.model_ops.attention import Attention +# from atom.model_ops.attention import Attention +from atom.model_ops.base_attention import Attention + from aiter.dist.parallel_state import ( get_pp_group, get_tp_group, diff --git a/atom/models/mixtral.py b/atom/models/mixtral.py index 2f2a18166c..7a3a34e14e 100644 --- a/atom/models/mixtral.py +++ b/atom/models/mixtral.py @@ -31,7 +31,8 @@ from transformers import MixtralConfig from aiter import QuantType -from atom.model_ops.attention import Attention +# from atom.model_ops.attention import Attention +from atom.model_ops.base_attention import Attention from aiter.dist.parallel_state import ( get_pp_group, get_tp_group, diff --git a/atom/models/qwen3.py b/atom/models/qwen3.py index 8fb0bf6ccc..b6e4039090 100644 --- a/atom/models/qwen3.py +++ b/atom/models/qwen3.py @@ -8,7 +8,8 @@ from atom.config import QuantizationConfig, Config from atom.model_ops.activation import SiluAndMul -from atom.model_ops.attention import Attention +# from atom.model_ops.attention import Attention +from atom.model_ops.base_attention import Attention from atom.model_ops.layernorm import RMSNorm from atom.model_ops.linear import ( QKVParallelLinear, @@ -81,6 +82,7 @@ def __init__( self.num_kv_heads, kv_cache_dtype=kv_cache_dtype, layer_num=layer_num, + use_mla=False, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) diff --git a/atom/utils/selector.py b/atom/utils/selector.py index d020485619..0515cb9da3 100644 --- a/atom/utils/selector.py +++ b/atom/utils/selector.py @@ -3,34 +3,7 @@ from abc import ABC, abstractmethod from functools import cache from atom.utils import resolve_obj_by_qualname -from atom.model_ops.attentions.backends import AttentionMetadataBuilder - -class AttentionBackend(ABC): - """Abstract class for attention backends.""" - # For some attention backends, we allocate an output tensor before - # calling the custom op. When piecewise cudagraph is enabled, this - # makes sure the output tensor is allocated inside the cudagraph. - accept_output_buffer: bool = False - - @staticmethod - @abstractmethod - def get_name() -> str: - raise NotImplementedError - - @staticmethod - @abstractmethod - def get_builder_cls() -> Type["AttentionMetadataBuilder"]: - raise NotImplementedError - - @staticmethod - @abstractmethod - def get_kv_cache_shape( - num_blocks: int, - block_size: int, - num_kv_heads: int, - head_size: int, - ) -> Tuple[int, ...]: - raise NotImplementedError +from atom.model_ops.attentions.backends import AttentionBackend def get_attn_backend( From 0baea4f023ee6edd010135349f10e42c4dd4de33 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Wed, 29 Oct 2025 13:20:38 +0800 Subject: [PATCH 098/119] add serving benchmark (#21) * add serving benchmark * add request support list * fix list for generate --- README.md | 23 ++ atom/bench/benchmark_throughput.py | 217 +++++++++++++++ atom/bench/example.py | 94 +------ atom/bench/one_batch.py | 26 +- atom/entrypoints/openai_server.py | 395 +++++++++++++-------------- atom/model_engine/arg_utils.py | 170 ++++++++++++ atom/model_engine/engine_core.py | 20 +- atom/model_engine/engine_core_mgr.py | 6 +- atom/model_engine/llm_engine.py | 83 +++++- 9 files changed, 696 insertions(+), 338 deletions(-) create mode 100644 atom/bench/benchmark_throughput.py create mode 100644 atom/model_engine/arg_utils.py diff --git a/README.md b/README.md index 968b6efdbd..64e363f61e 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,18 @@ Run performance tests to compare ATOM against vLLM: python -m atom.bench.one_batch --model Qwen/Qwen3-0.6B ``` +Run online throughput benchmark: + +start the server +```bash +python -m atom.entrypoints.openai_server --model Qwen/Qwen3-0.6B --cudagraph-capture-sizes [1,2,4,8,16,32,64,128] +``` +run benchmark +```bash +python -m atom.bench.benchmark_throughput -n 128 -r 128 -c 64 -i 1020 -o 1024 +``` + + ## πŸ“Š Performance Comparison ATOM demonstrates significant performance improvements over vLLM: @@ -79,6 +91,17 @@ ATOM demonstrates significant performance improvements over vLLM: | **Llama-3.1-8B-Instruct-FP8-KV** | ATOM | 4096 | 0.68s | **5,983.37 tok/s** | | Llama-3.1-8B-Instruct-FP8-KV | vLLM | 4096 | 1.68s | 2,432.62 tok/s | + +### Online serving throughput: + +Deepseek-V3 +| concurrency | IPS/QPS | prompts num | vLLM Throughput | ATOM Throughput | +|-------------|---------|-------------|-----------------|-----------------| +| 16 | 1024/1024 | 128 | 423.68 tok/s | **922.03 tok/s** | +| 32 | 1024/1024 | 128 | 629.06 tok/s | **1488.52 tok/s** | +| 64 | 1024/1024 | 128 | 760.22 tok/s | **2221.25 tok/s** | +| 128 | 1024/1024 | 128 | 1107.93 tok/s | **2254.88 tok/s** | + ### Accuracy Benchmarking First, install `lm-eval` to test model accuracy: diff --git a/atom/bench/benchmark_throughput.py b/atom/bench/benchmark_throughput.py new file mode 100644 index 0000000000..e6552de065 --- /dev/null +++ b/atom/bench/benchmark_throughput.py @@ -0,0 +1,217 @@ +import asyncio +import aiohttp +import time +from typing import List + + +async def send_chat_request(session: aiohttp.ClientSession, url: str, message: str, request_id: int, max_tokens: int = 100): + payload = { + "model": "Qwen/Qwen3-0.6B", + "messages": [ + {"role": "user", "content": message} + ], + "max_tokens": max_tokens, + "temperature": 0.7, + "ignore_eos": True + } + + start_time = time.time() + try: + async with session.post(url, json=payload) as response: + result = await response.json() + latency = time.time() - start_time + + if response.status == 200: + content = result["choices"][0]["message"]["content"] + usage = result["usage"] + print(f"βœ“ Request {request_id} completed in {latency:.2f}s") + print(f" Prompt: {message[:50]}...") + print(f" Response: {content[:100]}...") + print(f" Tokens: {usage['prompt_tokens']} input, {usage['completion_tokens']} output") + return {"success": True, "latency": latency, "request_id": request_id} + else: + print(f"βœ— Request {request_id} failed: {response.status}") + return {"success": False, "latency": latency, "request_id": request_id} + + except Exception as e: + latency = time.time() - start_time + print(f"βœ— Request {request_id} error: {e}") + return {"success": False, "latency": latency, "request_id": request_id, "error": str(e)} + + +async def send_completion_request(session: aiohttp.ClientSession, url: str, prompt: str, request_id: int, max_tokens: int = 100): + payload = { + "model": "Qwen/Qwen3-0.6B", + "prompt": prompt, + "max_tokens": max_tokens, + "temperature": 0.7, + "ignore_eos": True + } + + start_time = time.time() + try: + async with session.post(url, json=payload) as response: + result = await response.json() + latency = time.time() - start_time + + if response.status == 200: + text = result["choices"][0]["text"] + usage = result["usage"] + print(f"βœ“ Request {request_id} completed in {latency:.2f}s") + print(f" Prompt: {prompt[:50]}...") + print(f" Response: {text[:100]}...") + print(f" Tokens: {usage['prompt_tokens']} input, {usage['completion_tokens']} output") + return {"success": True, "latency": latency, "request_id": request_id} + else: + print(f"βœ— Request {request_id} failed: {response.status}") + return {"success": False, "latency": latency, "request_id": request_id} + + except Exception as e: + latency = time.time() - start_time + print(f"βœ— Request {request_id} error: {e}") + return {"success": False, "latency": latency, "request_id": request_id, "error": str(e)} + + +def generate_prompt_with_length(target_tokens: int) -> str: + words = ["hello", "world", "this", "is", "a", "test", "prompt", "for", "the", "language", + "model", "testing", "please", "generate", "response", "based", "on", "input"] + + result = [] + tokens_count = 0 + idx = 0 + + while tokens_count < target_tokens: + word = words[idx % len(words)] + result.append(word) + tokens_count += 1 + idx += 1 + + return " ".join(result) + + +async def test_concurrent_requests(base_url: str, num_requests: int = 60, request_rate: float = 2.0, + concurrency: int = 4, use_chat: bool = True, + input_tokens: int = 20, output_tokens: int = 100): + estimated_duration = num_requests / request_rate + + print(f"\n{'='*60}") + print(f"Starting Load Test") + print(f"Server URL: {base_url}") + print(f"Request Type: {'Chat Completion' if use_chat else 'Text Completion'}") + print(f"Total Requests: {num_requests}") + print(f"Request Rate: {request_rate} req/s") + print(f"Max Concurrency: {concurrency}") + print(f"Estimated Duration: ~{estimated_duration:.1f}s") + print(f"Input Tokens: ~{input_tokens} tokens") + print(f"Output Tokens: {output_tokens} tokens") + print(f"{'='*60}\n") + + request_interval = 1.0 / request_rate + + results = [] + active_tasks = set() + sent_count = 0 + + start_time = time.time() + + async with aiohttp.ClientSession() as session: + url = f"{base_url}/v1/chat/completions" if use_chat else f"{base_url}/v1/completions" + + while sent_count < num_requests or active_tasks: + if sent_count < num_requests and len(active_tasks) < concurrency: + sent_count += 1 + prompt = generate_prompt_with_length(input_tokens) + + if use_chat: + task = asyncio.create_task( + send_chat_request(session, url, prompt, sent_count, output_tokens) + ) + else: + task = asyncio.create_task( + send_completion_request(session, url, prompt, sent_count, output_tokens) + ) + + active_tasks.add(task) + task.add_done_callback(lambda t: active_tasks.discard(t)) + + if sent_count < num_requests: + next_request_time = start_time + sent_count * request_interval + sleep_time = max(0, next_request_time - time.time()) + if sleep_time > 0: + await asyncio.sleep(sleep_time) + else: + if active_tasks: + done, active_tasks = await asyncio.wait( + active_tasks, + return_when=asyncio.FIRST_COMPLETED + ) + results.extend([task.result() for task in done]) + + total_time = time.time() - start_time + + total_requests = len(results) + successful = sum(1 for r in results if r["success"]) + failed = total_requests - successful + + if results: + avg_latency = sum(r["latency"] for r in results) / total_requests + min_latency = min(r["latency"] for r in results) + max_latency = max(r["latency"] for r in results) + + latencies = sorted([r["latency"] for r in results]) + p50 = latencies[int(len(latencies) * 0.50)] + p90 = latencies[int(len(latencies) * 0.90)] + p95 = latencies[int(len(latencies) * 0.95)] + p99 = latencies[int(len(latencies) * 0.99)] + else: + avg_latency = min_latency = max_latency = 0 + p50 = p90 = p95 = p99 = 0 + + print(f"\n{'='*60}") + print(f"Test Completed!") + print(f"Total Time: {total_time:.2f}s") + print(f"Total Requests: {total_requests}") + print(f"Successful: {successful}/{total_requests} ({successful*100/max(total_requests,1):.1f}%)") + print(f"Failed: {failed}/{total_requests} ({failed*100/max(total_requests,1):.1f}%)") + print(f"\nLatency Statistics:") + print(f" Average: {avg_latency:.2f}s") + print(f" Min: {min_latency:.2f}s") + print(f" Max: {max_latency:.2f}s") + print(f" P50: {p50:.2f}s") + print(f" P90: {p90:.2f}s") + print(f" P95: {p95:.2f}s") + print(f" P99: {p99:.2f}s") + print(f"\nThroughput:") + print(f" Actual Request Rate: {total_requests/total_time:.2f} req/s") + print(f" Token Throughput: {total_requests*(input_tokens+output_tokens)/total_time:.2f} tokens/s") + print(f"{'='*60}\n") + + +async def main(): + import argparse + + parser = argparse.ArgumentParser(description="OpenAI API Server Load Testing") + parser.add_argument("--url", type=str, default="http://localhost:8000", help="Server URL") + parser.add_argument("--num-requests", "-n", type=int, default=60, help="Total number of requests") + parser.add_argument("--request-rate", "-r", type=float, default=2.0, help="Request rate (requests/second)") + parser.add_argument("--concurrency", "-c", type=int, default=4, help="Max concurrent requests") + parser.add_argument("--type", choices=["chat", "completion"], default="chat", help="Request type") + parser.add_argument("--input-tokens", "-i", type=int, default=1016, help="Input token count (approximate)") + parser.add_argument("--output-tokens", "-o", type=int, default=1024, help="Output token count (max_tokens)") + + args = parser.parse_args() + + use_chat = args.type == "chat" + await test_concurrent_requests( + base_url=args.url, + num_requests=args.num_requests, + request_rate=args.request_rate, + concurrency=args.concurrency, + use_chat=use_chat, + input_tokens=args.input_tokens, + output_tokens=args.output_tokens + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/atom/bench/example.py b/atom/bench/example.py index aeeff6f577..44758b3669 100644 --- a/atom/bench/example.py +++ b/atom/bench/example.py @@ -1,105 +1,33 @@ import argparse -import os -from typing import List import torch from transformers import AutoTokenizer -from atom import LLMEngine, SamplingParams -from atom.config import CompilationConfig +from atom import SamplingParams +from atom.model_engine.arg_utils import EngineArgs parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="config of test", ) -parser.add_argument( - "--kv_cache_dtype", - choices=["bf16", "fp8"], - type=str, - default="bf16", - help="""KV cache type. Default is 'bf16'. - e.g.: -kv_cache_dtype fp8""", -) - -parser.add_argument( - "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." -) - -parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1) - -parser.add_argument( - "--enforce-eager", action="store_true", help="Enforce eager mode execution." -) - -parser.add_argument( - "--enable_prefix_caching", action="store_true", help="Enable prefix caching." -) - -parser.add_argument("--port", type=int, default=8006, help="API server port") - - -parser.add_argument( - "--cudagraph-capture-sizes", - type=str, - default="[1,2,4,8,16]", - help="Sizes to capture cudagraph.", -) - -parser.add_argument("--level", type=int, default=3, help="The level of compilation") - -parser.add_argument( - "--torch-profiler-dir", - type=str, - default=None, - help="Directory to save torch profiler traces", -) +# Add engine arguments +EngineArgs.add_cli_args(parser) -parser.add_argument( - "--enable-expert-parallel", - action="store_true", - help="Enable expert parallel(EP MoE).", -) - -parser.add_argument("--block-size", type=int, default=16, help="KV cache block size.") +# Add example-specific arguments parser.add_argument( "--temperature", type=float, default=0.6, help="temperature for sampling" ) -parser.add_argument( - "--load_dummy", action="store_true", help="Skip loading model weights." -) - - -def parse_size_list(size_str: str) -> List[int]: - import ast - - try: - return ast.literal_eval(size_str) - except (ValueError, SyntaxError) as e: - raise ValueError(f"Error list size: {size_str}") from e - def main(): args = parser.parse_args() - model_name_or_path = args.model - tokenizer = AutoTokenizer.from_pretrained(model_name_or_path) - # If you want to use torch.compile, please --level=3 - llm = LLMEngine( - model_name_or_path, - enforce_eager=args.enforce_eager, - tensor_parallel_size=args.tensor_parallel_size, - kv_cache_dtype=args.kv_cache_dtype, - kv_cache_block_size=args.block_size, - port=args.port, - load_dummy=args.load_dummy, - enable_expert_parallel=args.enable_expert_parallel, - torch_profiler_dir=args.torch_profiler_dir, - compilation_config=CompilationConfig( - level=args.level, - cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes), - ), - ) + + # Create engine from args + engine_args = EngineArgs.from_cli_args(args) + llm = engine_args.create_engine() + + tokenizer = AutoTokenizer.from_pretrained(args.model) sampling_params = SamplingParams(temperature=args.temperature, max_tokens=256) prompts = [ diff --git a/atom/bench/one_batch.py b/atom/bench/one_batch.py index fa4cbc8289..c3fcd54c18 100644 --- a/atom/bench/one_batch.py +++ b/atom/bench/one_batch.py @@ -4,17 +4,16 @@ from transformers import AutoTokenizer -from atom import LLMEngine, SamplingParams -from atom.config import CompilationConfig +from atom import SamplingParams +from atom.model_engine.arg_utils import EngineArgs parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="config of test", ) -parser.add_argument( - "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." -) +# Add engine arguments +EngineArgs.add_cli_args(parser) # A very long prompt, total number of tokens is about 15k. LONG_PROMPT = [ @@ -32,19 +31,10 @@ def main(): max_ouput_len = 1024 args = parser.parse_args() - path = args.model - llm = LLMEngine( - path, - enforce_eager=False, - tensor_parallel_size=1, - kv_cache_block_size=16, - max_model_len=4096, - port=12345, - torch_profiler_dir="./log", - compilation_config=CompilationConfig( - level=3, - ), - ) + + # Create engine from args + engine_args = EngineArgs.from_cli_args(args) + llm = engine_args.create_engine() sampling_params = [ SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=16) diff --git a/atom/entrypoints/openai_server.py b/atom/entrypoints/openai_server.py index 3dac7be376..a56fbe0170 100644 --- a/atom/entrypoints/openai_server.py +++ b/atom/entrypoints/openai_server.py @@ -1,274 +1,249 @@ import argparse import asyncio -import json import time +from typing import List, Optional, Dict, Any, Union import uuid -from datetime import datetime -from typing import List, Optional, Dict, Any -import torch -import uvicorn from fastapi import FastAPI, HTTPException -from fastapi.responses import StreamingResponse, JSONResponse from pydantic import BaseModel +import uvicorn from transformers import AutoTokenizer -from atom import LLMEngine, SamplingParams -from atom.config import CompilationConfig - - -# Request models -class CompletionRequest(BaseModel): - model: str = "atom" - prompt: str | List[str] - max_tokens: Optional[int] = 256 - temperature: Optional[float] = 0.7 - top_p: Optional[float] = 1.0 - n: Optional[int] = 1 - stream: Optional[bool] = False - stop: Optional[str | List[str]] = None - presence_penalty: Optional[float] = 0.0 - frequency_penalty: Optional[float] = 0.0 +from atom import SamplingParams +from atom.model_engine.arg_utils import EngineArgs +# OpenAI API Models class ChatMessage(BaseModel): role: str content: str class ChatCompletionRequest(BaseModel): - model: str = "atom" + model: str messages: List[ChatMessage] + temperature: Optional[float] = 1.0 + top_p: Optional[float] = 1.0 max_tokens: Optional[int] = 256 - temperature: Optional[float] = 0.7 + stop: Optional[List[str]] = None + ignore_eos: Optional[bool] = False + + +class CompletionRequest(BaseModel): + model: str + prompt: Union[str, List[str]] # Support both single string and list of strings + temperature: Optional[float] = 1.0 top_p: Optional[float] = 1.0 - n: Optional[int] = 1 - stream: Optional[bool] = False - stop: Optional[str | List[str]] = None - presence_penalty: Optional[float] = 0.0 - frequency_penalty: Optional[float] = 0.0 + max_tokens: Optional[int] = 256 + stop: Optional[List[str]] = None + ignore_eos: Optional[bool] = False -# Global engine and tokenizer -llm_engine: Optional[LLMEngine] = None -tokenizer: Optional[Any] = None -# Lock to serialize generation calls -generation_lock: Optional[asyncio.Lock] = None -# Log file path -log_file_path: Optional[str] = None +class ChatCompletionResponse(BaseModel): + id: str + object: str = "chat.completion" + created: int + model: str + choices: List[Dict[str, Any]] + usage: Dict[str, int] -def parse_size_list(size_str: str) -> List[int]: - import ast - try: - return ast.literal_eval(size_str) - except (ValueError, SyntaxError) as e: - raise ValueError(f"Error list size: {size_str}") from e +class CompletionResponse(BaseModel): + id: str + object: str = "text_completion" + created: int + model: str + choices: List[Dict[str, Any]] + usage: Dict[str, int] -def log_request_response(request_id: str, prompts: List[str], outputs: List[Dict], endpoint: str): - """Log request inputs and outputs to JSONL file""" - if log_file_path is None: - return - - log_entry = { - "timestamp": datetime.now().isoformat(), - "request_id": request_id, - "endpoint": endpoint, - "prompts": prompts, - "outputs": [{"text": out["text"]} for out in outputs] - } - - try: - with open(log_file_path, "a") as f: - f.write(json.dumps(log_entry) + "\n") - except Exception as e: - print(f"Error writing to log file: {e}") +# Global engine +engine = None +tokenizer: Optional[AutoTokenizer] = None +model_name: str = "" -def init_engine(args): - global llm_engine, tokenizer, generation_lock, log_file_path +async def generate_async(prompt: str, sampling_params: SamplingParams) -> Dict[str, Any]: + """Async wrapper for engine generation.""" + global engine - tokenizer = AutoTokenizer.from_pretrained(args.model) - # LLMEngine needs a separate port for distributed communication - engine_port = args.port + 1000 # e.g., if API port is 8000, engine uses 9000 - llm_engine = LLMEngine( - args.model, - enforce_eager=args.enforce_eager, - tensor_parallel_size=args.tensor_parallel_size, - kv_cache_dtype=args.kv_cache_dtype, - port=engine_port, - torch_profiler_dir=args.torch_profiler_dir, - compilation_config=CompilationConfig( - level=args.level, - cudagraph_capture_sizes=parse_size_list(args.cudagraph_capture_sizes) - ) - ) - generation_lock = asyncio.Lock() - - # Set log file path if enabled - if args.log_requests: - log_file_path = args.log_file - print(f"Request/Response logging enabled: {log_file_path}") + # Use the new async generate method that handles each request independently + async for output in engine.generate_async(prompt, sampling_params): + return output -app = FastAPI(title="Atom OpenAI-Compatible API") +app = FastAPI(title="Atom OpenAI API Server") -@app.post("/v1/completions") -async def create_completion(request: CompletionRequest): - if llm_engine is None or generation_lock is None: - raise HTTPException(status_code=503, detail="Engine not initialized") - - request_id = f"cmpl-{uuid.uuid4().hex[:8]}" - prompts = [request.prompt] if isinstance(request.prompt, str) else request.prompt - - sampling_params = SamplingParams( - temperature=request.temperature, - max_tokens=request.max_tokens, - stop_strings=request.stop, - ) - - # Use lock to serialize generation calls and run in thread pool to avoid blocking - async with generation_lock: - loop = asyncio.get_event_loop() - outputs = await loop.run_in_executor( - None, - llm_engine.generate, - prompts, - sampling_params - ) - - # Log request and response - log_request_response(request_id, prompts, outputs, "/v1/completions") - - choices = [] - for i, output in enumerate(outputs): - choices.append({ - "text": output["text"], - "index": i, - "logprobs": None, - "finish_reason": "stop" - }) - - return JSONResponse({ - "id": request_id, - "object": "text_completion", - "created": int(time.time()), - "model": request.model, - "choices": choices, - "usage": { - "prompt_tokens": sum(len(tokenizer.encode(p)) for p in prompts), - "completion_tokens": sum(len(tokenizer.encode(c["text"])) for c in choices), - "total_tokens": sum(len(tokenizer.encode(p)) for p in prompts) + sum(len(tokenizer.encode(c["text"])) for c in choices) - } - }) - - -@app.post("/v1/chat/completions") -async def create_chat_completion(request: ChatCompletionRequest): - if llm_engine is None or tokenizer is None or generation_lock is None: - raise HTTPException(status_code=503, detail="Engine not initialized") - - request_id = f"chatcmpl-{uuid.uuid4().hex[:8]}" - - # Convert chat messages to prompt - messages = [{"role": msg.role, "content": msg.content} for msg in request.messages] +@app.post("/v1/chat/completions", response_model=ChatCompletionResponse) +async def chat_completions(request: ChatCompletionRequest): + """Handle chat completion requests.""" + global engine, tokenizer, model_name try: + # Convert messages to prompt using chat template prompt = tokenizer.apply_chat_template( - messages, + [{"role": msg.role, "content": msg.content} for msg in request.messages], tokenize=False, add_generation_prompt=True, ) + + # Create sampling params + sampling_params = SamplingParams( + temperature=request.temperature, + max_tokens=request.max_tokens, + stop_strings=request.stop, + ignore_eos=request.ignore_eos, + ) + + # Generate response + output = await generate_async(prompt, sampling_params) + + # Format response + response = ChatCompletionResponse( + id=f"chatcmpl-{uuid.uuid4().hex}", + created=int(time.time()), + model=request.model, + choices=[ + { + "index": 0, + "message": { + "role": "assistant", + "content": output["text"], + }, + "finish_reason": output["finish_reason"], + } + ], + usage={ + "prompt_tokens": output["num_tokens_input"], + "completion_tokens": output["num_tokens_output"], + "total_tokens": output["num_tokens_input"] + output["num_tokens_output"], + }, + ) + + return response + except Exception as e: - # Fallback: simple concatenation - prompt = "\n".join([f"{msg.role}: {msg.content}" for msg in request.messages]) - - sampling_params = SamplingParams( - temperature=request.temperature, - max_tokens=request.max_tokens, - stop_strings=request.stop, - ) + raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/v1/completions", response_model=CompletionResponse) +async def completions(request: CompletionRequest): + """Handle text completion requests.""" + global engine, model_name - # Use lock to serialize generation calls and run in thread pool to avoid blocking - async with generation_lock: - loop = asyncio.get_event_loop() - outputs = await loop.run_in_executor( - None, - llm_engine.generate, - [prompt], - sampling_params + try: + # Create sampling params + sampling_params = SamplingParams( + temperature=request.temperature, + max_tokens=request.max_tokens, + stop_strings=request.stop, + ignore_eos=request.ignore_eos, ) - - # Log request and response - log_request_response(request_id, [prompt], outputs, "/v1/chat/completions") - - completion_text = outputs[0]["text"] if outputs else "" - - return JSONResponse({ - "id": request_id, - "object": "chat.completion", - "created": int(time.time()), - "model": request.model, - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": completion_text, + + # Handle both single prompt and batch prompts + if isinstance(request.prompt, str): + # Single prompt + output = await generate_async(request.prompt, sampling_params) + choices = [ + { + "index": 0, + "text": output["text"], + "finish_reason": output["finish_reason"], + } + ] + total_prompt_tokens = output["num_tokens_input"] + total_completion_tokens = output["num_tokens_output"] + else: + # Batch prompts - process concurrently + tasks = [ + generate_async(prompt, sampling_params) + for prompt in request.prompt + ] + outputs = await asyncio.gather(*tasks) + + choices = [ + { + "index": i, + "text": output["text"], + "finish_reason": output["finish_reason"], + } + for i, output in enumerate(outputs) + ] + total_prompt_tokens = sum(output["num_tokens_input"] for output in outputs) + total_completion_tokens = sum(output["num_tokens_output"] for output in outputs) + + # Format response + response = CompletionResponse( + id=f"cmpl-{uuid.uuid4().hex}", + created=int(time.time()), + model=request.model, + choices=choices, + usage={ + "prompt_tokens": total_prompt_tokens, + "completion_tokens": total_completion_tokens, + "total_tokens": total_prompt_tokens + total_completion_tokens, }, - "finish_reason": "stop" - }], - "usage": { - "prompt_tokens": len(tokenizer.encode(prompt)), - "completion_tokens": len(tokenizer.encode(completion_text)), - "total_tokens": len(tokenizer.encode(prompt)) + len(tokenizer.encode(completion_text)) - } - }) + ) + + return response + + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) @app.get("/v1/models") async def list_models(): - return JSONResponse({ + """List available models.""" + global model_name + return { "object": "list", - "data": [{ - "id": "atom", - "object": "model", - "created": int(time.time()), - "owned_by": "atom" - }] - }) + "data": [ + { + "id": model_name, + "object": "model", + "created": int(time.time()), + "owned_by": "atom", + } + ], + } @app.get("/health") async def health(): - return {"status": "healthy", "engine_ready": llm_engine is not None} + """Health check endpoint.""" + return {"status": "ok"} def main(): - parser = argparse.ArgumentParser(description="Atom OpenAI-Compatible Serving") + global engine, tokenizer, model_name + + parser = argparse.ArgumentParser(description="Atom OpenAI API Server") + + # Add engine arguments + EngineArgs.add_cli_args(parser) - parser.add_argument("--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path") + # Add server-specific arguments parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host") - parser.add_argument("--port", type=int, default=8000, help="API server port (engine will use port+1000)") - parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1) - parser.add_argument("--kv_cache_dtype", choices=["bf16", "fp8"], type=str, default="bf16") - parser.add_argument("--enforce-eager", action="store_true", help="Enforce eager mode") - parser.add_argument("--enable_prefix_caching", action="store_true", help="Enable prefix caching") - parser.add_argument("--cudagraph-capture-sizes", type=str, default="[1,2,4,8,16]") - parser.add_argument("--level", type=int, default=3, help="Compilation level") - parser.add_argument("--torch-profiler-dir", type=str, default=None) - parser.add_argument("--log-requests", action="store_true", help="Enable request/response logging") - parser.add_argument("--log-file", type=str, default="serving_requests.jsonl", help="Path to log file for requests/responses") + parser.add_argument( + "--server-port", type=int, default=8000, + help="Server port (note: --port is used for internal engine communication)" + ) args = parser.parse_args() - print(f"Initializing engine with model: {args.model}") - init_engine(args) - print("Engine initialized successfully") + # Initialize tokenizer + print(f"Loading tokenizer from {args.model}...") + tokenizer = AutoTokenizer.from_pretrained(args.model) + model_name = args.model + + # Initialize engine + print(f"Initializing engine with model {args.model}...") + engine_args = EngineArgs.from_cli_args(args) + engine = engine_args.create_engine() - print(f"Starting server at http://{args.host}:{args.port}") - uvicorn.run(app, host=args.host, port=args.port, log_level="info") + print(f"Starting server on {args.host}:{args.server_port}...") + uvicorn.run(app, host=args.host, port=args.server_port) if __name__ == "__main__": diff --git a/atom/model_engine/arg_utils.py b/atom/model_engine/arg_utils.py new file mode 100644 index 0000000000..0c065a502d --- /dev/null +++ b/atom/model_engine/arg_utils.py @@ -0,0 +1,170 @@ +import argparse +from typing import List + +from atom import LLMEngine +from atom.config import CompilationConfig + + +def parse_size_list(size_str: str) -> List[int]: + """Parse a string representation of a list into a Python list.""" + import ast + + try: + return ast.literal_eval(size_str) + except (ValueError, SyntaxError) as e: + raise ValueError(f"Error parsing size list: {size_str}") from e + + +class EngineArgs: + """Arguments for configuring the LLM Engine.""" + + def __init__( + self, + model: str = "Qwen/Qwen3-0.6B", + tensor_parallel_size: int = 1, + enforce_eager: bool = False, + enable_prefix_caching: bool = False, + port: int = 8006, + kv_cache_dtype: str = "bf16", + block_size: int = 16, + max_model_len: int = 8192, + cudagraph_capture_sizes: str = "[1,2,4,8,16]", + level: int = 3, + load_dummy: bool = False, + enable_expert_parallel: bool = False, + torch_profiler_dir: str = None, + ): + self.model = model + self.tensor_parallel_size = tensor_parallel_size + self.enforce_eager = enforce_eager + self.enable_prefix_caching = enable_prefix_caching + self.port = port + self.kv_cache_dtype = kv_cache_dtype + self.block_size = block_size + self.max_model_len = max_model_len + self.cudagraph_capture_sizes = cudagraph_capture_sizes + self.level = level + self.load_dummy = load_dummy + self.enable_expert_parallel = enable_expert_parallel + self.torch_profiler_dir = torch_profiler_dir + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + """Add engine arguments to an argument parser.""" + # Model configuration + parser.add_argument( + "--model", type=str, default="Qwen/Qwen3-0.6B", help="Model name or path." + ) + + parser.add_argument( + "--tensor-parallel-size", + "-tp", + type=int, + default=1, + help="Tensor parallel size.", + ) + + parser.add_argument( + "--enforce-eager", + action="store_true", + help="Enforce eager mode execution.", + ) + + parser.add_argument( + "--enable_prefix_caching", + action="store_true", + help="Enable prefix caching.", + ) + + parser.add_argument("--port", type=int, default=8006, help="API server port") + + # KV cache configuration + parser.add_argument( + "--kv_cache_dtype", + choices=["bf16", "fp8"], + type=str, + default="bf16", + help="KV cache type. Default is 'bf16'.", + ) + + parser.add_argument( + "--block-size", type=int, default=16, help="KV cache block size." + ) + + parser.add_argument( + "--max-model-len", + type=int, + default=8192, + help="Maximum model context length.", + ) + + # Compilation configuration + parser.add_argument( + "--cudagraph-capture-sizes", + type=str, + default="[1,2,4,8,16]", + help="Sizes to capture cudagraph.", + ) + + parser.add_argument( + "--level", type=int, default=3, help="The level of compilation (0-3)." + ) + + # Advanced options + parser.add_argument( + "--load_dummy", action="store_true", help="Skip loading model weights." + ) + + parser.add_argument( + "--enable-expert-parallel", + action="store_true", + help="Enable expert parallel(EP MoE).", + ) + + parser.add_argument( + "--torch-profiler-dir", + type=str, + default=None, + help="Directory to save torch profiler traces", + ) + + return parser + + @classmethod + def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs": + """Create an EngineArgs instance from parsed command-line arguments.""" + return cls( + model=args.model, + tensor_parallel_size=args.tensor_parallel_size, + enforce_eager=args.enforce_eager, + enable_prefix_caching=args.enable_prefix_caching, + port=args.port, + kv_cache_dtype=args.kv_cache_dtype, + block_size=args.block_size, + max_model_len=getattr(args, "max_model_len", 8192), + cudagraph_capture_sizes=args.cudagraph_capture_sizes, + level=args.level, + load_dummy=args.load_dummy, + enable_expert_parallel=args.enable_expert_parallel, + torch_profiler_dir=args.torch_profiler_dir, + ) + + def create_engine(self) -> LLMEngine: + """Create and return an LLMEngine instance with the configured parameters.""" + return LLMEngine( + self.model, + enforce_eager=self.enforce_eager, + tensor_parallel_size=self.tensor_parallel_size, + kv_cache_dtype=self.kv_cache_dtype, + kv_cache_block_size=self.block_size, + max_model_len=self.max_model_len, + port=self.port, + load_dummy=self.load_dummy, + enable_expert_parallel=self.enable_expert_parallel, + torch_profiler_dir=self.torch_profiler_dir, + compilation_config=CompilationConfig( + level=self.level, + cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes), + ), + ) + diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 704e54c9ac..c2ff597e65 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -146,11 +146,12 @@ def _process_engine_step(self): def pull_and_process_input_queue(self): recv_reqs = [] while not self.input_queue.empty(): - seq = self.input_queue.get_nowait() - if seq.status == SequenceStatus.EXIT_ENGINE: - logger.debug(f"{self.label}: input_queue get {seq.status}") - return True - recv_reqs.append(seq) + seqs = self.input_queue.get_nowait() + for seq in seqs: + if seq.status == SequenceStatus.EXIT_ENGINE: + logger.debug(f"{self.label}: input_queue get exit engine") + return True + recv_reqs.append(seq) if len(recv_reqs) > 0: logger.info(f"{self.label}: put {len(recv_reqs)} reqs to scheduler") self.scheduler.extend(recv_reqs) @@ -175,13 +176,14 @@ def process_input_sockets(self, input_address: str): for input_socket, _ in poller.poll(): # (RequestType, RequestData) serialized_obj = input_socket.recv(copy=False) - request_type, req = pickle.loads(serialized_obj) + request_type, reqs = pickle.loads(serialized_obj) if request_type == EngineCoreRequestType.ADD: - logger.debug(f"{self.label}: input get {request_type} {req.id}") - self.input_queue.put_nowait(req) + req_ids = [req.id for req in reqs] + logger.debug(f"{self.label}: input get {request_type} {req_ids}") + self.input_queue.put_nowait(reqs) elif request_type == EngineCoreRequestType.SHUTDOWN: logger.debug(f"{self.label}: input get {request_type}") - self.input_queue.put_nowait(get_exit_sequence()) + self.input_queue.put_nowait([get_exit_sequence()]) alive = False reason = request_type logger.debug(f"{self.label}: input thread exit due to {reason}") diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 0f0b4b53e8..1cb7e35d72 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -102,10 +102,10 @@ def close(self): shutdown_sender.send(b"") self.output_queue_thread.join() - def add_request(self, seq: Sequence): - logger.debug(f"{self.label}: Add request, sequence id: {seq.id}") + def add_request(self, seqs: List[Sequence]): + logger.debug(f"{self.label}: Add request, sequence ids: {[seq.id for seq in seqs]}") self.input_socket.send_multipart( - [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.ADD, seq))], + [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.ADD, seqs))], copy=False, ) diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 42494caca2..bbc2d093cd 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -1,8 +1,9 @@ +import asyncio import itertools import logging import time from dataclasses import fields -from typing import List +from typing import List, Union import torch.multiprocessing as mp from tqdm.auto import tqdm @@ -31,16 +32,32 @@ def __init__(self, model, **kwargs): self.tokenizer, config.kv_cache_block_size ) self.core_mgr = CoreManager(config) + self._step_lock = None + self._pending_results = {} logger.info("LLMEngine init") def add_request( - self, prompt_or_tokens: str | list[int], sampling_params: SamplingParams + self, + prompt_or_tokens_list: List[Union[str, List[int]]], + sampling_params_list: SamplingParams | List[SamplingParams] ): - # 1. convet prompt to request - req = self.io_processor.preprocess(prompt_or_tokens, sampling_params) - - # 2. add req to scheduler - self.core_mgr.add_request(req) + # if sampling params is not list, use it for all prompts + if not isinstance(sampling_params_list, list): + sampling_params_iter = itertools.repeat(sampling_params_list) + else: + # otherwise check num elements first + if len(prompt_or_tokens_list) != len(sampling_params_list): + raise ValueError( + f"number of elements in prompt_or_tokens_list and sampling_params_list is different: " + f"{len(prompt_or_tokens_list)=} vs {len(sampling_params_list)=}" + ) + sampling_params_iter = sampling_params_list + + reqs = [] + for prompt, sampling_param in zip(prompt_or_tokens_list, sampling_params_iter): + req = self.io_processor.preprocess(prompt, sampling_param) + reqs.append(req) + self.core_mgr.add_request(reqs) def step(self) -> list[Sequence]: seqs = self.core_mgr.get_output() @@ -55,14 +72,7 @@ def generate( prompts: list[str], sampling_params: SamplingParams | list[SamplingParams], ) -> list[str]: - if isinstance(sampling_params, list): - iter_func = zip - else: - iter_func = itertools.product - sampling_params = [sampling_params] - for prompt, sp in iter_func(prompts, sampling_params): - self.add_request(prompt, sp) - + self.add_request(prompts, sampling_params) outputs = {} while not self.is_finished() and ( self.core_mgr.is_alive() or self.core_mgr.is_rest() @@ -74,6 +84,49 @@ def generate( outputs = [outputs[seq_id] for seq_id in sorted(outputs)] return outputs + async def generate_async( + self, + prompt: str, + sampling_params: SamplingParams, + ): + # Initialize lock on first use + if self._step_lock is None: + self._step_lock = asyncio.Lock() + + # Add the request and get its sequence ID + req = self.io_processor.preprocess(prompt, sampling_params) + seq_id = req.id + self.core_mgr.add_request([req]) + + while True: + # Check if result is already available + if seq_id in self._pending_results: + result = self._pending_results.pop(seq_id) + yield result + return + + if seq_id not in self.io_processor.requests: + break + + if not (self.core_mgr.is_alive() or self.core_mgr.is_rest()): + break + + # Coordinate step() calls across all concurrent requests + async with self._step_lock: + seqs = self.step() + outs = self.io_processor.postprocess(seqs) + + # Store all results so other waiting tasks can find them + self._pending_results.update(outs) + + if seq_id in self._pending_results: + result = self._pending_results.pop(seq_id) + yield result + return + + # Let other tasks run before trying again + await asyncio.sleep(0) + class InputOutputProcessor: From 5b583d0e85a586adfd9a241a2e35dcdcb81dff12 Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Thu, 30 Oct 2025 14:30:26 +0800 Subject: [PATCH 099/119] use mixed_sample_outer_exponential (#20) Co-authored-by: amd-ruitang3 --- atom/model_ops/sampler.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/atom/model_ops/sampler.py b/atom/model_ops/sampler.py index d8d21d2414..2b35e75261 100644 --- a/atom/model_ops/sampler.py +++ b/atom/model_ops/sampler.py @@ -1,5 +1,5 @@ import torch -from aiter import mixed_sample +from aiter import mixed_sample, mixed_sample_outer_exponential from aiter.ops.triton.softmax import softmax from aiter.ops.triton.topk import topk from torch import nn @@ -19,12 +19,19 @@ def forward( sampled_tokens = torch.empty( logits.size(0), dtype=torch.int, device=logits.device ) - mixed_sample(sampled_tokens, logits, temperatures, lambd=1.0, eps=self.eps) + exponential = ( + torch.empty((1, logits.shape[-1]), dtype=torch.float, device=logits.device) + .exponential_(1) + .expand(*logits.shape) + ) + mixed_sample_outer_exponential( + sampled_tokens, logits, exponential, temperatures, eps=self.eps + ) return sampled_tokens logits = logits.float() return torch.where( temperatures == 0, self.greedy_sample(logits), self.random_sample(logits) - ) + ).to(torch.int) def greedy_sample( self, logits: torch.Tensor # (token_num, vocab_size) From 47bc0b5e73056f92ec19ae59bcf2ecb91f9bf5f4 Mon Sep 17 00:00:00 2001 From: ZhangLirong Date: Thu, 30 Oct 2025 15:42:10 +0800 Subject: [PATCH 100/119] Fix weak ref error in level 3 in torch 2.6 and 2.7 (#23) * fix weak ref error in level 3 in torch 2.6 and 2.7 * refactor --- atom/model_ops/base_attention.py | 7 +------ atom/model_ops/layernorm.py | 29 ++++++++++++++++++++--------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py index 5eb8ac48da..ddf4076a6e 100644 --- a/atom/model_ops/base_attention.py +++ b/atom/model_ops/base_attention.py @@ -10,12 +10,7 @@ from atom.utils.context import get_context from atom.utils.custom_register import direct_register_custom_op -from atom.utils.forward_context import ( - AttentionMetadata, - ForwardContext, - get_forward_context, - set_forward_context, -) + from atom.utils import mark_spliting_op from .attention_mla import MLAModules from atom.config import get_current_atom_config diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py index 4dd1a22878..db06ed311e 100644 --- a/atom/model_ops/layernorm.py +++ b/atom/model_ops/layernorm.py @@ -1,6 +1,23 @@ +from typing import List, Tuple, Optional, Union import torch from torch import nn from aiter import rmsnorm2d_fwd, rmsnorm2d_fwd_with_add, rms_norm +from aiter.jit.utils.torch_guard import torch_compile_guard + +@torch_compile_guard() +def rmsnorm2d_fwd_(x: torch.Tensor, weight: torch.Tensor, eps: float, dim: int) -> torch.Tensor: + ori_shape = x.shape + x = x.reshape(-1, dim) + return rmsnorm2d_fwd(x, weight, eps).view(ori_shape) + +@torch_compile_guard() +def rmsnorm2d_fwd_with_add_(x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor, eps: float, dim: int) -> Tuple[torch.Tensor, torch.Tensor]: + ori_shape = x.shape + x = x.reshape(-1, dim) + out = torch.empty_like(x) + residual_out = torch.empty_like(x) + rmsnorm2d_fwd_with_add(out, x, residual, residual_out, weight, eps) + return out.view(ori_shape), residual_out.view(ori_shape) class RMSNorm(nn.Module): @@ -44,15 +61,9 @@ def forward( x: torch.Tensor, residual: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - ori_shape = x.shape - x = x.reshape(-1, self.dim) if residual is None: - return rmsnorm2d_fwd(x, self.weight, self.eps).view(ori_shape) + # return rmsnorm2d_fwd(x, self.weight, self.eps).view(ori_shape) + return rmsnorm2d_fwd_(x, self.weight, self.eps, self.dim) else: # return self.add_rms_forward(x, residual) - residual_out = torch.empty_like(x) - out = torch.empty_like(x) - rmsnorm2d_fwd_with_add( - out, x, residual, residual_out, self.weight, self.eps - ) - return out.view(ori_shape), residual_out.view(ori_shape) + return rmsnorm2d_fwd_with_add_(x, self.weight, residual, self.eps, self.dim) From cb802e28e49fa4ee5e77a927aa17d996e0024df8 Mon Sep 17 00:00:00 2001 From: ZhangLirong Date: Mon, 3 Nov 2025 15:24:51 +0800 Subject: [PATCH 101/119] Forward context support (#25) * draft on refactor context * enable cudagraph and reuse kv_cache creation * refactor and reuse one kv_cache global * fix core dump by adding context in embed_head * merge to one attn meta data in context * put bs,position so on to new context * make code cleaner --- atom/config.py | 18 +++ atom/model_engine/model_runner.py | 127 ++++++++++------- atom/model_ops/attention_mha.py | 46 +++---- atom/model_ops/attention_mla.py | 30 ++-- atom/model_ops/attentions/aiter_attention.py | 26 ++-- atom/model_ops/attentions/aiter_mla.py | 30 ++-- atom/model_ops/attentions/backends.py | 33 ++++- atom/model_ops/base_attention.py | 2 - atom/model_ops/embed_head.py | 10 +- atom/utils/context.py | 76 ---------- atom/utils/forward_context.py | 138 ++++++++++++++++--- 11 files changed, 326 insertions(+), 210 deletions(-) delete mode 100644 atom/utils/context.py diff --git a/atom/config.py b/atom/config.py index d13846885a..08f45facf9 100644 --- a/atom/config.py +++ b/atom/config.py @@ -10,6 +10,24 @@ from aiter.utility.dtypes import d_dtypes from transformers import AutoConfig, PretrainedConfig +@dataclass +class KVCacheTensor: + """ + A class for specifying how the workers should initialize the KV cache. + """ + layer_num: int + k_cache: torch.Tensor=torch.tensor([]) + v_cache: torch.Tensor=torch.tensor([]) + k_scale: torch.Tensor=None + v_scale: torch.Tensor=None + +@dataclass +class KVCacheConfig: + """ + The KV cache configuration of a model. + """ + kv_cache_tensors: list[KVCacheTensor] + class CUDAGraphMode(enum.Enum): """Constants for the cudagraph mode in CompilationConfig. diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 6c9b897252..f83d3e3da5 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -9,7 +9,7 @@ import tqdm from aiter import destroy_dist_env, dtypes, init_dist_env from aiter.dist.parallel_state import graph_capture -from atom.config import Config, set_current_atom_config +from atom.config import Config, set_current_atom_config, KVCacheTensor from atom.model_engine.scheduler import ScheduledBatch from atom.model_engine.sequence import Sequence from atom.model_loader.loader import load_model @@ -19,11 +19,10 @@ from atom.models.mixtral import MixtralForCausalLM from atom.models.qwen3 import Qwen3ForCausalLM from atom.utils import CpuGpuBuffer, init_exit_handler, get_hf_text_config -from atom.utils.context import get_context, reset_context, set_context from atom.utils.selector import get_attn_backend logger = logging.getLogger("atom") -from atom.utils.forward_context import AttentionMetadata, set_forward_context +from atom.utils.forward_context import AttentionMetaData, Context, get_forward_context, reset_forward_context, set_forward_context, set_kv_cache_data suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -377,6 +376,8 @@ def allocate_forward_vars(self): i64_kwargs = {"dtype": torch.int64, "device": self.device} i32_kwargs = {"dtype": torch.int32, "device": self.device} f32_kwargs = {"dtype": torch.float, "device": self.device} + + # TODO: remove it in forward_context self.forward_vars = { "input_ids": self.tokenID_processor.input_ids, "positions": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), @@ -440,6 +441,8 @@ def allocate_kv_cache(self, num_kvcache_blocks): hf_config = config.hf_config isMLA = hf_config.architectures[0].startswith("DeepseekV") num_kv_heads = hf_config.num_key_value_heads // self.world_size + + # Allocate raw KV cache tensors if isMLA: self.kv_cache = torch.zeros( hf_config.num_hidden_layers, @@ -470,21 +473,26 @@ def allocate_kv_cache(self, num_kvcache_blocks): dtype=dtypes.fp32, device="cuda", ) - + # Build KVCacheConfig + # lirong TODO: This is a simple solution to build KVCacheConfig, + # models with only one type of attention, but not support multi-type of attention models. + # We need to support it by kv_cache_group in the future. + kv_cache_tensors = [] layer_id = 0 x = 16 // self.kv_cache.element_size() for module in self.model.modules(): # Since use attention base and there are child in attention, add base condition if hasattr(module, "base_attention"): if hasattr(module, "use_mla") and not module.use_mla: - module.k_cache = self.kv_cache[0, layer_id].view( + # Non-MLA attention + k_cache = self.kv_cache[0, layer_id].view( config.num_kvcache_blocks, num_kv_heads, hf_config.head_dim // x, self.block_size, x, ) - module.v_cache = self.kv_cache[1, layer_id].view( + v_cache = self.kv_cache[1, layer_id].view( config.num_kvcache_blocks, num_kv_heads, hf_config.head_dim, @@ -494,30 +502,50 @@ def allocate_kv_cache(self, num_kvcache_blocks): if config.kv_cache_dtype == "fp8": module.k_scale = self.kv_scale[0, layer_id] module.v_scale = self.kv_scale[1, layer_id] - attention_metadata = AttentionMetadata( - k_cache=module.k_cache, - v_cache=module.v_cache, - k_scale=module.k_scale, - v_scale=module.v_scale, + + k_scale = module.k_scale + v_scale = module.v_scale + + # Store in KVCacheTensor + kv_cache_tensor = KVCacheTensor( + layer_num=layer_id, + k_cache=k_cache, + v_cache=v_cache, + k_scale=k_scale, + v_scale=v_scale, ) - set_forward_context(module.layer_num, attention_metadata) + kv_cache_tensors.append(kv_cache_tensor) + + module.k_cache = k_cache + module.v_cache = v_cache layer_id += 1 elif hasattr(module, "use_mla") and module.use_mla: - module.kv_cache = self.kv_cache[layer_id].view( + # MLA attention + kv_cache = self.kv_cache[layer_id].view( config.num_kvcache_blocks * self.block_size, 1, 576, ) - module.max_model_len = self.config.max_model_len - attention_metadata = AttentionMetadata( - k_cache=module.kv_cache, + + # Store in KVCacheTensor + kv_cache_tensor = KVCacheTensor( + layer_num=layer_id, + k_cache=kv_cache, v_cache=None, k_scale=None, v_scale=None, ) - set_forward_context(module.layer_num, attention_metadata) + kv_cache_tensors.append(kv_cache_tensor) + + module.kv_cache = kv_cache + module.max_model_len = self.config.max_model_len layer_id += 1 + + # Store KVCacheConfig + kv_cache_data = {f"layer_{i}": kv_cache_tensor for i, kv_cache_tensor in enumerate(kv_cache_tensors)} + # vllm use register_kv_caches to register kv_cache_data. We just set it to global here + set_kv_cache_data(kv_cache_data) if torch.distributed.is_initialized(): torch.distributed.barrier() return True @@ -546,7 +574,25 @@ def prepare_decode(self, batch: ScheduledBatch): return self.attn_metadata_builder.prepare_decode(batch, bs, self.forward_vars) + def prepare_intputs(self, batch: ScheduledBatch): + is_prefill = batch.total_tokens_num_prefill > 0 + bs = 0 + if not is_prefill: + scheduled_bs = batch.total_seqs_num_decode + seqs = list(batch.seqs.values()) + seqs = seqs[batch.total_seqs_num_prefill :] + assert len(seqs) == scheduled_bs + bs = ( + scheduled_bs + if self.enforce_eager + else next(x for x in self.graph_bs if x >= scheduled_bs) + ) + assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" + attn_metadata, positions = self.attn_metadata_builder.build(batch, self.forward_vars, bs) + context_bs = batch.total_seqs_num_prefill if is_prefill else batch.total_seqs_num_decode + context = Context(positions=positions, is_prefill=is_prefill, batch_size=context_bs, graph_bs=bs) + set_forward_context(attn_metadata=attn_metadata, atom_config=self.config, context=context) def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: temperatures = [seq.temperature for seq in batch.seqs.values()] @@ -566,26 +612,25 @@ def prepare_model(self, batch: ScheduledBatch): self.forward_vars["cu_seqlens_q"].np[1 : bs + 1] = cu_seqlens_q input_ids = self.tokenID_processor.prepare_input_ids(batch) - # print(f"{input_ids=}") + # print(f"input_ids: {input_ids}") - is_prefill = batch.total_tokens_num_prefill > 0 - prepare_func = self.prepare_prefill if is_prefill else self.prepare_decode - positions = prepare_func(batch) + self.prepare_intputs(batch) temperatures = self.prepare_sample(batch) return ( input_ids, - positions, temperatures, ) - def run_model(self, input_ids: torch.Tensor, positions: torch.Tensor): - ctx = get_context() - bs = ctx.batch_size - is_prefill = ctx.is_prefill + def run_model(self, input_ids: torch.Tensor): + forward_context = get_forward_context() + context = forward_context.context + bs = context.batch_size + is_prefill = context.is_prefill + positions = context.positions if is_prefill or self.enforce_eager or bs > self.graph_bs[-1]: hidden_states = self.model(input_ids, positions) else: - graph_bs = ctx.graph_bs + graph_bs = context.graph_bs self.graphs[graph_bs].replay() hidden_states = self.forward_vars["outputs"][:bs] return self.model.compute_logits(hidden_states) @@ -607,9 +652,9 @@ def postprocess( @torch.inference_mode() def forward(self, batch: ScheduledBatch) -> dict[int, int]: - input_ids, positions, temperatures = self.prepare_model(batch) - logits = self.run_model(input_ids, positions) - reset_context() + input_ids, temperatures = self.prepare_model(batch) + logits = self.run_model(input_ids) + reset_forward_context() return self.postprocess(batch, logits, temperatures) @torch.inference_mode() @@ -634,13 +679,6 @@ def capture_cudagraph(self): self.forward_vars["cu_seqlens_q"].copy_to_gpu(self.graph_bs[0] + 1) input_ids = self.forward_vars["input_ids"].gpu positions = self.forward_vars["positions"].gpu - slot_mapping = self.forward_vars["slot_mapping"].gpu - context_lens = self.forward_vars["context_lens"].gpu - block_tables = self.forward_vars["block_tables"].gpu - cu_seqlens_q = self.forward_vars["cu_seqlens_q"].gpu - kv_indptr = self.forward_vars["kv_indptr"].gpu - kv_indices = self.forward_vars["kv_indices"].gpu - kv_last_page_lens = self.forward_vars["kv_last_page_lens"].gpu outputs = self.forward_vars["outputs"] self.graphs: dict[int, torch.cuda.CUDAGraph] = dict() @@ -655,18 +693,8 @@ def capture_cudagraph(self): capture_range.set_description(f"Capturing {bs=}") graph = torch.cuda.CUDAGraph() - set_context( - False, - batch_size=bs, - slot_mapping=slot_mapping[:bs], - context_lens=context_lens[:bs], - block_tables=block_tables[:bs], - max_q_len=1, - cu_seqlens_q=cu_seqlens_q[: bs + 1], - kv_indptr=kv_indptr[: bs + 1], - kv_indices=kv_indices[:], - kv_last_page_lens=kv_last_page_lens[:bs], - ) + attn_metadata, context = self.attn_metadata_builder.build_for_cudagraph_capture(forward_vars=self.forward_vars, bs=bs) + set_forward_context(attn_metadata=attn_metadata, atom_config=self.config, context=context) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup @@ -676,6 +704,5 @@ def capture_cudagraph(self): self.graph_pool = graph.pool() self.graphs[bs] = graph torch.cuda.synchronize() - reset_context() self.graph_bs.sort(reverse=False) return time.time() - start_time, self.graph_bs diff --git a/atom/model_ops/attention_mha.py b/atom/model_ops/attention_mha.py index 71ab42596e..4331891b4c 100644 --- a/atom/model_ops/attention_mha.py +++ b/atom/model_ops/attention_mha.py @@ -8,15 +8,10 @@ from aiter.paged_attn import PagedAttention from torch import nn -from atom.utils.context import get_context -from atom.utils.custom_register import direct_register_custom_op from atom.utils.forward_context import ( - AttentionMetadata, ForwardContext, get_forward_context, - set_forward_context, ) -from atom.utils import mark_spliting_op from .attention_mla import MLAModules @@ -53,15 +48,17 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, position: t # o = torch.ops.aiter.unified_attention_with_output(q, k, v, # self.scale, self.kv_cache_dtype, self.layer_num) - context = get_context() - if context.slot_mapping.numel(): + forward_context: ForwardContext = get_forward_context() + attn_metadata = forward_context.attn_metadata + context = forward_context.context + + kv_cache_data = forward_context.kv_cache_data + if attn_metadata.slot_mapping.numel(): # not dummy run - forward_context: ForwardContext = get_forward_context() - attn_metadata_ = forward_context.no_compile_layers[self.layer_num] - k_cache = attn_metadata_.k_cache - v_cache = attn_metadata_.v_cache - k_scale = attn_metadata_.k_scale - v_scale = attn_metadata_.v_scale + k_cache = kv_cache_data[f"layer_{self.layer_num}"].k_cache + v_cache = kv_cache_data[f"layer_{self.layer_num}"].v_cache + k_scale = kv_cache_data[f"layer_{self.layer_num}"].k_scale + v_scale = kv_cache_data[f"layer_{self.layer_num}"].v_scale else: # dummy run before allocate kv_cache, thus we create manually k_cache = v_cache = torch.tensor([]) @@ -76,7 +73,7 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, position: t v_cache, k_scale, v_scale, - context.slot_mapping, + attn_metadata.slot_mapping, asm_layout=True, ) else: @@ -85,7 +82,7 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, position: t v, k_cache, v_cache, - context.slot_mapping, + attn_metadata.slot_mapping, kv_cache_dtype="auto", k_scale=None, v_scale=None, @@ -100,12 +97,12 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, position: t q, k, v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q=context.min_seqlen_q, - dropout_p=context.dropout_p, + cu_seqlens_q=attn_metadata.cu_seqlens_q, + cu_seqlens_k=attn_metadata.cu_seqlens_k, + max_seqlen_q=attn_metadata.max_seqlen_q, + max_seqlen_k=attn_metadata.max_seqlen_k, + min_seqlen_q=attn_metadata.min_seqlen_q, + dropout_p=attn_metadata.dropout_p, softmax_scale=self.scale, causal=True, ) @@ -114,15 +111,14 @@ def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, position: t q, k_cache, v_cache, - context.block_tables, - context.context_lens, - context.block_tables.stride(0), + attn_metadata.block_tables, + attn_metadata.context_lens, + attn_metadata.block_tables.stride(0), K_QScale=k_scale, V_QScale=v_scale, out_=None, high_precision=0, ) - o = o.view(-1, self.num_heads * self.head_dim) return o diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index 7c206fa8c4..5ff8fdacc4 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -1,9 +1,8 @@ from dataclasses import dataclass import torch from torch import nn -from atom.utils.context import get_context, Context from atom.utils.custom_register import direct_register_custom_op -from atom.utils.forward_context import ForwardContext, get_forward_context +from atom.utils.forward_context import AttentionMetaData, ForwardContext, get_forward_context from atom.model_ops.linear import ColumnParallelLinear, RowParallelLinear from atom.model_ops.utils import get_and_maybe_dequant_weights from aiter.rotary_embedding import RotaryEmbedding @@ -220,7 +219,7 @@ def _forward_prefill( kv_c_normed: torch.Tensor, k_pe: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, - context: Context, + context: AttentionMetaData, ) -> torch.Tensor: assert context is not None @@ -254,7 +253,7 @@ def _forward_decode( q_nope: torch.Tensor, q_pe: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, - context: Context, + context: AttentionMetaData, ) -> torch.Tensor: assert kv_c_and_k_pe_cache.numel() > 0 assert context is not None @@ -289,11 +288,16 @@ def forward( positions: torch.Tensor, ) -> torch.Tensor: # kv_cache = self.kv_cache - context = get_context() - if context.slot_mapping.numel(): - forward_context: ForwardContext = get_forward_context() - attn_metadata_ = forward_context.no_compile_layers[self.layer_num] - kv_cache = attn_metadata_.k_cache + forward_context: ForwardContext = get_forward_context() + attn_metadata = forward_context.attn_metadata + context = forward_context.context + if attn_metadata.slot_mapping.numel(): + # not dummy run + kv_cache_data = forward_context.kv_cache_data + if f"layer_{self.layer_num}" in kv_cache_data: + kv_cache = kv_cache_data[f"layer_{self.layer_num}"].k_cache + else: + kv_cache = torch.tensor([]) else: # dummy run before allocate kv_cache, thus we create manually kv_cache = torch.tensor([]) @@ -308,13 +312,13 @@ def forward( k_c_normed, k_pe.squeeze(1), kv_cache, - context.slot_mapping.flatten(), + attn_metadata.slot_mapping.flatten(), kv_cache_dtype=self.kv_cache_dtype, scale=self._k_scale, ) output = self._forward_prefill( - prefill_q, k_c_normed, k_pe, kv_cache, context + prefill_q, k_c_normed, k_pe, kv_cache, attn_metadata ) else: decode_ql_nope, decode_q_pe = self._q_proj_and_k_up_proj(q) @@ -325,13 +329,13 @@ def forward( k_c_normed, k_pe.squeeze(1), kv_cache, - context.slot_mapping.flatten(), + attn_metadata.slot_mapping.flatten(), kv_cache_dtype=self.kv_cache_dtype, scale=self._k_scale, ) output = self._forward_decode( - decode_ql_nope, decode_q_pe, kv_cache, context + decode_ql_nope, decode_q_pe, kv_cache, attn_metadata ) return output diff --git a/atom/model_ops/attentions/aiter_attention.py b/atom/model_ops/attentions/aiter_attention.py index ffbd291099..3e46a7943b 100644 --- a/atom/model_ops/attentions/aiter_attention.py +++ b/atom/model_ops/attentions/aiter_attention.py @@ -1,11 +1,12 @@ from dataclasses import dataclass from typing import Type, Optional + +from atom.utils.forward_context import AttentionMetaData, Context from .backends import CommonAttentionBuilder, AttentionBackend import torch import numpy as np from atom.model_engine.scheduler import ScheduledBatch -from atom.utils.context import set_context from atom.model_ops.attention_mha import Attention @@ -26,7 +27,7 @@ class AiterAttentionMetadataBuilder(CommonAttentionBuilder): BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] def __init__(self, block_size: int): - self.block_size = block_size + super().__init__(block_size) # Call parent __init__ to initialize _cached_kv_cache_data def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): scheduled_bs = batch.total_seqs_num_decode @@ -60,14 +61,23 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): ] ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} - set_context( - False, - batch_size=scheduled_bs, - graph_bs=bs, + attn_metadata = AttentionMetaData( dropout_p=dropout_p, max_q_len=max_q_len, **ctx, ) - positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) - return positions + positions=var["positions"].copy_to_gpu(sum_scheduled_tokens) + return attn_metadata, positions + + def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: + attn_matadata = AttentionMetaData( + slot_mapping=forward_vars["slot_mapping"].gpu[:bs], + context_lens=forward_vars["context_lens"].gpu[:bs], + block_tables=forward_vars["block_tables"].gpu[:bs], + max_q_len=1, + cu_seqlens_q=forward_vars["cu_seqlens_q"].gpu[: bs + 1], + ) + positions=forward_vars["positions"].copy_to_gpu(bs) + context = Context(positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs) + return attn_matadata, context diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index c54db5183b..15c1709fec 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -1,11 +1,13 @@ from dataclasses import dataclass from typing import Type, Optional + +from atom.config import KVCacheConfig, KVCacheTensor +from atom.utils.forward_context import AttentionMetaData, Context from .backends import CommonAttentionBuilder, AttentionBackend import torch import numpy as np from atom.model_engine.scheduler import ScheduledBatch -from atom.utils.context import set_context from atom.model_ops.attention_mla import MLAAttention @@ -27,7 +29,7 @@ class AiterMLAMetadataBuilder(CommonAttentionBuilder): BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] def __init__(self, block_size: int): - self.block_size = block_size + super().__init__(block_size) # Call parent __init__ to initialize _cached_kv_cache_data assert self.block_size == 1, "AITER MLA requires only block size 1." def prepare(self, batch: ScheduledBatch): @@ -116,13 +118,25 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): ("kv_last_page_lens", bs), ] ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} - set_context( - False, - batch_size=scheduled_bs, - graph_bs=bs, + attn_metadata = AttentionMetaData( dropout_p=dropout_p, max_q_len=max_q_len, **ctx, ) - positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) - return positions + positions=var["positions"].copy_to_gpu(sum_scheduled_tokens) + return attn_metadata, positions + + def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: + attn_matadata = AttentionMetaData( + slot_mapping=forward_vars["slot_mapping"].gpu[:bs], + context_lens=forward_vars["context_lens"].gpu[:bs], + block_tables=forward_vars["block_tables"].gpu[:bs], + max_q_len=1, + cu_seqlens_q=forward_vars["cu_seqlens_q"].gpu[: bs + 1], + kv_indptr=forward_vars["kv_indptr"].gpu[: bs + 1], + kv_indices=forward_vars["kv_indices"].gpu[:], + kv_last_page_lens=forward_vars["kv_last_page_lens"].gpu[:bs], + ) + positions=forward_vars["positions"].copy_to_gpu(bs) + context = Context(positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs) + return attn_matadata, context diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py index f11e3c92e3..b0f558ce60 100644 --- a/atom/model_ops/attentions/backends.py +++ b/atom/model_ops/attentions/backends.py @@ -1,9 +1,9 @@ from abc import ABC, abstractmethod from typing import Type, Tuple, Generic, Optional, List, TypeVar, Dict, Any from atom.model_engine.scheduler import ScheduledBatch -from atom.utils.context import set_context from atom.model_ops.attention_mla import MLAModules +from atom.utils.forward_context import AttentionMetaData import torch from torch import nn @@ -72,6 +72,17 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): def prepare_prefill(self, batch: ScheduledBatch, forward_vars): raise NotImplementedError + @abstractmethod + def build(self, + batch: ScheduledBatch, + forward_vars, + bs: int): + raise NotImplementedError + + @abstractmethod + def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: + raise NotImplementedError + class CommonAttentionBuilder(AttentionMetadataBuilder[T], Generic[T]): def __init__(self, block_size: int): @@ -119,19 +130,27 @@ def prepare_prefill(self, batch: ScheduledBatch, forward_vars): ("slot_mapping", len(slot_mapping)), ] ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} - set_context( - True, - batch_size=bs, + attn_metadata = AttentionMetaData( cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), - # slot_mapping=slot_mapping.cuda(non_blocking=True), max_seqlen_q=max_seqlen_q, max_seqlen_k=max_seqlen_k, min_seqlen_q=min_seqlen_q, dropout_p=dropout_p, **ctx, ) - return var["positions"].copy_to_gpu(sum_scheduled_tokens) - + positions=var["positions"].copy_to_gpu(sum_scheduled_tokens) + + return attn_metadata, positions + # return var["positions"].copy_to_gpu(sum_scheduled_tokens) + + def build(self, + batch: ScheduledBatch, + forward_vars, + bs: int): + if batch.total_tokens_num_prefill > 0: + return self.prepare_prefill(batch, forward_vars) + else: + return self.prepare_decode(batch, bs, forward_vars) class AttentionImpl(nn.Module): diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py index ddf4076a6e..55e0000315 100644 --- a/atom/model_ops/base_attention.py +++ b/atom/model_ops/base_attention.py @@ -8,8 +8,6 @@ from aiter.paged_attn import PagedAttention from torch import nn -from atom.utils.context import get_context -from atom.utils.custom_register import direct_register_custom_op from atom.utils import mark_spliting_op from .attention_mla import MLAModules diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 7b8534a63f..09020cc225 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -7,8 +7,7 @@ from aiter.tuned_gemm import tgemm from torch import nn -from atom.utils.context import get_context - +from atom.utils.forward_context import ForwardContext, get_forward_context class VocabParallelEmbedding(nn.Module): @@ -67,9 +66,12 @@ def __init__( self.register_parameter("bias", None) def forward(self, x: torch.Tensor): - context = get_context() + forward_context: ForwardContext = get_forward_context() + context = forward_context.context + attn_metadata = forward_context.attn_metadata + # context = get_context() if context.is_prefill: - last_indices = context.cu_seqlens_q[1:] - 1 + last_indices = attn_metadata.cu_seqlens_q[1:] - 1 x = x[last_indices].contiguous() logits = tgemm.mm(x, self.weight, self.bias) if self.tp_size > 1: diff --git a/atom/utils/context.py b/atom/utils/context.py deleted file mode 100644 index 2038ebcdaf..0000000000 --- a/atom/utils/context.py +++ /dev/null @@ -1,76 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - -import torch - - -@dataclass -class Context: - is_prefill: bool = False - batch_size: int = 0 - graph_bs: int = 0 - cu_seqlens_q: Optional[torch.Tensor] = None - cu_seqlens_k: Optional[torch.Tensor] = None - max_seqlen_q: int = 0 - max_seqlen_k: int = 0 - min_seqlen_q: int = 0 - slot_mapping: Optional[torch.Tensor] = None - context_lens: Optional[torch.Tensor] = None - block_tables: Optional[torch.Tensor] = None - dropout_p: float = 0.0 - - max_q_len: Optional[int] = None - kv_indptr: Optional[torch.Tensor] = None - kv_indices: Optional[torch.Tensor] = None - kv_last_page_lens: Optional[torch.Tensor] = None - - -_CONTEXT = Context() - - -def get_context(): - return _CONTEXT - - -def set_context( - is_prefill: bool, - batch_size: int, - graph_bs=0, - cu_seqlens_q=None, - cu_seqlens_k=None, - max_seqlen_q=0, - max_seqlen_k=0, - min_seqlen_q=0, - slot_mapping=None, - context_lens=None, - block_tables=None, - dropout_p=0.0, - max_q_len=None, - kv_indptr=None, - kv_indices=None, - kv_last_page_lens=None, -): - global _CONTEXT - _CONTEXT = Context( - is_prefill, - batch_size, - graph_bs, - cu_seqlens_q, - cu_seqlens_k, - max_seqlen_q, - max_seqlen_k, - min_seqlen_q, - slot_mapping, - context_lens, - block_tables, - dropout_p, - max_q_len, - kv_indptr, - kv_indices, - kv_last_page_lens, - ) - - -def reset_context(): - global _CONTEXT - _CONTEXT = Context() diff --git a/atom/utils/forward_context.py b/atom/utils/forward_context.py index 7e031fd35d..0ad9841b74 100644 --- a/atom/utils/forward_context.py +++ b/atom/utils/forward_context.py @@ -1,45 +1,149 @@ from collections import defaultdict from contextlib import contextmanager -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union +from dataclasses import dataclass, field, fields +from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Set, Dict, Union +from atom.config import Config, KVCacheTensor import torch +from abc import ABC, abstractmethod @dataclass -class AttentionMetadata: - k_cache: torch.Tensor - v_cache: torch.Tensor - k_scale: torch.Tensor - v_scale: torch.Tensor +class Context: + # This context is used to store the basic context of the forward. + positions: torch.Tensor + is_prefill: bool = False + batch_size: int = 0 + graph_bs: int = 0 + def __init__( + self, + positions: torch.Tensor, + is_prefill: bool = False, + batch_size: int = 0, + graph_bs: int = 0, + ): + self.positions = positions + self.is_prefill = is_prefill + self.batch_size = batch_size + self.graph_bs = graph_bs + + +@dataclass +class AttentionMetaData: + """Attention metadata for prefill and decode batched together.""" + + cu_seqlens_q: Optional[torch.Tensor] = None + cu_seqlens_k: Optional[torch.Tensor] = None + max_seqlen_q: int = 0 + max_seqlen_k: int = 0 + min_seqlen_q: int = 0 + slot_mapping: Optional[torch.Tensor] = None + context_lens: Optional[torch.Tensor] = None + block_tables: Optional[torch.Tensor] = None + dropout_p: float = 0.0 + + max_q_len: Optional[int] = None + kv_indptr: Optional[torch.Tensor] = None + kv_indices: Optional[torch.Tensor] = None + kv_last_page_lens: Optional[torch.Tensor] = None + + def __init__( + self, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k: Optional[torch.Tensor] = None, + max_seqlen_q: int = 0, + max_seqlen_k: int = 0, + min_seqlen_q: int = 0, + slot_mapping: Optional[torch.Tensor] = None, + context_lens: Optional[torch.Tensor] = None, + block_tables: Optional[torch.Tensor] = None, + dropout_p: float = 0.0, + max_q_len: Optional[int] = None, + kv_indptr: Optional[torch.Tensor] = None, + kv_indices: Optional[torch.Tensor] = None, + kv_last_page_lens: Optional[torch.Tensor] = None, + ): + self.cu_seqlens_q = cu_seqlens_q + self.cu_seqlens_k = cu_seqlens_k + self.max_seqlen_q = max_seqlen_q + self.max_seqlen_k = max_seqlen_k + self.min_seqlen_q = min_seqlen_q + self.slot_mapping = slot_mapping + self.context_lens = context_lens + self.block_tables = block_tables + self.dropout_p = dropout_p + self.max_q_len = max_q_len + self.kv_indptr = kv_indptr + self.kv_indices = kv_indices + self.kv_last_page_lens = kv_last_page_lens + + def asdict_zerocopy(self, skip_fields: Optional[Set[str]] = None) -> Dict[str, Any]: + """Similar to dataclasses.asdict, but avoids deepcopying.""" + if skip_fields is None: + skip_fields = set() + # Note that if we add dataclasses as fields, they will need + # similar handling. + return { + field.name: getattr(self, field.name) + for field in fields(self) + if field.name not in skip_fields + } @dataclass class ForwardContext: # copy from vllm_config.compilation_config.static_forward_context no_compile_layers: dict[int, Any] = field(default_factory=dict) - """ - Type AttentionMetadata for v0, - Type Dict[str, AttentionMetadata] for v1, map from layer_name of each - attention layer to its attention metadata - set dynamically for each forward pass - """ + + attn_metadata: Optional[ + Union["AttentionMetaData", dict[str, "AttentionMetaData"]] + ] = None + + kv_cache_data: dict[str, KVCacheTensor] = None + + context: Optional[Context] = None + def __post_init__(self): - if not hasattr(self, 'no_compile_layers') or self.no_compile_layers is None: + if not hasattr(self, "no_compile_layers") or self.no_compile_layers is None: self.no_compile_layers = {} + if self.attn_metadata is None: + self.attn_metadata = {} _forward_context: Optional[ForwardContext] = ForwardContext() +_forward_kv_cache_context: Optional[ForwardContext] = ForwardContext() + def get_forward_context() -> ForwardContext: """Get the current forward context.""" assert _forward_context is not None, ( "Forward context is not set. " - "Please use `set_forward_context` to set the forward context.") + "Please use `set_forward_context` to set the forward context." + ) return _forward_context -def set_forward_context(layer_id: int, attention_metadata: AttentionMetadata) -> None: +def set_forward_context( + attn_metadata: AttentionMetaData, atom_config: Config, context: Context +) -> None: + + global _forward_context + _forward_context = ForwardContext( + attn_metadata=attn_metadata, + no_compile_layers=atom_config.compilation_config.static_forward_context, + kv_cache_data=_forward_kv_cache_context.kv_cache_data, + context=context, + ) + # _forward_context.attn_metadata = attn_metadata + # _forward_context.no_compile_layers = atom_config.compilation_config.static_forward_context + # _forward_context = ForwardContext(no_compile_layers=atom_config.compilation_config.static_forward_context, attn_metadata=attn_metadata) + +def reset_forward_context() -> None: global _forward_context - _forward_context.no_compile_layers[layer_id] = attention_metadata + _forward_context = ForwardContext() + + +def set_kv_cache_data(kv_cache_data: dict[int, KVCacheTensor]) -> None: + global _forward_kv_cache_context + _forward_kv_cache_context.kv_cache_data = kv_cache_data From 42e104bee27a58588386b19a7cf2110f4fa87eb5 Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Tue, 4 Nov 2025 22:39:44 +0800 Subject: [PATCH 102/119] Ds v32 new 2 (#24) * add deepseekv32 * fix bug & use aiter kernels for indexer * format * update * update2 * update * aligned dim to avoid d2d in inductor * update for mla sparse decode * fix --------- Co-authored-by: valarLip <340077269@qq.com> Co-authored-by: ZhangLirong-amd --- atom/config.py | 31 ++- atom/model_engine/model_runner.py | 99 ++++++-- atom/model_ops/attention_mla.py | 263 +++++++++++++++----- atom/model_ops/attentions/aiter_mla.py | 125 +++++----- atom/model_ops/attentions/backends.py | 21 +- atom/model_ops/base_attention.py | 1 + atom/model_ops/fp8_mqa_logits.py | 189 ++++++++++++++ atom/model_ops/layernorm.py | 70 +++++- atom/models/deepseek_v2.py | 332 +++++++++++++++++++++++-- atom/utils/forward_context.py | 9 + 10 files changed, 955 insertions(+), 185 deletions(-) create mode 100644 atom/model_ops/fp8_mqa_logits.py diff --git a/atom/config.py b/atom/config.py index 08f45facf9..bb8af34905 100644 --- a/atom/config.py +++ b/atom/config.py @@ -161,7 +161,6 @@ class CompilationConfig: Note that this flag is only effective when cudagraph_mode is PIECEWISE. """ - inductor_compile_config: dict = field(default_factory=dict) """Additional configurations for inductor. - None: use default configurations.""" @@ -332,6 +331,34 @@ def get_quant_config(config: PretrainedConfig) -> QuantizationConfig: return QuantizationConfig(quant_type, quant_dtype, is_dynamic) +_CONFIG_REGISTRY: dict[str, str] = { + "deepseek_v32": "deepseek_v3", +} + + +def get_hf_config(model: str) -> PretrainedConfig: + config_dict, _ = PretrainedConfig.get_config_dict( + model, + ) + model_type = config_dict.get("model_type") + + def _get_hf_token() -> str | None: + token = os.getenv("HF_TOKEN") + if token and token.strip(): + return token + return None + + if model_type in _CONFIG_REGISTRY: + config_class = AutoConfig.for_model(_CONFIG_REGISTRY[model_type]) + return config_class.from_pretrained( + model, + # revision=revision, + # code_revision=code_revision, + token=_get_hf_token(), + ) + return AutoConfig.from_pretrained(model) + + @dataclass class Config: model: str @@ -378,7 +405,7 @@ def __post_init__(self): self.kv_cache_block_size % 16 == 0 or self.kv_cache_block_size == 1 ), f"kv_cache_block_size ({self.kv_cache_block_size}) must be a multiple of 16 or 1" assert 1 <= self.tensor_parallel_size <= 8 - self.hf_config = AutoConfig.from_pretrained(self.model) + self.hf_config = get_hf_config(self.model) self.quant_config = get_quant_config(self.hf_config) self.max_model_len = min( self.max_model_len, self.hf_config.max_position_embeddings diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index f83d3e3da5..888ee80bad 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -29,6 +29,7 @@ "LlamaForCausalLM": LlamaForCausalLM, "MixtralForCausalLM": MixtralForCausalLM, "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, + "DeepseekV32ForCausalLM": DeepseekV2ForCausalLM, } # seed=1 # np.random.seed(seed) @@ -206,7 +207,12 @@ def __init__(self, rank: int, config: Config): self.block_size, use_mla=self.use_mla, ) - self.attn_metadata_builder = self.attn_backend.get_builder_cls()(self.block_size) + self.attn_metadata_builder = self.attn_backend.get_builder_cls()( + self.block_size + ) + self.is_deepseek_v32 = ( + hasattr(hf_config, "index_topk") if self.use_mla else False + ) # Initialize profiler for this rank self.profiler = None @@ -260,18 +266,22 @@ def __init__(self, rank: int, config: Config): def is_deepseek_mla(self) -> bool: if not hasattr(self.hf_text_config, "model_type"): return False - elif self.hf_text_config.model_type in \ - ('deepseek_v2', 'deepseek_v3', 'deepseek_mtp'): + elif self.hf_text_config.model_type in ( + "deepseek_v2", + "deepseek_v3", + "deepseek_v32", + "deepseek_mtp", + ): return self.hf_text_config.kv_lora_rank is not None - elif self.hf_text_config.model_type == 'eagle': + elif self.hf_text_config.model_type == "eagle": # if the model is an EAGLE module, check for the # underlying architecture - return self.hf_text_config.model.model_type in \ - ('deepseek_v2', 'deepseek_v3') \ + return ( + self.hf_text_config.model.model_type in ("deepseek_v2", "deepseek_v3") and self.hf_text_config.kv_lora_rank is not None + ) return False - def _make_buffer( self, *size: Union[int, torch.SymInt], dtype: torch.dtype, numpy: bool = True ) -> CpuGpuBuffer: @@ -336,6 +346,7 @@ def stop_profiler(self): return True def warmup_model(self): + start_time = time.time() torch.cuda.empty_cache() torch.cuda.reset_peak_memory_stats() max_num_batched_tokens, max_model_len = ( @@ -365,6 +376,9 @@ def warmup_model(self): self.forward(dummy_batch) self.tokenID_processor.clean() torch.cuda.empty_cache() + logger.info( + f"{self.label}: warmup_model {time.time() - start_time:.2f} seconds with {num_seqs} reqs {total_tokens_num} tokens" + ) def allocate_forward_vars(self): config = self.config @@ -391,6 +405,16 @@ def allocate_forward_vars(self): "kv_last_page_lens": CpuGpuBuffer(max_bs, **i32_kwargs), "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), } + if self.is_deepseek_v32: + self.forward_vars["cu_seqlen_ke"] = CpuGpuBuffer( + max_num_batched_tokens, **i32_kwargs + ) + self.forward_vars["cu_seqlen_ks"] = CpuGpuBuffer( + max_num_batched_tokens, **i32_kwargs + ) + self.forward_vars["sparse_kv_indptr"] = CpuGpuBuffer( + max_bs + 1, **i32_kwargs + ) self.forward_vars["cu_seqlens_q"].cpu.copy_( torch.arange(0, max_bs + 1, step=1, dtype=torch.int32) ) @@ -411,14 +435,22 @@ def get_num_blocks(self): current = torch.cuda.memory_stats()["allocated_bytes.all.current"] torch.set_default_device("cpu") num_kv_heads = hf_config.num_key_value_heads // self.world_size - isMLA = hf_config.architectures[0].startswith("DeepseekV") - if isMLA: + if self.use_mla: block_bytes = ( hf_config.num_hidden_layers * self.block_size * 576 * dtypes.d_dtypes[config.kv_cache_dtype].itemsize ) + if self.is_deepseek_v32: + index_dim = hf_config.index_head_dim + 4 + aligned_index_dim = ((index_dim + 15) // 16) * 16 + block_bytes += ( + hf_config.num_hidden_layers + * self.block_size + * aligned_index_dim + * dtypes.fp8.itemsize + ) else: block_bytes = ( 2 @@ -439,11 +471,8 @@ def allocate_kv_cache(self, num_kvcache_blocks): config = self.config config.num_kvcache_blocks = num_kvcache_blocks hf_config = config.hf_config - isMLA = hf_config.architectures[0].startswith("DeepseekV") num_kv_heads = hf_config.num_key_value_heads // self.world_size - - # Allocate raw KV cache tensors - if isMLA: + if self.use_mla: self.kv_cache = torch.zeros( hf_config.num_hidden_layers, config.num_kvcache_blocks, @@ -452,6 +481,19 @@ def allocate_kv_cache(self, num_kvcache_blocks): dtype=dtypes.d_dtypes[config.kv_cache_dtype], device="cuda", ) + if self.is_deepseek_v32: + # Align last dimension to 16 bytes for fp8 (1 byte per element) + # to avoid unaligned memory access in torch inductor + index_dim = hf_config.index_head_dim + 4 + aligned_index_dim = ((index_dim + 15) // 16) * 16 + self.index_cache = torch.zeros( + hf_config.num_hidden_layers, + config.num_kvcache_blocks, + self.block_size, + aligned_index_dim, + dtype=dtypes.fp8, + device="cuda", + ) else: self.kv_cache = torch.zeros( 2, @@ -527,7 +569,16 @@ def allocate_kv_cache(self, num_kvcache_blocks): 1, 576, ) - + module.max_model_len = self.config.max_model_len + if self.is_deepseek_v32 and module.indexer is not None: + # Use aligned dimension to avoid memory copy in torch inductor + module.indexer.k_cache.kv_cache[0] = self.index_cache[ + layer_id + ].view( + config.num_kvcache_blocks * self.block_size, + 1, + aligned_index_dim, + ) # Store in KVCacheTensor kv_cache_tensor = KVCacheTensor( layer_num=layer_id, @@ -559,7 +610,6 @@ def prepare_block_tables(self, seqs: list[Sequence]): def prepare_prefill(self, batch: ScheduledBatch): return self.attn_metadata_builder.prepare_prefill(batch, self.forward_vars) - def prepare_decode(self, batch: ScheduledBatch): scheduled_bs = batch.total_seqs_num_decode seqs = list(batch.seqs.values()) @@ -576,7 +626,10 @@ def prepare_decode(self, batch: ScheduledBatch): def prepare_intputs(self, batch: ScheduledBatch): is_prefill = batch.total_tokens_num_prefill > 0 - bs = 0 + bs = batch.total_seqs_num + num_scheduled_tokens = batch.num_scheduled_tokens + cu_seqlens_q, arange = self._get_cumsum_and_arange(num_scheduled_tokens) + self.forward_vars["cu_seqlens_q"].np[1 : bs + 1] = cu_seqlens_q if not is_prefill: scheduled_bs = batch.total_seqs_num_decode seqs = list(batch.seqs.values()) @@ -588,6 +641,7 @@ def prepare_intputs(self, batch: ScheduledBatch): else next(x for x in self.graph_bs if x >= scheduled_bs) ) assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" + self.forward_vars["cu_seqlens_q"].np[scheduled_bs + 1 : bs + 1] = self.forward_vars["cu_seqlens_q"].np[scheduled_bs] attn_metadata, positions = self.attn_metadata_builder.build(batch, self.forward_vars, bs) context_bs = batch.total_seqs_num_prefill if is_prefill else batch.total_seqs_num_decode @@ -603,13 +657,7 @@ def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: def prepare_model(self, batch: ScheduledBatch): total_tokens_num = batch.total_tokens_num - assert total_tokens_num > 0 - bs = batch.total_seqs_num - seqs = batch.seqs.values() - - num_scheduled_tokens = batch.num_scheduled_tokens - cu_seqlens_q, arange = self._get_cumsum_and_arange(num_scheduled_tokens) - self.forward_vars["cu_seqlens_q"].np[1 : bs + 1] = cu_seqlens_q + assert total_tokens_num > 0 input_ids = self.tokenID_processor.prepare_input_ids(batch) # print(f"input_ids: {input_ids}") @@ -642,8 +690,9 @@ def postprocess( temperatures: torch.Tensor, ) -> dict[int, int]: sampled_tokens = self.sampler(logits, temperatures) - # print(f"{logits=}") - # print(f"{sampled_tokens=}") + # if self.rank == 0: + # print(f"{logits=}") + # print(f"{sampled_tokens=}") token_ids = self.tokenID_processor.prepare_sampled_ids( batch, sampled_tokens, diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index 5ff8fdacc4..bdf3a3e865 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -1,41 +1,35 @@ +import logging from dataclasses import dataclass +from typing import Optional, Tuple + import torch -from torch import nn -from atom.utils.custom_register import direct_register_custom_op -from atom.utils.forward_context import AttentionMetaData, ForwardContext, get_forward_context -from atom.model_ops.linear import ColumnParallelLinear, RowParallelLinear -from atom.model_ops.utils import get_and_maybe_dequant_weights -from aiter.rotary_embedding import RotaryEmbedding -from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip - batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as _aiter_triton_fp8_bmm, -) from aiter import ( - flash_attn_varlen_func, concat_and_cache_mla, dtypes, - QuantType, - gemm_a8w8_blockscale, - get_hip_quant, + flash_attn_varlen_func, ) - from aiter.mla import mla_decode_fwd -from aiter.jit.utils.torch_guard import torch_compile_guard - -from typing import Optional, Tuple +from aiter.rotary_embedding import RotaryEmbedding +from torch import nn from tqdm import tqdm -import logging -from aiter.dist.parallel_state import get_tp_group -from atom.utils import mark_spliting_op +from atom.model_ops.utils import get_and_maybe_dequant_weights +from atom.utils.forward_context import AttentionMetaData, ForwardContext, get_forward_context + +from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip + batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as _aiter_triton_fp8_bmm, +) +torch.set_printoptions(threshold=10_000) logger = logging.getLogger("atom") + # MLA Specific Arguments @dataclass class MLAModules: - """Modules used in MLA. - """ + """Modules used in MLA.""" + q_lora_rank: Optional[int] kv_lora_rank: int qk_nope_head_dim: int @@ -46,6 +40,7 @@ class MLAModules: q_proj: Optional[torch.nn.Module] kv_b_proj: torch.nn.Module o_proj: torch.nn.Module + indexer: Optional[torch.nn.Module] def dynamic_per_batched_tensor_quant( @@ -68,7 +63,7 @@ def __init__( num_kv_heads: int, kv_cache_dtype: str, layer_num: int = 0, - mla_modules: MLAModules=None, + mla_modules: MLAModules = None, ) -> None: super().__init__() self.num_heads = num_heads @@ -89,6 +84,11 @@ def __init__( self.kv_b_proj = mla_modules.kv_b_proj self.kv_cache = torch.tensor([]) self._k_scale = torch.tensor(1.0, dtype=torch.float32) + self.topk_indices_buffer = ( + mla_modules.indexer.topk_indices_buffer + if mla_modules.indexer is not None + else None + ) self.layer_num = layer_num def process_weights_after_loading(self): @@ -126,33 +126,33 @@ def process_weights_after_loading(self): # triton kernel to avoid runtime compilation for unseen batch sizes # Pre-compile for batch sizes 1 to 1024 to cover most use-cases. # On DS-R1, this step adds roughly 50s to the model loading time. - max_batch_size = 1024 # [ToDo] Find the optimal upper limit - pre_compilation_list = list(range(1, max_batch_size + 1)) - if get_tp_group().is_first_rank: - pre_compilation_list = tqdm( - pre_compilation_list, - desc="[Aiter Triton] Pre-compiling fp8 BMM kernel", - total=max_batch_size, - ) - - for m in pre_compilation_list: - x = torch.empty( - (self.W_K.shape[0], m, self.W_K.shape[2]), - dtype=torch.bfloat16, - device=self.W_K.device, - ) - _aiter_triton_fp8_bmm( - x, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True - ) - - x = torch.empty( - (self.W_V.shape[0], m, self.W_V.shape[2]), - dtype=torch.bfloat16, - device=self.W_V.device, - ) - _aiter_triton_fp8_bmm( - x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True - ) + # max_batch_size = 1024 # [ToDo] Find the optimal upper limit + # pre_compilation_list = list(range(1, max_batch_size + 1)) + # if get_tp_group().is_first_rank: + # pre_compilation_list = tqdm( + # pre_compilation_list, + # desc="[Aiter Triton] Pre-compiling fp8 BMM kernel", + # total=max_batch_size, + # ) + + # for m in pre_compilation_list: + # x = torch.empty( + # (self.W_K.shape[0], m, self.W_K.shape[2]), + # dtype=torch.bfloat16, + # device=self.W_K.device, + # ) + # _aiter_triton_fp8_bmm( + # x, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True + # ) + + # x = torch.empty( + # (self.W_V.shape[0], m, self.W_V.shape[2]), + # dtype=torch.bfloat16, + # device=self.W_V.device, + # ) + # _aiter_triton_fp8_bmm( + # x, self.W_V, self.W_V_scale, group_size=128, transpose_bm=True + # ) else: # Convert from (L, N, V) to (N, L, V) self.W_UV = W_UV.transpose(0, 1) @@ -219,9 +219,9 @@ def _forward_prefill( kv_c_normed: torch.Tensor, k_pe: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, - context: AttentionMetaData, + attn_metadata: AttentionMetaData, ) -> torch.Tensor: - assert context is not None + assert attn_metadata is not None kv_nope = self.kv_b_proj(kv_c_normed).view( -1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim @@ -236,12 +236,12 @@ def _forward_prefill( q=q, k=k, v=v, - cu_seqlens_q=context.cu_seqlens_q, - cu_seqlens_k=context.cu_seqlens_k, - max_seqlen_q=context.max_seqlen_q, - max_seqlen_k=context.max_seqlen_k, - min_seqlen_q=context.min_seqlen_q, - dropout_p=context.dropout_p, + cu_seqlens_q=attn_metadata.cu_seqlens_q, + cu_seqlens_k=attn_metadata.cu_seqlens_k, + max_seqlen_q=attn_metadata.max_seqlen_q, + max_seqlen_k=attn_metadata.max_seqlen_k, + min_seqlen_q=attn_metadata.min_seqlen_q, + dropout_p=attn_metadata.dropout_p, softmax_scale=self.scale, causal=True, ) @@ -253,10 +253,10 @@ def _forward_decode( q_nope: torch.Tensor, q_pe: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, - context: AttentionMetaData, + attn_metadata: AttentionMetaData, ) -> torch.Tensor: assert kv_c_and_k_pe_cache.numel() > 0 - assert context is not None + assert attn_metadata is not None B = q_nope.shape[0] q = torch.cat([q_nope, q_pe], dim=-1) @@ -265,16 +265,27 @@ def _forward_decode( ) kv_buffer = kv_c_and_k_pe_cache.unsqueeze(2) + paged_kv_indptr = attn_metadata.kv_indptr + paged_kv_indices = attn_metadata.kv_indices + if self.topk_indices_buffer is not None: + paged_kv_indptr = attn_metadata.sparse_kv_indptr + paged_kv_indices = triton_convert_req_index_to_global_index( + attn_metadata.cu_seqlens_q, + attn_metadata.kv_indptr, + paged_kv_indptr, + attn_metadata.kv_indices, + self.topk_indices_buffer[:B], + ) self._aiter_mla_decode_fwd( q, kv_buffer, o, - context.cu_seqlens_q, - context.max_q_len, - context.kv_indptr, - context.kv_indices, - context.kv_last_page_lens, + attn_metadata.cu_seqlens_q, + attn_metadata.max_q_len, + paged_kv_indptr, + paged_kv_indices, + attn_metadata.kv_last_page_lens, self.scale, ) @@ -339,3 +350,125 @@ def forward( ) return output + + +import triton +import triton.language as tl + + +@triton.jit +def _convert_req_index_to_global_index_kernel( + qo_indptr, # int32 [num_requests] + kv_indptr, # int32 [num_requests+1] + page_kv_indptr, # int32 [num_requests+1] + kv_indices, # int32 [num_requests * max_num_blocks_per_req] + token_indices_ptr, # int32 [num_tokens, NUM_TOPK_TOKENS] + out_kv_indices, # int32 + # shapes (compile-time where possible) + NUM_TOPK_TOKENS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK_N: tl.constexpr, # tile width along columns + # strides (in elements) + ti_stride0, + ti_stride1, +): + # program_id(0) -> batch_id (row) + # program_id(1) -> tile index along columns + batch_id = tl.program_id(0) + tile_id = tl.program_id(1) + + # Each program covers BLOCK_N consecutive columns + indice_id = tile_id * BLOCK_N + tl.arange(0, BLOCK_N) + + # Load request id for this token (no mask: grid is exact) + kv_start = tl.load(kv_indptr + batch_id) + kv_end = tl.load(kv_indptr + batch_id + 1) + out_kv_start = tl.load(page_kv_indptr + batch_id) + kv_len = kv_end - kv_start + qo_start = tl.load(qo_indptr + batch_id) + qo_end = tl.load(qo_indptr + batch_id + 1) + + for token_id in range(qo_start, qo_end): + # Load token indices for this tile + ti_ptr = token_indices_ptr + token_id * ti_stride0 + indice_id * ti_stride1 + tok = tl.load(ti_ptr) # int32 + + # Guard block_table access + valid_mask = (indice_id < kv_len) & (indice_id < NUM_TOPK_TOKENS) + out_val = tl.load( + kv_indices + kv_start + tok, + mask=valid_mask, + other=0, + ) + + # Store results + out_ptr_ij = out_kv_indices + out_kv_start + indice_id + tl.store( + out_ptr_ij, + out_val, + mask=valid_mask, + ) + + +def triton_convert_req_index_to_global_index( + qo_indptr: torch.Tensor, # int32 [num_tokens + 1] + kv_indptr: torch.Tensor, # int32 [num_tokens + 1] + page_kv_indptr: torch.Tensor, # int32 [num_tokens + 1] + kv_indices: torch.Tensor, # int32 [total_kv_seqlen] + token_indices: torch.Tensor, # int32 [num_tokens, NUM_TOPK_TOKENS] + BLOCK_SIZE: int = 1, # page_block_size = 1 for now + NUM_TOPK_TOKENS: int = 2048, + BLOCK_N: int = 128, # tile width along columns +): + """ + out[token_id, indice_id] = + block_table[req_id[token_id], + token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_SIZE + + token_indices[token_id, indice_id] % BLOCK_SIZE + + Only when token_indices[token_id, indice_id] == -1 do we output -1. + For safety, we also output -1 if the derived block_id would be + out-of-bounds. + """ + assert kv_indices.dtype == torch.int32 + assert token_indices.dtype == torch.int32 + assert token_indices.shape[1] == NUM_TOPK_TOKENS + assert NUM_TOPK_TOKENS % BLOCK_N == 0, ( + f"NUM_TOPK_TOKENS ({NUM_TOPK_TOKENS}) must be divisible by" + f"BLOCK_N ({BLOCK_N})" + ) + + num_batch = kv_indptr.shape[0] - 1 + tiles_per_row = NUM_TOPK_TOKENS // BLOCK_N + + # Ensure contiguous tensors on the same device + qo_indptr_c = qo_indptr.contiguous() + kv_indptr_c = kv_indptr.contiguous() + kv_indices_c = kv_indices.contiguous() + token_indices_c = token_indices.contiguous() + page_kv_indptr_c = page_kv_indptr.contiguous() + # TODO: not support mtp + new_kv_indices = torch.empty_like(kv_indices) + + # Strides in elements + ti_stride0, ti_stride1 = token_indices_c.stride() + + # Exact 2D grid: tokens Γ— column tiles + grid = (num_batch, tiles_per_row) + + _convert_req_index_to_global_index_kernel[grid]( + qo_indptr_c, + kv_indptr_c, + page_kv_indptr_c, + kv_indices_c, + token_indices_c, + new_kv_indices, + # shapes / constexprs + NUM_TOPK_TOKENS, + BLOCK_SIZE, + BLOCK_N, + # strides + ti_stride0, + ti_stride1, + ) + return new_kv_indices diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index 15c1709fec..5c19f25d14 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -7,8 +7,13 @@ import torch import numpy as np +import numpy as np +import torch from atom.model_engine.scheduler import ScheduledBatch from atom.model_ops.attention_mla import MLAAttention +from atom.config import get_current_atom_config + +from .backends import AttentionBackend, CommonAttentionBuilder class AiterMLABackend(AttentionBackend): @@ -26,50 +31,18 @@ def get_impl_cls() -> Type["MLAAttention"]: class AiterMLAMetadataBuilder(CommonAttentionBuilder): - BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] def __init__(self, block_size: int): - super().__init__(block_size) # Call parent __init__ to initialize _cached_kv_cache_data + super().__init__( + block_size + ) # Call parent __init__ to initialize _cached_kv_cache_data assert self.block_size == 1, "AITER MLA requires only block size 1." - def prepare(self, batch: ScheduledBatch): - self.paged_kv_indices: list[int] = [] - self.paged_kv_indptr: list[int] = [0] - self.paged_kv_last_page_lens: list[int] = [] - self.total_blocks = 0 - - seqs = list(batch.seqs.values()) - for seq in seqs: - current_seq_len = seq.num_tokens - self._update_paged_kv_tensors(seq.block_table, current_seq_len) - - def _update_paged_kv_tensors(self, block_table: list[int], seq_len: int): - # Get the number of valid blocks based on sequence length. - # If seq_len = 16, block_size = 16, - # block_table_bound is 1 with 1 valid block. - # If seq_len = 15, block_size = 16, - # block_table_bound is 0 + 1 with 1 valid block. - self.total_blocks += len(block_table) - block_table_bound = ( - seq_len // self.block_size + 1 - if seq_len % self.block_size != 0 - else seq_len // self.block_size - ) - self.paged_kv_indices.extend(block_table[:block_table_bound]) - self.paged_kv_indptr.append(self.paged_kv_indptr[-1] + block_table_bound) - - last_page_len = seq_len % self.block_size - if last_page_len == 0: - last_page_len = self.block_size - self.paged_kv_last_page_lens.append(last_page_len) - - def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): scheduled_bs = batch.total_seqs_num_decode seqs = list(batch.seqs.values()) dropout_p = 0.0 max_q_len = 1 - self.total_blocks = 0 context_lens = [seq.num_tokens for seq in seqs] positions = context_lens @@ -77,56 +50,69 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): seq.block_table[-1] * self.block_size + seq.last_block_num_tokens - 1 for seq in seqs ] - slot_mapping.extend([-1] * (bs - scheduled_bs)) - - self.prepare(batch) + var = forward_vars + # prepare_block_tables + block_tables = var["block_tables"].np + for i, seq in enumerate(seqs): + block_tables[i] = 0 + block_tables[i, : seq.num_blocks] = seq.block_table sum_scheduled_tokens = batch.total_tokens_num_decode - var = forward_vars - var["slot_mapping"].np[:bs] = slot_mapping + var["slot_mapping"].np[:scheduled_bs] = slot_mapping + var["slot_mapping"].np[scheduled_bs:bs] = -1 var["positions"].np[:sum_scheduled_tokens] = positions var["context_lens"].np[:scheduled_bs] = context_lens + + sum_blocks = 0 + for seq in seqs: + var["kv_indices"].np[ + sum_blocks : sum_blocks + seq.num_blocks + ] = seq.block_table + sum_blocks += seq.num_blocks + kv_indptr = np.cumsum([seq.num_blocks for seq in seqs]) + var["kv_indptr"].np[1 : scheduled_bs + 1] = kv_indptr + var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = sum_blocks + var["kv_last_page_lens"].np[:scheduled_bs] = [ + seq.last_block_num_tokens for seq in seqs + ] + var["kv_last_page_lens"].np[scheduled_bs:bs] = 0 vars_used = [ ("slot_mapping", bs), # TODO: MTP support ("context_lens", bs), + ("block_tables", bs), + ("cu_seqlens_q", bs + 1), + ("kv_indptr", bs + 1), + ("kv_indices", sum_blocks), + ("kv_last_page_lens", bs), ] - - if len(self.paged_kv_indptr) > 0: - # extend to the maximum number of blocks as returned by the scheduler - self.paged_kv_indices.extend([0] * (self.total_blocks - len(self.paged_kv_indices))) - var["kv_indices"].np[: self.total_blocks] = np.array( - self.paged_kv_indices, dtype=np.int64 + config = get_current_atom_config() + if hasattr(config.hf_config, "index_topk"): + index_topk = config.hf_config.index_topk + sparse_context_lens = np.clip(var["context_lens"].np[:bs], None, index_topk) + var["sparse_kv_indptr"].np[1 : bs + 1] = np.cumsum( + sparse_context_lens, dtype=np.int32 ) - var["kv_indptr"].np[: scheduled_bs + 1] = np.array( - self.paged_kv_indptr, dtype=np.int64 - ) - var["kv_indptr"].np[scheduled_bs + 1 : bs + 1] = var["kv_indptr"].np[ - scheduled_bs - ] - var["kv_last_page_lens"].np[:scheduled_bs] = np.array( - self.paged_kv_last_page_lens, dtype=np.int64 - ) - var["kv_last_page_lens"].np[scheduled_bs:bs] = 0 - vars_used = [ - ("slot_mapping", bs), # TODO: MTP support - ("context_lens", bs), - # ("block_tables", bs), - ("cu_seqlens_q", bs + 1), - ("kv_indices", sum(context_lens)), - ("kv_indptr", bs + 1), - ("kv_last_page_lens", bs), - ] + var["sparse_kv_indptr"].np[scheduled_bs : bs + 1] = var[ + "sparse_kv_indptr" + ].np[scheduled_bs] + vars_used.append(("sparse_kv_indptr", bs + 1)) + ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} attn_metadata = AttentionMetaData( dropout_p=dropout_p, max_q_len=max_q_len, **ctx, ) - positions=var["positions"].copy_to_gpu(sum_scheduled_tokens) + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) return attn_metadata, positions def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: + sparse_kv_indptr = ( + forward_vars["sparse_kv_indptr"].gpu + if "sparse_kv_indptr" in forward_vars + else None + ) attn_matadata = AttentionMetaData( slot_mapping=forward_vars["slot_mapping"].gpu[:bs], context_lens=forward_vars["context_lens"].gpu[:bs], @@ -136,7 +122,10 @@ def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaDat kv_indptr=forward_vars["kv_indptr"].gpu[: bs + 1], kv_indices=forward_vars["kv_indices"].gpu[:], kv_last_page_lens=forward_vars["kv_last_page_lens"].gpu[:bs], + sparse_kv_indptr=sparse_kv_indptr, + ) + positions = forward_vars["positions"].copy_to_gpu(bs) + context = Context( + positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs ) - positions=forward_vars["positions"].copy_to_gpu(bs) - context = Context(positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs) return attn_matadata, context diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py index b0f558ce60..988f2c53c7 100644 --- a/atom/model_ops/attentions/backends.py +++ b/atom/model_ops/attentions/backends.py @@ -2,12 +2,14 @@ from typing import Type, Tuple, Generic, Optional, List, TypeVar, Dict, Any from atom.model_engine.scheduler import ScheduledBatch from atom.model_ops.attention_mla import MLAModules +import numpy as np from atom.utils.forward_context import AttentionMetaData import torch from torch import nn -T = TypeVar('T', bound="BroadcastableModelInput") +T = TypeVar("T", bound="BroadcastableModelInput") + class BroadcastableModelInput(ABC): @@ -35,6 +37,7 @@ def from_broadcasted_tensor_dict( class AttentionBackend(ABC): """Abstract class for attention backends.""" + # For some attention backends, we allocate an output tensor before # calling the custom op. When piecewise cudagraph is enabled, this # makes sure the output tensor is allocated inside the cudagraph. @@ -55,7 +58,6 @@ def get_impl_cls() -> Type["AttentionImpl"]: return AttentionImpl - class AttentionMetadataBuilder(ABC, Generic[T]): """Abstract class for attention metadata builders.""" @@ -129,6 +131,21 @@ def prepare_prefill(self, batch: ScheduledBatch, forward_vars): ("cu_seqlens_q", bs + 1), ("slot_mapping", len(slot_mapping)), ] + if "cu_seqlen_ke" in var and "cu_seqlen_ks" in var: + block_tables = forward_vars["block_tables"].np + for i, seq in enumerate(seqs): + block_tables[i] = 0 + if len(seq.block_table) > 0: + block_tables[i, : seq.num_blocks] = seq.block_table + var["cu_seqlen_ke"].np[:sum_scheduled_tokens] = ( + np.arange(sum_scheduled_tokens, dtype=np.int32) + 1 + ) + counts = var["cu_seqlens_q"].np[1 : bs + 1] - var["cu_seqlens_q"].np[:bs] + var["cu_seqlen_ks"].np[:sum_scheduled_tokens] = np.repeat( + var["cu_seqlens_q"].np[:bs], counts + ) + vars_used.append(("cu_seqlen_ke", sum_scheduled_tokens)) + vars_used.append(("cu_seqlen_ks", sum_scheduled_tokens)) ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} attn_metadata = AttentionMetaData( cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py index 55e0000315..015be5bbbc 100644 --- a/atom/model_ops/base_attention.py +++ b/atom/model_ops/base_attention.py @@ -68,6 +68,7 @@ def __init__( self.use_mla = use_mla self.base_attention = None self.kv_cache = torch.tensor([]) + self.indexer = mla_modules.indexer atom_config = get_current_atom_config() block_size = atom_config.kv_cache_block_size diff --git a/atom/model_ops/fp8_mqa_logits.py b/atom/model_ops/fp8_mqa_logits.py new file mode 100644 index 0000000000..b8d6ee5881 --- /dev/null +++ b/atom/model_ops/fp8_mqa_logits.py @@ -0,0 +1,189 @@ +import triton +import triton.language as tl +import torch +import math + + +@triton.jit +def fp8_mqa_logits_kernel( + Q_ptr, # fp8e4m3 [seq_len, H, D] + KV_ptr, # fp8e4m3 [seq_len_kv, D] + kv_scales_ptr, # fp32 [seq_len_kv] + weights_ptr, # fp32 [seq_len, H] + cu_start_ptr, # int32 [seq_len] + cu_end_ptr, # int32 [seq_len] + logits_ptr, # fp32 [seq_len, seq_len_kv] + seq_len, + seq_len_kv, + NUM_HEADS: tl.constexpr, + HEAD_SIZE: tl.constexpr, + # strides + stride_q_s, + stride_q_h: tl.constexpr, + stride_q_d: tl.constexpr, + stride_kv_s, + stride_kv_d: tl.constexpr, + stride_w_s, + stride_w_h: tl.constexpr, + stride_logits_s, + stride_logits_k, + # block sizes + BLOCK_Q: tl.constexpr, + BLOCK_KV: tl.constexpr, +): + pid_q = tl.program_id(0) + + row_offsets = pid_q * BLOCK_Q + tl.arange(0, BLOCK_Q) + q_mask = row_offsets < seq_len + + h_inds = tl.arange(0, NUM_HEADS) + d_inds = tl.arange(0, HEAD_SIZE) + + # load Q[BLOCK_Q, NUM_HEADS, HEAD_SIZE] + q_ptrs = ( + Q_ptr + + row_offsets[:, None, None] * stride_q_s + + h_inds[None, :, None] * stride_q_h + + d_inds[None, None, :] * stride_q_d + ) + + q_block = tl.load( + q_ptrs, mask=q_mask[:, None, None], other=0.0, cache_modifier=".cg" + ) + q_block = tl.reshape(q_block, (BLOCK_Q * NUM_HEADS, HEAD_SIZE)) + + w_ptrs = ( + weights_ptr + row_offsets[:, None] * stride_w_s + h_inds[None, :] * stride_w_h + ) + w_block = tl.load(w_ptrs, mask=q_mask[:, None], other=0.0, cache_modifier=".cg").to( + tl.float32 + ) + + # Load start/end for each row in this block + start_inds = tl.load(cu_start_ptr + row_offsets, mask=q_mask, other=seq_len_kv) + end_inds = tl.load(cu_end_ptr + row_offsets, mask=q_mask, other=0) + + # Compute kv tile range, both ends are inclusive + block_min_start = tl.min(start_inds) + block_max_end = tl.max(end_inds) + + block_min_start = tl.maximum(block_min_start, 0) + block_max_end = tl.minimum(block_max_end, seq_len_kv) + + kv_tile_start = block_min_start // BLOCK_KV + kv_tile_end = (block_max_end + BLOCK_KV - 1) // BLOCK_KV + + # Loop over KV tiles + for kv_tile_ind in tl.range(kv_tile_start, kv_tile_end): + kv_col_offsets = kv_tile_ind * BLOCK_KV + tl.arange(0, BLOCK_KV) + kv_col_mask = kv_col_offsets < seq_len_kv + + # Load KV tile [HEAD_SIZE, BLOCK_KV] + kv_ptrs = ( + KV_ptr + + kv_col_offsets[None, :] * stride_kv_s + + d_inds[:, None] * stride_kv_d + ) + kv_block = tl.load(kv_ptrs, mask=kv_col_mask[None, :], other=0.0) + # [BLOCK_Q*NUM_HEADS, BLOCK_KV] = [BLOCK_Q*NUM_HEADS, HEAD_SIZE] x [HEAD_SIZE, BLOCK_KV] + scores = tl.dot(q_block, kv_block, input_precision="ieee").to(tl.float32) + + # Multiply by kv_scales (broadcast along rows) + kv_scales = tl.load( + kv_scales_ptr + kv_col_offsets, mask=kv_col_mask, other=0.0 + ).to(tl.float32) + scores = scores * kv_scales[None, :] + scores = tl.reshape(scores, (BLOCK_Q, NUM_HEADS, BLOCK_KV)) + + # ReLU + scores = tl.maximum(scores, 0.0) + + # Apply per-row head weights and sum over heads + scores = scores * w_block[:, :, None] + # [BLOCK_Q, BK] + scores = tl.sum(scores, axis=1) + kv_cols = kv_col_offsets[None, :] + # [BQ, BK] + in_window = (kv_cols >= start_inds[:, None]) & (kv_cols < end_inds[:, None]) + store_mask = ( + (row_offsets[:, None] < seq_len) + & (kv_col_offsets[None, :] < seq_len_kv) + & in_window + ) + scores = tl.where(store_mask, scores, float("-inf")) + + # Store to logits [BLOCK_Q, BK] + logits_ptrs = ( + logits_ptr + + row_offsets[:, None] * stride_logits_s + + kv_col_offsets[None, :] * stride_logits_k + ) + tl.store(logits_ptrs, scores, mask=store_mask) + + +def fp8_mqa_logits( + Q, + KV, + kv_scales, + weights, + cu_starts, + cu_ends, +): + """ + Q: [seq_len, NUM_HEADS, HEAD_SIZE], dtype float8 + KV: [seq_len_kv, HEAD_SIZE], dtype float8 + kv_scales: [seq_len_kv], dtype float32 + weights: [seq_len, NUM_HEADS], dtype float32 + cu_starts: [seq_len], dtype int32 + cu_ends: [seq_len], dtype int32 + + Returns: + logits: [seq_len, seq_len_kv], dtype float32 (must be initialized to -inf, because of causal masking) + """ + # TODO (cagri): double check what value to put for causally masked logits, 0 or -inf? + # TODO (cagri): Tune/optimize + BLOCK_Q = 1 + BLOCK_KV = 128 + seq_len, num_heads, head_size = Q.shape + seq_len_kv = KV.shape[0] + # TODO (cagri): Currently assuming num_heads and head_size is power of 2. + assert (num_heads & (num_heads - 1) == 0), "num q. heads should be power of 2." + assert (head_size & (head_size - 1) == 0), "head size should be power of 2." + # Initialize with -inf because of causal masking + logits = torch.full((seq_len, seq_len_kv), + fill_value=-float('inf'), + dtype=torch.float32, device=Q.device) + + stride_q_s, stride_q_h, stride_q_d = Q.stride() + stride_kv_s, stride_kv_d = KV.stride() + stride_w_s, stride_w_h = weights.stride() + stride_logits_s, stride_logits_k = logits.stride() + fp8_mqa_logits_kernel[(triton.cdiv(seq_len, BLOCK_Q),)]( + Q_ptr=Q, + KV_ptr=KV, + kv_scales_ptr=kv_scales, + weights_ptr=weights, + cu_start_ptr=cu_starts, + cu_end_ptr=cu_ends, + logits_ptr=logits, + seq_len=seq_len, + seq_len_kv=seq_len_kv, + NUM_HEADS=num_heads, + HEAD_SIZE=head_size, + stride_q_s=stride_q_s, + stride_q_h=stride_q_h, + stride_q_d=stride_q_d, + stride_kv_s=stride_kv_s, + stride_kv_d=stride_kv_d, + stride_w_s=stride_w_s, + stride_w_h=stride_w_h, + stride_logits_s=stride_logits_s, + stride_logits_k=stride_logits_k, + BLOCK_Q=BLOCK_Q, + BLOCK_KV=BLOCK_KV, + num_warps=4, + num_stages=2, + waves_per_eu=2, + ) + + return logits \ No newline at end of file diff --git a/atom/model_ops/layernorm.py b/atom/model_ops/layernorm.py index db06ed311e..7262e4ce19 100644 --- a/atom/model_ops/layernorm.py +++ b/atom/model_ops/layernorm.py @@ -1,17 +1,29 @@ from typing import List, Tuple, Optional, Union import torch from torch import nn -from aiter import rmsnorm2d_fwd, rmsnorm2d_fwd_with_add, rms_norm +from aiter import ( + rmsnorm2d_fwd, + rmsnorm2d_fwd_with_add, + rms_norm, + layernorm2d_fwd, + layernorm2d_fwd_with_add, +) from aiter.jit.utils.torch_guard import torch_compile_guard + @torch_compile_guard() -def rmsnorm2d_fwd_(x: torch.Tensor, weight: torch.Tensor, eps: float, dim: int) -> torch.Tensor: +def rmsnorm2d_fwd_( + x: torch.Tensor, weight: torch.Tensor, eps: float, dim: int +) -> torch.Tensor: ori_shape = x.shape x = x.reshape(-1, dim) return rmsnorm2d_fwd(x, weight, eps).view(ori_shape) + @torch_compile_guard() -def rmsnorm2d_fwd_with_add_(x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor, eps: float, dim: int) -> Tuple[torch.Tensor, torch.Tensor]: +def rmsnorm2d_fwd_with_add_( + x: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor, eps: float, dim: int +) -> Tuple[torch.Tensor, torch.Tensor]: ori_shape = x.shape x = x.reshape(-1, dim) out = torch.empty_like(x) @@ -67,3 +79,55 @@ def forward( else: # return self.add_rms_forward(x, residual) return rmsnorm2d_fwd_with_add_(x, self.weight, residual, self.eps, self.dim) + + +@torch_compile_guard() +def layernorm2d_fwd_( + x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float, dim: int +) -> torch.Tensor: + ori_shape = x.shape + x = x.reshape(-1, dim) + return layernorm2d_fwd(x, weight, bias, eps).view(ori_shape) + + +@torch_compile_guard() +def layernorm2d_fwd_with_add_( + x: torch.Tensor, + weight: torch.Tensor, + residual: torch.Tensor, + bias: torch.Tensor, + eps: float, + dim: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + ori_shape = x.shape + x = x.reshape(-1, dim) + out = torch.empty_like(x) + residual_out = torch.empty_like(x) + layernorm2d_fwd_with_add(out, x, residual, residual_out, weight, bias, eps) + return out.view(ori_shape), residual_out.view(ori_shape) + + +class LayerNorm(nn.Module): + + def __init__( + self, + dim: int, + eps: float = 1e-6, + ) -> None: + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + self.bias = nn.Parameter(torch.zeros(dim)) + + def forward( + self, + x: torch.Tensor, + residual: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + if residual is None: + return layernorm2d_fwd_(x, self.weight, self.bias, self.eps, self.dim) + else: + return layernorm2d_fwd_with_add_( + x, self.weight, residual, self.bias, self.eps, self.dim + ) diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index 424a68eece..c11cf2c798 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -25,39 +25,62 @@ from typing import Any, Dict, Iterable, Optional, Set, Tuple, Union import torch +from aiter import ( + QuantType, + concat_and_cache_mla, + cp_gather_indexer_k_quant_cache, + dtypes, + flash_attn_varlen_func, + gemm_a8w8_blockscale, + get_hip_quant, + indexer_k_quant_and_cache, + topk_per_row, + topk_per_row_decode, +) +from aiter.dist.communication_op import tensor_model_parallel_all_reduce +from aiter.dist.parallel_state import ( + get_pp_group, + get_tensor_model_parallel_world_size, + get_tp_group, +) +from aiter.ops.triton.pa_mqa_logits import ( + deepgemm_fp8_paged_mqa_logits, + deepgemm_fp8_paged_mqa_logits_stage1, +) +from aiter.rotary_embedding import get_rope from torch import nn from transformers import PretrainedConfig +from atom.config import Config, QuantizationConfig, get_current_atom_config +from atom.model_ops.activation import SiluAndMul from atom.model_ops.attention_mla import MLAModules from atom.model_ops.base_attention import Attention -from atom.config import QuantizationConfig, Config -from aiter.dist.parallel_state import (get_pp_group, - get_tensor_model_parallel_world_size) -from aiter.dist.communication_op import tensor_model_parallel_all_reduce -from atom.model_ops.activation import SiluAndMul -from atom.model_ops.moe import FusedMoE -from atom.model_ops.layernorm import RMSNorm -from atom.model_ops.linear import (ColumnParallelLinear, +from atom.model_ops.embed_head import ParallelLMHead, VocabParallelEmbedding +from atom.model_ops.fp8_mqa_logits import fp8_mqa_logits +from atom.model_ops.layernorm import LayerNorm, RMSNorm +from atom.model_ops.linear import ( + ColumnParallelLinear, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, ) -from aiter.rotary_embedding import get_rope -from atom.model_ops.embed_head import ( - ParallelLMHead, VocabParallelEmbedding) - -from atom.models.utils import (PPMissingLayer, - make_empty_intermediate_tensors_factory, make_layers, - maybe_prefix, - IntermediateTensors, -) - +from atom.model_ops.moe import FusedMoE from atom.model_ops.topK import ( + is_rocm_aiter_fuse_routed_scaling_factor, is_rocm_aiter_fusion_shared_expert_enabled, - is_rocm_aiter_fuse_routed_scaling_factor ) +from atom.models.utils import ( + IntermediateTensors, + PPMissingLayer, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) +from atom.utils.forward_context import get_forward_context +from atom.utils.custom_register import direct_register_custom_op from atom.utils.decorators import support_torch_compile +# from vllm.model_executor.layers.quantization.utils.fp8_utils import per_token_group_quant_fp8 class DeepseekV2MLP(nn.Module): @@ -191,6 +214,240 @@ def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: return 0.1 * mscale * math.log(scale) + 1.0 +class DeepseekV32IndexerCache(nn.Module): + + def __init__(self, head_dim: int, dtype: torch.dtype, prefix: str, + cache_config: str): + super().__init__() + self.kv_cache = [torch.tensor([])] + self.head_dim = head_dim + self.prefix = prefix + self.cache_config = cache_config + self.dtype = dtype + + +def sparse_attn_indexer( + hidden_states: torch.Tensor, + k_cache_prefix: str, + kv_cache: torch.Tensor, + q_fp8: torch.Tensor, + k: torch.Tensor, + weights: torch.Tensor, + quant_block_size: int, + scale_fmt: Optional[str], + topk_tokens: int, + head_dim: int, + max_model_len: int, + total_seq_lens: int, + topk_indices_buffer: torch.Tensor, +) -> torch.Tensor: + # careful! this will be None in dummy run + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata + context = forward_context.context + slot_mapping = attn_metadata.slot_mapping + if kv_cache.numel() == 0: + # dummy runner + return weights + num_decode_tokens = context.batch_size if not context.is_prefill else 0 + indexer_k_quant_and_cache( + k, + kv_cache, + slot_mapping, + quant_block_size, + scale_fmt, + ) + if context.is_prefill: + prefill_metadata = attn_metadata + num_prefills = context.batch_size + total_seq_lens = hidden_states.shape[0] + k_fp8 = torch.empty( + [total_seq_lens, head_dim], + device=k.device, + dtype=dtypes.fp8) + k_scale = torch.empty( + [total_seq_lens, 1], + device=k.device, + dtype=torch.float32) + if prefill_metadata.block_tables.shape[0] < num_prefills: + new_shape = (num_prefills, prefill_metadata.block_tables.shape[1]) + prefill_metadata.block_tables = torch.full(new_shape, -1, dtype=torch.long, device=prefill_metadata.block_tables.device) + cp_gather_indexer_k_quant_cache( + kv_cache, + k_fp8, + k_scale.view(dtypes.fp8), + prefill_metadata.block_tables, + prefill_metadata.cu_seqlens_q, + # num_prefills, + ) + cu_seqlen_ks = prefill_metadata.cu_seqlen_ks + cu_seqlen_ke = prefill_metadata.cu_seqlen_ke + num_tokens = hidden_states.shape[0] + logits = fp8_mqa_logits(Q=q_fp8[num_decode_tokens:num_tokens], KV=k_fp8, kv_scales=k_scale, weights=weights[num_decode_tokens:num_tokens], + cu_starts=cu_seqlen_ks, cu_ends=cu_seqlen_ke) + num_rows = logits.shape[0] + assert topk_tokens == 2048, "top_k_per_row assumes size 2048" + topk_indices = topk_indices_buffer[ + num_decode_tokens:num_tokens, :topk_tokens + ] + topk_per_row( + logits, + cu_seqlen_ks, + cu_seqlen_ke, + topk_indices, + num_rows, + logits.stride(0), + logits.stride(1), + ) + else: + decode_metadata = attn_metadata + # kv_cache size requirement [num_block, block_size, n_head, head_dim], + # we only have [num_block, block_size, head_dim], + kv_cache = kv_cache.unsqueeze(-2) + padded_q_fp8_decode_tokens = q_fp8[:num_decode_tokens].reshape(context.batch_size, -1, *q_fp8.shape[1:]) + # TODO: move and optimize below logic with triton kernels + batch_size = padded_q_fp8_decode_tokens.shape[0] + next_n = padded_q_fp8_decode_tokens.shape[1] + assert batch_size == context.batch_size + num_padded_tokens = batch_size * next_n + batch_size, next_n, heads, _ = padded_q_fp8_decode_tokens.shape + logits = torch.empty([batch_size * next_n, max_model_len], dtype=torch.float32, device="cuda") + deepgemm_fp8_paged_mqa_logits(padded_q_fp8_decode_tokens, kv_cache, weights[:num_padded_tokens], logits, decode_metadata.context_lens, attn_metadata.block_tables, max_model_len) + num_rows = logits.shape[0] + assert topk_tokens == 2048, "top_k_per_row assumes size 2048" + topk_indices = topk_indices_buffer[ + :num_decode_tokens, :topk_tokens + ] + topk_per_row_decode( + logits, + next_n, + decode_metadata.context_lens, + topk_indices, + num_rows, + logits.stride(0), + logits.stride(1), + ) + return weights + +def sparse_attn_indexer_fake( + hidden_states: torch.Tensor, + k_cache_prefix: str, + kv_cache: torch.Tensor, + q_fp8: torch.Tensor, + k: torch.Tensor, + weights: torch.Tensor, + quant_block_size: int, + scale_fmt: Optional[str], + topk_tokens: int, + head_dim: int, + max_model_len: int, + total_seq_lens: int, + topk_indices_buffer: torch.Tensor, +) -> torch.Tensor: + # profile run + # NOTE(Chen): create the max possible flattened_kv. So that + # profile_run can get correct memory usage. + _flattened_kv = torch.empty([total_seq_lens, head_dim + 4], + device=k.device, + dtype=torch.uint8) + _k_fp8 = _flattened_kv[..., :head_dim].view( + torch.float8_e4m3fn).contiguous() + _k_scale = _flattened_kv[..., head_dim:].view(torch.float32).contiguous() + return weights + +direct_register_custom_op( + op_name="sparse_attn_indexer", + op_func=sparse_attn_indexer, + mutates_args=["topk_indices_buffer"], + fake_impl=sparse_attn_indexer_fake, +) + +class Indexer(nn.Module): + + def __init__(self, + atom_config: Config, + config: PretrainedConfig, + hidden_size: int, + q_lora_rank: int, + quant_config: Optional[QuantizationConfig], + cache_config: str, + topk_indices_buffer: Optional[torch.Tensor], + prefix: str = ""): + super().__init__() + self.atom_config = atom_config + self.config = config + # self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"] + self.topk_tokens = config.index_topk + self.n_head = config.index_n_heads # 64 + self.head_dim = config.index_head_dim # 128 + self.rope_dim = config.qk_rope_head_dim # 64 + self.q_lora_rank = q_lora_rank # 1536 + # no tensor parallel, just replicated + self.wq_b = ReplicatedLinear(self.q_lora_rank, + self.head_dim * self.n_head, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.wq_b") + self.wk = ReplicatedLinear(hidden_size, + self.head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.wk") + self.k_norm = LayerNorm(self.head_dim, eps=1e-6) + self.weights_proj = ReplicatedLinear(hidden_size, + self.n_head, + quant_config=None, + prefix=f"{prefix}.weights_proj") + self.softmax_scale = self.head_dim**-0.5 + + self.scale_fmt = "ue8m0" + self.quant_func = get_hip_quant(QuantType.per_1x128) + self.quant_block_size = 128 # TODO: get from config + self.topk_indices_buffer = topk_indices_buffer + + # TODO (zyongye) change dim to fp8 later to (self.head_dim + 4) + self.k_cache = DeepseekV32IndexerCache(head_dim=self.head_dim + 4, + dtype=torch.uint8, + prefix=f"{prefix}.k_cache", + cache_config=cache_config) + self.max_model_len = atom_config.max_model_len + self.prefix = prefix + self.max_total_seq_len = atom_config.max_num_seqs * self.max_model_len + # register_metadata_builder("indexer_attn_metadata", self.k_cache.get_attn_backend().get_builder_cls()) + + def forward(self, hidden_states: torch.Tensor, qr: torch.Tensor, positions, + rotary_emb) -> torch.Tensor: + q = self.wq_b(qr) + q = q.view(-1, self.n_head, self.head_dim) + q_pe, q_nope = torch.split( + q, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1) + + k = self.wk(hidden_states) + k = self.k_norm(k) + k_pe, k_nope = torch.split( + k, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1) + + q_pe, k_pe = rotary_emb(positions, q_pe, k_pe) + + # we only quant q here since k quant is fused with cache insertion + q = q.view(-1, self.head_dim) + + q_fp8, q_scale = self.quant_func(q, quant_dtype=dtypes.fp8) + q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim) + q_scale = q_scale.view(-1, self.n_head, 1) + + weights = self.weights_proj(hidden_states) + weights = weights.unsqueeze( + -1) * q_scale * self.softmax_scale * self.n_head**-0.5 + weights = weights.squeeze(-1) + + return torch.ops.aiter.sparse_attn_indexer( + hidden_states, self.k_cache.prefix, self.k_cache.kv_cache[0], + q_fp8, k, weights, self.quant_block_size, self.scale_fmt, + self.topk_tokens, self.head_dim, self.max_model_len, + self.max_total_seq_len, self.topk_indices_buffer) + + class DeepseekV2MLAAttention(nn.Module): """ Main reference: DeepseekV2 paper, and FlashInfer Implementation @@ -216,6 +473,7 @@ def __init__( quant_config: Optional[QuantizationConfig] = None, prefix: str = "", layer_num: int = 0, + topk_indices_buffer: Optional[torch.Tensor] = None, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -293,6 +551,22 @@ def __init__( mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale + + self.is_v32 = hasattr(config, "index_topk") + + if self.is_v32: + self.indexer = Indexer( + get_current_atom_config(), + config, + hidden_size, + q_lora_rank, + quant_config, + cache_config, + topk_indices_buffer, + f"{prefix}.indexer", + ) + else: + self.indexer = None # In the MLA backend, kv_cache includes both k_c and # pe (i.e. decoupled position embeddings). In particular, # the concat_and_cache_mla op requires @@ -311,6 +585,7 @@ def __init__( q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj, kv_b_proj=self.kv_b_proj, o_proj=self.o_proj, + indexer=self.indexer, ) self.mla_attn = Attention( @@ -321,7 +596,7 @@ def __init__( kv_cache_dtype=cache_config, layer_num=layer_num, use_mla=True, - mla_modules=mla_modules + mla_modules=mla_modules, ) self.prefix = prefix @@ -339,6 +614,9 @@ def forward( kv_c, k_pe = torch.split(self.kv_a_proj_with_mqa(hidden_states), [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) kv_c_normed = self.kv_a_layernorm(kv_c) + if self.is_v32 and self.indexer is not None: + _topk_indices = self.indexer(hidden_states, hidden_states_or_q_c, positions, self.rotary_emb) + return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, @@ -351,6 +629,7 @@ def __init__( self, config: PretrainedConfig, prefix: str, + topk_indices_buffer: Optional[torch.Tensor] = None, cache_config: str = "bf16", quant_config: Optional[QuantizationConfig] = None, layer_num: int = 0, @@ -383,6 +662,7 @@ def __init__( quant_config=quant_config, prefix=f"{prefix}.self_attn", layer_num=layer_num, + topk_indices_buffer=topk_indices_buffer, ) if (config.n_routed_experts is not None @@ -467,6 +747,17 @@ def __init__( self.config = config self.vocab_size = config.vocab_size + self.is_v32 = hasattr( + config, "index_topk" + ) + if self.is_v32: + topk_tokens = config.index_topk + topk_indices_buffer = torch.empty(atom_config.max_num_batched_tokens, + topk_tokens, + dtype=torch.int32, + device="cuda") + else: + topk_indices_buffer = None if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( @@ -481,6 +772,7 @@ def __init__( lambda prefix, layer_num=None: DeepseekV2DecoderLayer( config, prefix, + topk_indices_buffer=topk_indices_buffer, cache_config=cache_config, quant_config=quant_config, layer_num=layer_num, diff --git a/atom/utils/forward_context.py b/atom/utils/forward_context.py index 0ad9841b74..2c0bf4b237 100644 --- a/atom/utils/forward_context.py +++ b/atom/utils/forward_context.py @@ -46,6 +46,9 @@ class AttentionMetaData: kv_indptr: Optional[torch.Tensor] = None kv_indices: Optional[torch.Tensor] = None kv_last_page_lens: Optional[torch.Tensor] = None + cu_seqlen_ks: Optional[torch.Tensor] = None + cu_seqlen_ke: Optional[torch.Tensor] = None + sparse_kv_indptr: Optional[torch.Tensor] = None def __init__( self, @@ -62,6 +65,9 @@ def __init__( kv_indptr: Optional[torch.Tensor] = None, kv_indices: Optional[torch.Tensor] = None, kv_last_page_lens: Optional[torch.Tensor] = None, + cu_seqlen_ks: Optional[torch.Tensor] = None, + cu_seqlen_ke: Optional[torch.Tensor] = None, + sparse_kv_indptr: Optional[torch.Tensor] = None, ): self.cu_seqlens_q = cu_seqlens_q self.cu_seqlens_k = cu_seqlens_k @@ -76,6 +82,9 @@ def __init__( self.kv_indptr = kv_indptr self.kv_indices = kv_indices self.kv_last_page_lens = kv_last_page_lens + self.cu_seqlen_ks = cu_seqlen_ks + self.cu_seqlen_ke = cu_seqlen_ke + self.sparse_kv_indptr = sparse_kv_indptr def asdict_zerocopy(self, skip_fields: Optional[Set[str]] = None) -> Dict[str, Any]: """Similar to dataclasses.asdict, but avoids deepcopying.""" From a556796fe273a88b75ea14e544224353f6f0418e Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Wed, 5 Nov 2025 18:10:40 +0800 Subject: [PATCH 103/119] Profile (#26) * add online profile and more performance statistics * fix and reformat * refactor * rename and add readme * add warm up for now to skip capturing cuda graph while profiling offline --- README.md | 30 +++- atom/bench/benchmark_throughput.py | 119 ++++++++++--- atom/bench/one_batch.py | 60 ------- atom/entrypoints/openai_server.py | 60 ++++--- atom/examples/profile_offline.py | 95 +++++++++++ atom/examples/profile_online.py | 159 ++++++++++++++++++ .../simple_inference.py} | 16 -- atom/model_engine/engine_core.py | 10 +- atom/model_engine/engine_core_mgr.py | 7 + atom/model_engine/llm_engine.py | 23 ++- atom/model_engine/scheduler.py | 4 + atom/model_engine/sequence.py | 1 + 12 files changed, 460 insertions(+), 124 deletions(-) delete mode 100644 atom/bench/one_batch.py create mode 100644 atom/examples/profile_offline.py create mode 100644 atom/examples/profile_online.py rename atom/{bench/example.py => examples/simple_inference.py} (64%) diff --git a/README.md b/README.md index 64e363f61e..a2a2a9de79 100644 --- a/README.md +++ b/README.md @@ -55,18 +55,40 @@ pip install . The default optimization level is 3 (running with torch compile). Supported models include **Qwen**, **Llama**, and **Mixtral**. ```bash -python -m atom.bench.example --model meta-llama/Meta-Llama-3-8B +python -m atom.examples.simple_inference --model meta-llama/Meta-Llama-3-8B ``` > **Note:** First-time execution may take approximately 10 minutes for model compilation. -### Performance Benchmarking +### Performance profiling -Run performance tests to compare ATOM against vLLM: +Profile offline inference +```bash +python -m atom.examples.profile_offline --model Qwen/Qwen3-0.6B +``` +Or profile offline with custom input length +```bash +python -m atom.examples.profile_offline --model Qwen/Qwen3-0.6B --random-input --input-length 1024 --output-length 32 +``` + +Profile online inference, after starting the server +```bash +python -m atom.examples.profile_online +``` +Or profile online with custom input length +```bash +python -m atom.examples.profile_online --model Qwen/Qwen3-0.6B --random-input --input-length 1024 --output-length 32 +``` +Or directly send start profile and stop profile reuqest ```bash -python -m atom.bench.one_batch --model Qwen/Qwen3-0.6B +curl -s -S -X POST http://127.0.0.1:8000/start_profile ``` +```bash +curl -s -S -X POST http://127.0.0.1:8000/stop_profile +``` + +### Performance Benchmarking Run online throughput benchmark: diff --git a/atom/bench/benchmark_throughput.py b/atom/bench/benchmark_throughput.py index e6552de065..e8f18fa194 100644 --- a/atom/bench/benchmark_throughput.py +++ b/atom/bench/benchmark_throughput.py @@ -2,6 +2,8 @@ import aiohttp import time from typing import List +import random +from transformers import AutoTokenizer async def send_chat_request(session: aiohttp.ClientSession, url: str, message: str, request_id: int, max_tokens: int = 100): @@ -24,11 +26,21 @@ async def send_chat_request(session: aiohttp.ClientSession, url: str, message: s if response.status == 200: content = result["choices"][0]["message"]["content"] usage = result["usage"] + ttft = usage.get("ttft_s", 0.0) + tpot = usage.get("tpot_s", 0.0) print(f"βœ“ Request {request_id} completed in {latency:.2f}s") print(f" Prompt: {message[:50]}...") print(f" Response: {content[:100]}...") print(f" Tokens: {usage['prompt_tokens']} input, {usage['completion_tokens']} output") - return {"success": True, "latency": latency, "request_id": request_id} + print(f" TTFT: {ttft:.3f}s, TPOT: {tpot:.3f}s") + return { + "success": True, + "latency": latency, + "request_id": request_id, + "ttft": ttft, + "tpot": tpot, + "num_tokens": usage['completion_tokens'] + } else: print(f"βœ— Request {request_id} failed: {response.status}") return {"success": False, "latency": latency, "request_id": request_id} @@ -57,11 +69,21 @@ async def send_completion_request(session: aiohttp.ClientSession, url: str, prom if response.status == 200: text = result["choices"][0]["text"] usage = result["usage"] + ttft = usage.get("ttft_s", 0.0) + tpot = usage.get("tpot_s", 0.0) print(f"βœ“ Request {request_id} completed in {latency:.2f}s") print(f" Prompt: {prompt[:50]}...") print(f" Response: {text[:100]}...") print(f" Tokens: {usage['prompt_tokens']} input, {usage['completion_tokens']} output") - return {"success": True, "latency": latency, "request_id": request_id} + print(f" TTFT: {ttft:.3f}s, TPOT: {tpot:.3f}s") + return { + "success": True, + "latency": latency, + "request_id": request_id, + "ttft": ttft, + "tpot": tpot, + "num_tokens": usage['completion_tokens'] + } else: print(f"βœ— Request {request_id} failed: {response.status}") return {"success": False, "latency": latency, "request_id": request_id} @@ -72,25 +94,19 @@ async def send_completion_request(session: aiohttp.ClientSession, url: str, prom return {"success": False, "latency": latency, "request_id": request_id, "error": str(e)} -def generate_prompt_with_length(target_tokens: int) -> str: - words = ["hello", "world", "this", "is", "a", "test", "prompt", "for", "the", "language", - "model", "testing", "please", "generate", "response", "based", "on", "input"] - - result = [] - tokens_count = 0 - idx = 0 - - while tokens_count < target_tokens: - word = words[idx % len(words)] - result.append(word) - tokens_count += 1 - idx += 1 - - return " ".join(result) +def generate_random_prompt(tokenizer: AutoTokenizer, input_length: int) -> str: + """Generate random prompt with exact token length""" + vocab_size = tokenizer.vocab_size + random_token_ids = [random.randint(0, vocab_size - 1) for _ in range(input_length)] + prompt = tokenizer.decode(random_token_ids, skip_special_tokens=True) + # Re-encode to verify and truncate to exact length + token_ids = tokenizer.encode(prompt, add_special_tokens=False)[:input_length] + prompt = tokenizer.decode(token_ids, skip_special_tokens=True) + return prompt -async def test_concurrent_requests(base_url: str, num_requests: int = 60, request_rate: float = 2.0, - concurrency: int = 4, use_chat: bool = True, +async def test_concurrent_requests(base_url: str, tokenizer: AutoTokenizer, num_requests: int = 60, + request_rate: float = 2.0, concurrency: int = 4, use_chat: bool = True, input_tokens: int = 20, output_tokens: int = 100): estimated_duration = num_requests / request_rate @@ -120,7 +136,7 @@ async def test_concurrent_requests(base_url: str, num_requests: int = 60, reques while sent_count < num_requests or active_tasks: if sent_count < num_requests and len(active_tasks) < concurrency: sent_count += 1 - prompt = generate_prompt_with_length(input_tokens) + prompt = generate_random_prompt(tokenizer, input_tokens) if use_chat: task = asyncio.create_task( @@ -153,7 +169,10 @@ async def test_concurrent_requests(base_url: str, num_requests: int = 60, reques successful = sum(1 for r in results if r["success"]) failed = total_requests - successful - if results: + # Calculate statistics for successful requests + successful_results = [r for r in results if r["success"]] + + if successful_results: avg_latency = sum(r["latency"] for r in results) / total_requests min_latency = min(r["latency"] for r in results) max_latency = max(r["latency"] for r in results) @@ -163,9 +182,43 @@ async def test_concurrent_requests(base_url: str, num_requests: int = 60, reques p90 = latencies[int(len(latencies) * 0.90)] p95 = latencies[int(len(latencies) * 0.95)] p99 = latencies[int(len(latencies) * 0.99)] + + # Calculate TTFT and TPOT statistics + ttfts = [r.get("ttft", 0) for r in successful_results if r.get("ttft", 0) > 0] + tpots = [r.get("tpot", 0) for r in successful_results if r.get("tpot", 0) > 0] + + if ttfts: + avg_ttft = sum(ttfts) / len(ttfts) + min_ttft = min(ttfts) + max_ttft = max(ttfts) + ttfts_sorted = sorted(ttfts) + p50_ttft = ttfts_sorted[int(len(ttfts_sorted) * 0.50)] + p90_ttft = ttfts_sorted[int(len(ttfts_sorted) * 0.90)] + p95_ttft = ttfts_sorted[int(len(ttfts_sorted) * 0.95)] + p99_ttft = ttfts_sorted[int(len(ttfts_sorted) * 0.99)] + else: + avg_ttft = min_ttft = max_ttft = 0 + p50_ttft = p90_ttft = p95_ttft = p99_ttft = 0 + + if tpots: + avg_tpot = sum(tpots) / len(tpots) + min_tpot = min(tpots) + max_tpot = max(tpots) + tpots_sorted = sorted(tpots) + p50_tpot = tpots_sorted[int(len(tpots_sorted) * 0.50)] + p90_tpot = tpots_sorted[int(len(tpots_sorted) * 0.90)] + p95_tpot = tpots_sorted[int(len(tpots_sorted) * 0.95)] + p99_tpot = tpots_sorted[int(len(tpots_sorted) * 0.99)] + else: + avg_tpot = min_tpot = max_tpot = 0 + p50_tpot = p90_tpot = p95_tpot = p99_tpot = 0 else: avg_latency = min_latency = max_latency = 0 p50 = p90 = p95 = p99 = 0 + avg_ttft = min_ttft = max_ttft = 0 + p50_ttft = p90_ttft = p95_ttft = p99_ttft = 0 + avg_tpot = min_tpot = max_tpot = 0 + p50_tpot = p90_tpot = p95_tpot = p99_tpot = 0 print(f"\n{'='*60}") print(f"Test Completed!") @@ -181,6 +234,22 @@ async def test_concurrent_requests(base_url: str, num_requests: int = 60, reques print(f" P90: {p90:.2f}s") print(f" P95: {p95:.2f}s") print(f" P99: {p99:.2f}s") + print(f"\nTTFT (Time To First Token) Statistics:") + print(f" Average: {avg_ttft:.3f}s") + print(f" Min: {min_ttft:.3f}s") + print(f" Max: {max_ttft:.3f}s") + print(f" P50: {p50_ttft:.3f}s") + print(f" P90: {p90_ttft:.3f}s") + print(f" P95: {p95_ttft:.3f}s") + print(f" P99: {p99_ttft:.3f}s") + print(f"\nTPOT (Time Per Output Token) Statistics:") + print(f" Average: {avg_tpot:.3f}s") + print(f" Min: {min_tpot:.3f}s") + print(f" Max: {max_tpot:.3f}s") + print(f" P50: {p50_tpot:.3f}s") + print(f" P90: {p90_tpot:.3f}s") + print(f" P95: {p95_tpot:.3f}s") + print(f" P99: {p99_tpot:.3f}s") print(f"\nThroughput:") print(f" Actual Request Rate: {total_requests/total_time:.2f} req/s") print(f" Token Throughput: {total_requests*(input_tokens+output_tokens)/total_time:.2f} tokens/s") @@ -192,18 +261,24 @@ async def main(): parser = argparse.ArgumentParser(description="OpenAI API Server Load Testing") parser.add_argument("--url", type=str, default="http://localhost:8000", help="Server URL") + parser.add_argument("--model", type=str, required=True, help="Model name or path for tokenizer") parser.add_argument("--num-requests", "-n", type=int, default=60, help="Total number of requests") parser.add_argument("--request-rate", "-r", type=float, default=2.0, help="Request rate (requests/second)") parser.add_argument("--concurrency", "-c", type=int, default=4, help="Max concurrent requests") parser.add_argument("--type", choices=["chat", "completion"], default="chat", help="Request type") - parser.add_argument("--input-tokens", "-i", type=int, default=1016, help="Input token count (approximate)") + parser.add_argument("--input-tokens", "-i", type=int, default=1024, help="Input token count") parser.add_argument("--output-tokens", "-o", type=int, default=1024, help="Output token count (max_tokens)") args = parser.parse_args() + # Initialize tokenizer + print("Loading tokenizer...") + tokenizer = AutoTokenizer.from_pretrained(args.model) + use_chat = args.type == "chat" await test_concurrent_requests( base_url=args.url, + tokenizer=tokenizer, num_requests=args.num_requests, request_rate=args.request_rate, concurrency=args.concurrency, diff --git a/atom/bench/one_batch.py b/atom/bench/one_batch.py deleted file mode 100644 index c3fcd54c18..0000000000 --- a/atom/bench/one_batch.py +++ /dev/null @@ -1,60 +0,0 @@ -import argparse -import time -from random import randint, seed - -from transformers import AutoTokenizer - -from atom import SamplingParams -from atom.model_engine.arg_utils import EngineArgs - -parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, - description="config of test", -) - -# Add engine arguments -EngineArgs.add_cli_args(parser) - -# A very long prompt, total number of tokens is about 15k. -LONG_PROMPT = [ - "Hello, my name is", - "The president of the United States is", - "The capital of France is", - "The future of AI is", -] * 64 - - -def main(): - seed(0) - num_seqs = 256 - max_input_len = 1024 - max_ouput_len = 1024 - - args = parser.parse_args() - - # Create engine from args - engine_args = EngineArgs.from_cli_args(args) - llm = engine_args.create_engine() - - sampling_params = [ - SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=16) - for _ in range(num_seqs) - ] - - print("------start generating------") - - # warm up - outputs_ = llm.generate(["Benchmark: "], SamplingParams()) - t = time.time() - outputs = llm.generate(LONG_PROMPT, sampling_params) - t = time.time() - t - total_tokens = sum(sp.max_tokens for sp in sampling_params) - throughput = total_tokens / t - - print( - f"Total: {total_tokens}tok, Time: {t:.2f}s, Throughput: {throughput:.2f}tok/s" - ) - - -if __name__ == "__main__": - main() diff --git a/atom/entrypoints/openai_server.py b/atom/entrypoints/openai_server.py index a56fbe0170..de582f44f6 100644 --- a/atom/entrypoints/openai_server.py +++ b/atom/entrypoints/openai_server.py @@ -13,7 +13,6 @@ from atom.model_engine.arg_utils import EngineArgs -# OpenAI API Models class ChatMessage(BaseModel): role: str content: str @@ -31,7 +30,7 @@ class ChatCompletionRequest(BaseModel): class CompletionRequest(BaseModel): model: str - prompt: Union[str, List[str]] # Support both single string and list of strings + prompt: Union[str, List[str]] temperature: Optional[float] = 1.0 top_p: Optional[float] = 1.0 max_tokens: Optional[int] = 256 @@ -45,7 +44,7 @@ class ChatCompletionResponse(BaseModel): created: int model: str choices: List[Dict[str, Any]] - usage: Dict[str, int] + usage: Dict[str, Any] class CompletionResponse(BaseModel): @@ -54,7 +53,7 @@ class CompletionResponse(BaseModel): created: int model: str choices: List[Dict[str, Any]] - usage: Dict[str, int] + usage: Dict[str, Any] # Global engine @@ -67,7 +66,6 @@ async def generate_async(prompt: str, sampling_params: SamplingParams) -> Dict[s """Async wrapper for engine generation.""" global engine - # Use the new async generate method that handles each request independently async for output in engine.generate_async(prompt, sampling_params): return output @@ -81,14 +79,12 @@ async def chat_completions(request: ChatCompletionRequest): global engine, tokenizer, model_name try: - # Convert messages to prompt using chat template prompt = tokenizer.apply_chat_template( [{"role": msg.role, "content": msg.content} for msg in request.messages], tokenize=False, add_generation_prompt=True, ) - # Create sampling params sampling_params = SamplingParams( temperature=request.temperature, max_tokens=request.max_tokens, @@ -96,10 +92,8 @@ async def chat_completions(request: ChatCompletionRequest): ignore_eos=request.ignore_eos, ) - # Generate response output = await generate_async(prompt, sampling_params) - # Format response response = ChatCompletionResponse( id=f"chatcmpl-{uuid.uuid4().hex}", created=int(time.time()), @@ -118,6 +112,9 @@ async def chat_completions(request: ChatCompletionRequest): "prompt_tokens": output["num_tokens_input"], "completion_tokens": output["num_tokens_output"], "total_tokens": output["num_tokens_input"] + output["num_tokens_output"], + "ttft_s": output.get("ttft", 0.0), + "tpot_s": output.get("tpot", 0.0), + "latency_s": output.get("latency", 0.0), }, ) @@ -133,7 +130,6 @@ async def completions(request: CompletionRequest): global engine, model_name try: - # Create sampling params sampling_params = SamplingParams( temperature=request.temperature, max_tokens=request.max_tokens, @@ -141,9 +137,7 @@ async def completions(request: CompletionRequest): ignore_eos=request.ignore_eos, ) - # Handle both single prompt and batch prompts if isinstance(request.prompt, str): - # Single prompt output = await generate_async(request.prompt, sampling_params) choices = [ { @@ -155,7 +149,6 @@ async def completions(request: CompletionRequest): total_prompt_tokens = output["num_tokens_input"] total_completion_tokens = output["num_tokens_output"] else: - # Batch prompts - process concurrently tasks = [ generate_async(prompt, sampling_params) for prompt in request.prompt @@ -173,7 +166,15 @@ async def completions(request: CompletionRequest): total_prompt_tokens = sum(output["num_tokens_input"] for output in outputs) total_completion_tokens = sum(output["num_tokens_output"] for output in outputs) - # Format response + if isinstance(request.prompt, str): + ttft = output.get("ttft", 0.0) + tpot = output.get("tpot", 0.0) + latency = output.get("latency", 0.0) + else: + ttft = sum(output.get("ttft", 0.0) for output in outputs) / len(outputs) if outputs else 0.0 + tpot = sum(output.get("tpot", 0.0) for output in outputs) / len(outputs) if outputs else 0.0 + latency = sum(output.get("latency", 0.0) for output in outputs) / len(outputs) if outputs else 0.0 + response = CompletionResponse( id=f"cmpl-{uuid.uuid4().hex}", created=int(time.time()), @@ -183,6 +184,9 @@ async def completions(request: CompletionRequest): "prompt_tokens": total_prompt_tokens, "completion_tokens": total_completion_tokens, "total_tokens": total_prompt_tokens + total_completion_tokens, + "ttft_s": ttft, + "tpot_s": tpot, + "latency_s": latency, }, ) @@ -215,15 +219,35 @@ async def health(): return {"status": "ok"} +@app.post("/start_profile") +async def start_profile(): + """Start profiling.""" + global engine + try: + engine.start_profile() + return {"status": "success", "message": "Profiling started"} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to start profiling: {str(e)}") + + +@app.post("/stop_profile") +async def stop_profile(): + """Stop profiling and generate trace files.""" + global engine + try: + engine.stop_profile() + return {"status": "success", "message": "Profiling stopped. Trace files generated."} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to stop profiling: {str(e)}") + + def main(): global engine, tokenizer, model_name parser = argparse.ArgumentParser(description="Atom OpenAI API Server") - # Add engine arguments EngineArgs.add_cli_args(parser) - # Add server-specific arguments parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host") parser.add_argument( "--server-port", type=int, default=8000, @@ -231,13 +255,11 @@ def main(): ) args = parser.parse_args() - - # Initialize tokenizer + print(f"Loading tokenizer from {args.model}...") tokenizer = AutoTokenizer.from_pretrained(args.model) model_name = args.model - # Initialize engine print(f"Initializing engine with model {args.model}...") engine_args = EngineArgs.from_cli_args(args) engine = engine_args.create_engine() diff --git a/atom/examples/profile_offline.py b/atom/examples/profile_offline.py new file mode 100644 index 0000000000..97dfcccb7a --- /dev/null +++ b/atom/examples/profile_offline.py @@ -0,0 +1,95 @@ +import argparse +import random +from transformers import AutoTokenizer + +from atom import LLMEngine, SamplingParams +from atom.model_engine.arg_utils import EngineArgs + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description="Offline profiling example for Atom LLM Engine", +) + +EngineArgs.add_cli_args(parser) + +parser.add_argument( + "--input-length", + type=int, + default=128, + help="Input prompt length in tokens (approximate, used with --random-input)", +) +parser.add_argument( + "--output-length", + type=int, + default=32, + help="Output generation length in tokens", +) +parser.add_argument( + "--random-input", + action="store_true", + help="Use random repeated words as input. Otherwise use a predefined meaningful text.", +) + + +def main(): + args = parser.parse_args() + + # Set default torch_profiler_dir to current directory if not provided + if args.torch_profiler_dir is None: + args.torch_profiler_dir = "./profiler_traces" + print("Warning: --torch-profiler-dir not specified, using default: ./profiler_traces") + + print("\nInitializing LLM engine...") + engine_args = EngineArgs.from_cli_args(args) + llm = engine_args.create_engine() + + tokenizer = AutoTokenizer.from_pretrained(args.model) + + if args.random_input: + vocab_size = tokenizer.vocab_size + random_token_ids = [random.randint(0, vocab_size - 1) for _ in range(args.input_length)] + prompt = tokenizer.decode(random_token_ids, skip_special_tokens=True) + # Re-encode to verify and truncate to exact length + token_ids = tokenizer.encode(prompt, add_special_tokens=False)[:args.input_length] + prompt = tokenizer.decode(token_ids, skip_special_tokens=True) + print(f"Using randomly generated input with length: {len(token_ids)} tokens") + else: + prompt = """Artificial intelligence has revolutionized numerous fields in recent years, transforming the way we live, work, and interact with technology. Machine learning algorithms have become increasingly sophisticated, enabling computers to perform tasks that were once thought to require human intelligence. From natural language processing to computer vision, AI systems are now capable of understanding and generating human language, recognizing objects in images, and even creating original artwork and music. The applications of AI continue to expand rapidly across industries.""" + print(f"Input_prompt: {prompt}") + + sampling_params = SamplingParams( + temperature=0.6, + max_tokens=args.output_length, + ignore_eos=True, + ) + + # TODO: remove warm up and skip cuda capture process in engine while profiling. + print("Warming up...") + _ = llm.generate(["warmup"], sampling_params) + print("Warm up done") + + print("\n" + "=" * 70) + print("Starting profiling...") + print("=" * 70) + + llm.start_profile() + + outputs = llm.generate([prompt], sampling_params) + + llm.stop_profile() + + print("\n" + "=" * 70) + print("Profiling completed!") + print("Profiler traces have been saved to:", args.torch_profiler_dir) + print("=" * 70) + + if not args.random_input: + print("Generated Output:") + for output in outputs: + generated_text = output['text'] + print(f"Output: {generated_text}\n") + + +if __name__ == "__main__": + main() + diff --git a/atom/examples/profile_online.py b/atom/examples/profile_online.py new file mode 100644 index 0000000000..f9a94655c9 --- /dev/null +++ b/atom/examples/profile_online.py @@ -0,0 +1,159 @@ +import requests +import time +import argparse +import random +from transformers import AutoTokenizer + + +def start_profiling(base_url: str): + """Start profiling on the server""" + print("Starting profiler...") + response = requests.post(f"{base_url}/start_profile") + print(f"Response: {response.json()}") + return response.status_code == 200 + + +def stop_profiling(base_url: str): + """Stop profiling and generate trace files""" + print("Stopping profiler...") + response = requests.post(f"{base_url}/stop_profile") + print(f"Response: {response.json()}") + return response.status_code == 200 + + +def send_completion_request(base_url: str, prompt: str, max_tokens: int): + """Send a completion request to the server""" + print(f"Sending completion request with {len(prompt.split())} words, max_tokens={max_tokens}") + response = requests.post( + f"{base_url}/v1/completions", + json={ + "model": "test", + "prompt": prompt, + "max_tokens": max_tokens, + "temperature": 0.7, + } + ) + if response.status_code == 200: + return response.json() + return None + + +def send_chat_request(base_url: str, message: str, max_tokens: int): + """Send a chat completion request to the server""" + print(f"Sending chat request with {len(message.split())} words, max_tokens={max_tokens}") + response = requests.post( + f"{base_url}/v1/chat/completions", + json={ + "model": "test", + "messages": [ + {"role": "user", "content": message} + ], + "max_tokens": max_tokens, + "temperature": 0.7, + } + ) + return response.status_code == 200 + + +def generate_random_prompt(tokenizer: AutoTokenizer, input_length: int) -> str: + """Generate random prompt with exact token length""" + vocab_size = tokenizer.vocab_size + random_token_ids = [random.randint(0, vocab_size - 1) for _ in range(input_length)] + prompt = tokenizer.decode(random_token_ids, skip_special_tokens=True) + # Re-encode to verify and truncate to exact length + token_ids = tokenizer.encode(prompt, add_special_tokens=False)[:input_length] + prompt = tokenizer.decode(token_ids, skip_special_tokens=True) + return prompt + + +def main(): + parser = argparse.ArgumentParser(description="Profile online inference requests") + parser.add_argument( + "--input-length", + type=int, + default=128, + help="Input prompt length in tokens (used with --random-input)", + ) + parser.add_argument( + "--output-len", + type=int, + default=32, + help="Number of output tokens to generate (max_tokens) (default: 32)" + ) + parser.add_argument( + "--port", + type=int, + default=8000, + help="Server port (default: 8000)" + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Server host (default: localhost)" + ) + parser.add_argument( + "--random-input", + action="store_true", + help="Use random generated input. Otherwise use a predefined prompt.", + ) + parser.add_argument( + "--model", + type=str, + help="Model name or path for tokenizer (required when using --random-input)", + ) + + args = parser.parse_args() + + base_url = f"http://{args.host}:{args.port}" + + print("=" * 60) + print("Atom OpenAI Server Profiling Example") + print("=" * 60) + + # Prepare input prompt + if args.random_input: + if not args.model: + print("Error: --model is required when using --random-input") + return + print("Loading tokenizer...") + tokenizer = AutoTokenizer.from_pretrained(args.model) + input_prompt = generate_random_prompt(tokenizer, args.input_length) + print(f"Using randomly generated input with length: {args.input_length} tokens") + else: + input_prompt = "hello, who are you?" + print(f"Using predefined prompt: {input_prompt}") + + # Start profiling + if not start_profiling(base_url): + print("Failed to start profiling!") + return + + time.sleep(2) + + result = send_completion_request(base_url, input_prompt, args.output_len) + if result is None: + print("Warning: Request failed or returned no result") + + time.sleep(2) + + # Stop profiling + print("\n" + "=" * 60) + if not stop_profiling(base_url): + print("Failed to stop profiling!") + return + + # Print output results for non-random input + if not args.random_input and result: + print("\n" + "=" * 60) + print("Generated Output:") + print("=" * 60) + if "choices" in result and len(result["choices"]) > 0: + generated_text = result["choices"][0]["text"] + print(f"Output: {generated_text}\n") + print("=" * 60) + + +if __name__ == "__main__": + main() + diff --git a/atom/bench/example.py b/atom/examples/simple_inference.py similarity index 64% rename from atom/bench/example.py rename to atom/examples/simple_inference.py index 44758b3669..6e2d9dc45e 100644 --- a/atom/bench/example.py +++ b/atom/examples/simple_inference.py @@ -47,22 +47,6 @@ def main(): ] print("This is prompts:", prompts) - # # Calculate total token number of prompts - # prompt_token_ids = [llm.tokenizer.encode(prompt) for prompt in prompts] - # prompt_lens = [len(token_ids) for token_ids in prompt_token_ids] - # # Create warmup inputs with same shapes as all prompts - # # Generate random token IDs for each prompt length to match expected input shapes - # warmup_prompts = [] - # for i, prompt_len in enumerate(prompt_lens): - # warmup_prompt = torch.randint( - # 0, llm.tokenizer.vocab_size, size=(prompt_len,) - # ).tolist() - # warmup_prompts.append(warmup_prompt) - # # Run warmup with the same batch structure as the actual prompts (no profiling) - # _ = llm.generate(warmup_prompts, sampling_params) - - # generate (with profiling) - # outputs_ = llm.generate(["Benchmark: "], SamplingParams()) outputs = llm.generate(prompts, sampling_params) for prompt, output in zip(prompts, outputs): diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index c2ff597e65..ade23a6530 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -117,9 +117,7 @@ def run_engine(config: Config, input_address: str, output_address: str): engine: EngineCore = None try: engine = EngineCore(config, input_address, output_address) - engine.start_profiler() engine.busy_loop() - engine.stop_profiler() except Exception as e: logger.error(f"run_engine: exception: {e}", exc_info=True) raise e @@ -181,6 +179,14 @@ def process_input_sockets(self, input_address: str): req_ids = [req.id for req in reqs] logger.debug(f"{self.label}: input get {request_type} {req_ids}") self.input_queue.put_nowait(reqs) + elif request_type == EngineCoreRequestType.UTILITY: + # Handle utility commands like start_profile/stop_profile + cmd = reqs.get("cmd") if isinstance(reqs, dict) else None + logger.debug(f"{self.label}: input get UTILITY command: {cmd}") + if cmd == "start_profile": + self.start_profiler() + elif cmd == "stop_profile": + self.stop_profiler() elif request_type == EngineCoreRequestType.SHUTDOWN: logger.debug(f"{self.label}: input get {request_type}") self.input_queue.put_nowait([get_exit_sequence()]) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 1cb7e35d72..2975035d53 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -108,6 +108,13 @@ def add_request(self, seqs: List[Sequence]): [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.ADD, seqs))], copy=False, ) + + def send_utility_command(self, cmd: str): + logger.debug(f"{self.label}: Send utility command: {cmd}") + self.input_socket.send_multipart( + [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.UTILITY, {"cmd": cmd}))], + copy=False, + ) def _shutdown_engine_core(self): if self.engine_core_process is not None and self.engine_core_process.is_alive(): diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index bbc2d093cd..36f8bdb3c1 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -126,6 +126,14 @@ async def generate_async( # Let other tasks run before trying again await asyncio.sleep(0) + + def start_profile(self): + self.core_mgr.send_utility_command("start_profile") + logger.info("Profiling started") + + def stop_profile(self): + self.core_mgr.send_utility_command("stop_profile") + logger.info("Profiling stopped. Trace files should be generated.") class InputOutputProcessor: @@ -173,10 +181,21 @@ def postprocess(self, reqs: List[Sequence]): self.requests.pop(req.id) output_str = self.tokenizer.decode(req.completion_token_ids) req.leave_time = time.time() + + # Calculate TTFT (Time To First Token) and TPOT (Time Per Output Token) + ttft = 0.0 + tpot = 0.0 + if req.first_token_time > 0: + ttft = req.first_token_time - req.arrive_time + # Calculate TPOT only if there are multiple output tokens + if req.num_completion_tokens > 1: + tpot = (req.leave_time - req.first_token_time) / (req.num_completion_tokens - 1) + print( f"Request {req.id} finished with reason {req.leave_reason}. " f"Input tokens: {req.num_prompt_tokens}, output tokens: {req.num_completion_tokens}, " - f"latency: {req.leave_time - req.arrive_time:.2f}s" + f"latency: {req.leave_time - req.arrive_time:.2f}s, " + f"TTFT: {ttft:.3f}s, TPOT: {tpot:.3f}s" ) outputs[req.id] = { "text": output_str, @@ -185,6 +204,8 @@ def postprocess(self, reqs: List[Sequence]): "finish_reason": req.leave_reason, "num_tokens_input": req.num_prompt_tokens, "num_tokens_output": req.num_completion_tokens, + "ttft": ttft, # Time to first token in seconds + "tpot": tpot, # Time per output token in seconds } return outputs diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index ad6bdfc5c1..ffd3fb1fca 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -1,4 +1,5 @@ import logging +import time from collections import deque from dataclasses import dataclass, field from typing import Any, Optional, cast @@ -161,6 +162,9 @@ def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): else: seq.append_token(token_id) + if seq.num_completion_tokens == 1 and seq.first_token_time == 0.0: + seq.first_token_time = time.time() + leave_reason = None # Check if sequence ends with any stop sequence for stop_seq in seq.stop_token_sequences: diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 64ab59b5ab..1949554050 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -48,6 +48,7 @@ def __init__( # statistics fields self.arrive_time = 0.0 + self.first_token_time = 0.0 self.leave_time = 0.0 self.leave_reason = "" From 19d5a2cd57d891d1fe5d63d2e346ba6de43aef8f Mon Sep 17 00:00:00 2001 From: ZhangLirong Date: Fri, 7 Nov 2025 16:14:10 +0800 Subject: [PATCH 104/119] Fix some bug in atom (#28) * fix some bug in atom * make cudagraph bs default in config --- atom/model_engine/arg_utils.py | 6 +++--- atom/model_engine/model_runner.py | 17 ++--------------- atom/model_ops/base_attention.py | 2 +- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/atom/model_engine/arg_utils.py b/atom/model_engine/arg_utils.py index 0c065a502d..afe5d2ab2b 100644 --- a/atom/model_engine/arg_utils.py +++ b/atom/model_engine/arg_utils.py @@ -102,8 +102,8 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: parser.add_argument( "--cudagraph-capture-sizes", type=str, - default="[1,2,4,8,16]", - help="Sizes to capture cudagraph.", + default=None, + help="Sizes to capture cudagraph. Example: [1,2,4,8,16]", ) parser.add_argument( @@ -164,7 +164,7 @@ def create_engine(self) -> LLMEngine: torch_profiler_dir=self.torch_profiler_dir, compilation_config=CompilationConfig( level=self.level, - cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes), + cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes) if self.cudagraph_capture_sizes else None, ), ) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 888ee80bad..aafc49a327 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -610,20 +610,6 @@ def prepare_block_tables(self, seqs: list[Sequence]): def prepare_prefill(self, batch: ScheduledBatch): return self.attn_metadata_builder.prepare_prefill(batch, self.forward_vars) - def prepare_decode(self, batch: ScheduledBatch): - scheduled_bs = batch.total_seqs_num_decode - seqs = list(batch.seqs.values()) - seqs = seqs[batch.total_seqs_num_prefill :] - assert len(seqs) == scheduled_bs - bs = ( - scheduled_bs - if self.enforce_eager - else next(x for x in self.graph_bs if x >= scheduled_bs) - ) - assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" - - return self.attn_metadata_builder.prepare_decode(batch, bs, self.forward_vars) - def prepare_intputs(self, batch: ScheduledBatch): is_prefill = batch.total_tokens_num_prefill > 0 bs = batch.total_seqs_num @@ -638,7 +624,8 @@ def prepare_intputs(self, batch: ScheduledBatch): bs = ( scheduled_bs if self.enforce_eager - else next(x for x in self.graph_bs if x >= scheduled_bs) + else next((x for x in self.graph_bs if x >= scheduled_bs), scheduled_bs) + # Use cudagraph and padding to batch_size, if bs > graph_bs, use eager mode ) assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" self.forward_vars["cu_seqlens_q"].np[scheduled_bs + 1 : bs + 1] = self.forward_vars["cu_seqlens_q"].np[scheduled_bs] diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py index 015be5bbbc..052d1b1648 100644 --- a/atom/model_ops/base_attention.py +++ b/atom/model_ops/base_attention.py @@ -68,7 +68,7 @@ def __init__( self.use_mla = use_mla self.base_attention = None self.kv_cache = torch.tensor([]) - self.indexer = mla_modules.indexer + self.indexer = mla_modules.indexer if mla_modules is not None else None atom_config = get_current_atom_config() block_size = atom_config.kv_cache_block_size From 524d043f00799adac6e73ae91b66a522a8650d3f Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Fri, 7 Nov 2025 21:29:39 +0800 Subject: [PATCH 105/119] use aiter gemm_a8w8_blockscale_bpreshuffle (#29) --- atom/model_ops/linear.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index b66b5787ad..211953e442 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -8,7 +8,7 @@ dtypes, gemm_a4w4, gemm_a8w8, - gemm_a8w8_blockscale, + gemm_a8w8_blockscale_bpreshuffle, gemm_a8w8_bpreshuffle, get_hip_quant, get_triton_quant, @@ -168,7 +168,7 @@ def process_weights_after_loading(self): ) if ( self.quant_type == QuantType.per_Token and self.params_dtype == dtypes.fp8 - ) or (self.quant_type == QuantType.per_1x32): + ) or (self.quant_type in [QuantType.per_1x32, QuantType.per_1x128]): self.weight.data = shuffle_weight(self.weight.data, (16, 16)) def forward( @@ -179,6 +179,8 @@ def forward( else: if x_scale is None: quant_func = get_hip_quant(self.quant_type) + if self.quant_type.value == QuantType.per_1x128.value: + quant_func = functools_partial(quant_func, transpose_scale=True) x, x_scale = quant_func( x, quant_dtype=self.params_dtype, @@ -213,7 +215,7 @@ def forward( dtype=otype, ) elif self.quant_type.value == QuantType.per_1x128.value: - y = gemm_a8w8_blockscale( + y = gemm_a8w8_blockscale_bpreshuffle( x, self.weight, x_scale, self.weight_scale, dtype=otype ) if self.bias is not None: From ba3339d7f148f527cd1f29008e144094dd04db86 Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Sat, 8 Nov 2025 08:43:59 +0000 Subject: [PATCH 106/119] apply black format --- atom/examples/simple_inference.py | 18 ++++--- atom/model_engine/model_runner.py | 74 +++++++++++++++++++------- atom/model_ops/attentions/aiter_mla.py | 3 ++ 3 files changed, 67 insertions(+), 28 deletions(-) diff --git a/atom/examples/simple_inference.py b/atom/examples/simple_inference.py index 6e2d9dc45e..598436b5c2 100644 --- a/atom/examples/simple_inference.py +++ b/atom/examples/simple_inference.py @@ -21,21 +21,23 @@ def main(): + prompts = [ + "introduce yourself", + "list all prime numbers within 100", + "1+2+3=?", + "ε¦‚δ½•εœ¨δΈ€δΈͺζœˆε†…ε’žθ‚Œ10ε…¬ζ–€", + ] args = parser.parse_args() - + args.cudagraph_capture_sizes = str([1, len(prompts)]) + # Create engine from args engine_args = EngineArgs.from_cli_args(args) llm = engine_args.create_engine() - + tokenizer = AutoTokenizer.from_pretrained(args.model) sampling_params = SamplingParams(temperature=args.temperature, max_tokens=256) - prompts = [ - "introduce yourself", - "list all prime numbers within 100", - "1+2+3=?", - "ε¦‚δ½•εœ¨δΈ€δΈͺζœˆε†…ε’žθ‚Œ10ε…¬ζ–€", - ] + prompts = [ tokenizer.apply_chat_template( [{"role": "user", "content": prompt}], diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index aafc49a327..ce0324979a 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -22,7 +22,14 @@ from atom.utils.selector import get_attn_backend logger = logging.getLogger("atom") -from atom.utils.forward_context import AttentionMetaData, Context, get_forward_context, reset_forward_context, set_forward_context, set_kv_cache_data +from atom.utils.forward_context import ( + AttentionMetaData, + Context, + get_forward_context, + reset_forward_context, + set_forward_context, + set_kv_cache_data, +) suppot_model_arch_dict = { "Qwen3ForCausalLM": Qwen3ForCausalLM, @@ -31,9 +38,9 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, "DeepseekV32ForCausalLM": DeepseekV2ForCausalLM, } -# seed=1 -# np.random.seed(seed) -# torch.cuda.manual_seed_all(seed) +seed = 34567 +np.random.seed(seed) +torch.cuda.manual_seed_all(seed) class tokenIDProcessor: @@ -516,7 +523,7 @@ def allocate_kv_cache(self, num_kvcache_blocks): device="cuda", ) # Build KVCacheConfig - # lirong TODO: This is a simple solution to build KVCacheConfig, + # lirong TODO: This is a simple solution to build KVCacheConfig, # models with only one type of attention, but not support multi-type of attention models. # We need to support it by kv_cache_group in the future. kv_cache_tensors = [] @@ -547,7 +554,7 @@ def allocate_kv_cache(self, num_kvcache_blocks): k_scale = module.k_scale v_scale = module.v_scale - + # Store in KVCacheTensor kv_cache_tensor = KVCacheTensor( layer_num=layer_id, @@ -557,7 +564,7 @@ def allocate_kv_cache(self, num_kvcache_blocks): v_scale=v_scale, ) kv_cache_tensors.append(kv_cache_tensor) - + module.k_cache = k_cache module.v_cache = v_cache @@ -588,13 +595,16 @@ def allocate_kv_cache(self, num_kvcache_blocks): v_scale=None, ) kv_cache_tensors.append(kv_cache_tensor) - + module.kv_cache = kv_cache module.max_model_len = self.config.max_model_len layer_id += 1 - + # Store KVCacheConfig - kv_cache_data = {f"layer_{i}": kv_cache_tensor for i, kv_cache_tensor in enumerate(kv_cache_tensors)} + kv_cache_data = { + f"layer_{i}": kv_cache_tensor + for i, kv_cache_tensor in enumerate(kv_cache_tensors) + } # vllm use register_kv_caches to register kv_cache_data. We just set it to global here set_kv_cache_data(kv_cache_data) if torch.distributed.is_initialized(): @@ -627,13 +637,28 @@ def prepare_intputs(self, batch: ScheduledBatch): else next((x for x in self.graph_bs if x >= scheduled_bs), scheduled_bs) # Use cudagraph and padding to batch_size, if bs > graph_bs, use eager mode ) - assert bs >= scheduled_bs, f"current decode {scheduled_bs=} > max graph_bs{bs}" - self.forward_vars["cu_seqlens_q"].np[scheduled_bs + 1 : bs + 1] = self.forward_vars["cu_seqlens_q"].np[scheduled_bs] - attn_metadata, positions = self.attn_metadata_builder.build(batch, self.forward_vars, bs) - context_bs = batch.total_seqs_num_prefill if is_prefill else batch.total_seqs_num_decode + assert ( + bs >= scheduled_bs + ), f"current decode {scheduled_bs=} > max graph_bs{bs}" + self.forward_vars["cu_seqlens_q"].np[scheduled_bs + 1 : bs + 1] = ( + self.forward_vars["cu_seqlens_q"].np[scheduled_bs] + ) + attn_metadata, positions = self.attn_metadata_builder.build( + batch, self.forward_vars, bs + ) + context_bs = ( + batch.total_seqs_num_prefill if is_prefill else batch.total_seqs_num_decode + ) - context = Context(positions=positions, is_prefill=is_prefill, batch_size=context_bs, graph_bs=bs) - set_forward_context(attn_metadata=attn_metadata, atom_config=self.config, context=context) + context = Context( + positions=positions, + is_prefill=is_prefill, + batch_size=context_bs, + graph_bs=bs, + ) + set_forward_context( + attn_metadata=attn_metadata, atom_config=self.config, context=context + ) def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: temperatures = [seq.temperature for seq in batch.seqs.values()] @@ -644,10 +669,11 @@ def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: def prepare_model(self, batch: ScheduledBatch): total_tokens_num = batch.total_tokens_num - assert total_tokens_num > 0 + assert total_tokens_num > 0 input_ids = self.tokenID_processor.prepare_input_ids(batch) - # print(f"input_ids: {input_ids}") + # if self.rank == 0: + # print(f"input_ids: {input_ids}") self.prepare_intputs(batch) temperatures = self.prepare_sample(batch) @@ -729,8 +755,16 @@ def capture_cudagraph(self): capture_range.set_description(f"Capturing {bs=}") graph = torch.cuda.CUDAGraph() - attn_metadata, context = self.attn_metadata_builder.build_for_cudagraph_capture(forward_vars=self.forward_vars, bs=bs) - set_forward_context(attn_metadata=attn_metadata, atom_config=self.config, context=context) + attn_metadata, context = ( + self.attn_metadata_builder.build_for_cudagraph_capture( + forward_vars=self.forward_vars, bs=bs + ) + ) + set_forward_context( + attn_metadata=attn_metadata, + atom_config=self.config, + context=context, + ) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index 5c19f25d14..ce149adc85 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -105,6 +105,9 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): **ctx, ) positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) + # if str(positions.device) == "cuda:0": + # for el, var in ctx.items(): + # print(f"{el}: {var}") return attn_metadata, positions def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: From c8ab77afeb74d7d6b116a53eb9edcbbe6e0406e2 Mon Sep 17 00:00:00 2001 From: amd-ruitang3 <145657428+amd-ruitang3@users.noreply.github.com> Date: Sun, 9 Nov 2025 15:34:55 +0800 Subject: [PATCH 107/119] mla_qkv_proj_fused (#30) * mla_qkv_proj_fused * update * fix for torch compile --------- Co-authored-by: valarLip <340077269@qq.com> --- atom/model_ops/linear.py | 44 ++++++++++++++++++++++++ atom/models/deepseek_v2.py | 70 +++++++++++++++++++++++++------------- 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 211953e442..3bf7d4e526 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -427,3 +427,47 @@ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): start_idx = self.tp_rank * shard_size loaded_weight = loaded_weight.narrow(self.tp_dim, start_idx, shard_size) param.weight_loader_process(param_data, loaded_weight) + + +class MergedReplicatedLinear(ReplicatedLinear): + def __init__( + self, + input_size: int, + output_size: list[int], + bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + **kwargs, + ): + self.output_sizes = output_size + super().__init__( + input_size, + sum(output_size), # ? + bias=bias, + quant_config=quant_config, + ) + + def weight_loader( + self, + param: nn.Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[int] = None, + ): # ? + param_data = param.data + assert loaded_shard_id is not None + assert loaded_shard_id < len(self.output_sizes) + if param is getattr(self, "weight_scale", None) or param is getattr( + self, "input_scale", None + ): + if self.quant_type == QuantType.per_1x128: + shard_offset = ( + sum(self.output_sizes[:loaded_shard_id]) + 128 - 1 + ) // 128 + shard_size = (self.output_sizes[loaded_shard_id] + 128 - 1) // 128 + elif self.quant_type == QuantType.per_Tensor: + shard_offset = loaded_shard_id + shard_size = 1 + else: + shard_offset = sum(self.output_sizes[:loaded_shard_id]) + shard_size = self.output_sizes[loaded_shard_id] + param_data = param_data.narrow(0, shard_offset, shard_size) + param.weight_loader_process(param_data, loaded_weight) diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index c11cf2c798..c2200779f2 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -63,6 +63,7 @@ MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, + MergedReplicatedLinear ) from atom.model_ops.moe import FusedMoE from atom.model_ops.topK import ( @@ -496,11 +497,16 @@ def __init__( self.layer_num = layer_num if self.q_lora_rank is not None: - self.q_a_proj = ReplicatedLinear(self.hidden_size, - self.q_lora_rank, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.q_a_proj") + # self.q_a_proj = ReplicatedLinear(self.hidden_size, + # self.q_lora_rank, + # bias=False, + # quant_config=quant_config, + # prefix=f"{prefix}.q_a_proj") + self.fused_qkv_a_proj = MergedReplicatedLinear( + self.hidden_size, + [self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim], + bias=False, + quant_config=quant_config) self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps) self.q_b_proj = ColumnParallelLinear(q_lora_rank, @@ -517,12 +523,12 @@ def __init__( quant_config=quant_config, prefix=f"{prefix}.q_proj") - self.kv_a_proj_with_mqa = ReplicatedLinear( - self.hidden_size, - self.kv_lora_rank + self.qk_rope_head_dim, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.kv_a_proj_with_mqa") + self.kv_a_proj_with_mqa = ReplicatedLinear( + self.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.kv_a_proj_with_mqa") self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=config.rms_norm_eps) self.kv_b_proj = ColumnParallelLinear( @@ -551,7 +557,6 @@ def __init__( mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale - self.is_v32 = hasattr(config, "index_topk") if self.is_v32: @@ -607,12 +612,18 @@ def forward( hidden_states: torch.Tensor, ) -> torch.Tensor: if self.q_lora_rank is not None: - ckq = self.q_a_proj(hidden_states) - hidden_states_or_q_c = self.q_a_layernorm(ckq) + qkv_lora = self.fused_qkv_a_proj(hidden_states) + # ckq = self.q_a_proj(hidden_states) + q_c, kv_c, k_pe = torch.split( + qkv_lora, + [self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], + dim=-1, + ) + hidden_states_or_q_c = self.q_a_layernorm(q_c) else: hidden_states_or_q_c = hidden_states - kv_c, k_pe = torch.split(self.kv_a_proj_with_mqa(hidden_states), - [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) + kv_c, k_pe = torch.split(self.kv_a_proj_with_mqa(hidden_states), + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1) kv_c_normed = self.kv_a_layernorm(kv_c) if self.is_v32 and self.indexer is not None: _topk_indices = self.indexer(hidden_states, hidden_states_or_q_c, positions, self.rotary_emb) @@ -833,13 +844,12 @@ def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: class DeepseekV2ForCausalLM(nn.Module): - packed_modules_mapping = { - "q_proj": ("qkv_proj", "q"), - "k_proj": ("qkv_proj", "k"), - "v_proj": ("qkv_proj", "v"), - "gate_proj": ("gate_up_proj", 0), - "up_proj": ("gate_up_proj", 1), - } + # packed_modules_mapping = { + # "q_a_proj" : ("fused_qkv_a_proj", 0), + # "kv_a_proj_with_mqa": ("fused_qkv_a_proj", 1), + # "gate_proj": ("gate_up_proj", 0), + # "up_proj": ("gate_up_proj", 1), + # } def __init__( self, @@ -852,6 +862,20 @@ def __init__( quant_config = atom_config.quant_config self.config = config self.quant_config = quant_config + + if hasattr(config, 'q_lora_rank') and config.q_lora_rank is not None: + self.packed_modules_mapping = { + "q_a_proj": ("fused_qkv_a_proj", 0), + "kv_a_proj_with_mqa": ("fused_qkv_a_proj", 1), + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + else: + self.packed_modules_mapping = { + "gate_proj": ("gate_up_proj", 0), + "up_proj": ("gate_up_proj", 1), + } + self.model = DeepseekV2Model(atom_config=atom_config, prefix=maybe_prefix(prefix, "model"), layer_type = layer_type) From 8fd0e7341d26e5053943f66cef1c3a16362d79cd Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:30:34 +0800 Subject: [PATCH 108/119] use aiter persistent mla (#27) * use aiter persistent mla * fix * use get_mla_metadata_info_v1 --- atom/model_engine/model_runner.py | 20 ++-- atom/model_ops/attention_mla.py | 56 +++++------ atom/model_ops/attentions/aiter_attention.py | 4 +- atom/model_ops/attentions/aiter_mla.py | 99 ++++++++++++++++++-- atom/model_ops/attentions/backends.py | 3 +- atom/model_ops/base_attention.py | 2 +- atom/utils/forward_context.py | 19 ++++ 7 files changed, 150 insertions(+), 53 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index ce0324979a..188062415b 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -207,16 +207,10 @@ def __init__(self, rank: int, config: Config): self.enforce_eager = config.enforce_eager self.world_size = config.tensor_parallel_size self.rank = rank + self.device = torch.device(f"cuda:{rank}") self.label = f"Model Runner{rank}/{self.world_size}" self.hf_text_config = get_hf_text_config(hf_config) self.use_mla = self.is_deepseek_mla() - self.attn_backend = get_attn_backend( - self.block_size, - use_mla=self.use_mla, - ) - self.attn_metadata_builder = self.attn_backend.get_builder_cls()( - self.block_size - ) self.is_deepseek_v32 = ( hasattr(hf_config, "index_topk") if self.use_mla else False ) @@ -231,9 +225,7 @@ def __init__(self, rank: int, config: Config): self.graph_bs = [0] # for eager fallback - device = torch.device(f"cuda:{rank}") - torch.cuda.set_device(device) - self.device = device + torch.cuda.set_device(self.device) os.environ["MASTER_ADDR"] = self.config.master_addr os.environ["MASTER_PORT"] = str(self.config.port) init_dist_env(self.world_size, rankID=rank) @@ -241,6 +233,14 @@ def __init__(self, rank: int, config: Config): default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) torch.set_default_device("cuda") + self.attn_backend = get_attn_backend( + self.block_size, + use_mla=self.use_mla, + ) + self.attn_metadata_builder = self.attn_backend.get_builder_cls()( + self.block_size, + self.device, + ) self.tokenID_processor = tokenIDProcessor( self.config.max_num_batched_tokens, self.device ) diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index bdf3a3e865..ff082fc870 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -14,7 +14,11 @@ from tqdm import tqdm from atom.model_ops.utils import get_and_maybe_dequant_weights -from atom.utils.forward_context import AttentionMetaData, ForwardContext, get_forward_context +from atom.utils.forward_context import ( + AttentionMetaData, + ForwardContext, + get_forward_context, +) from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as _aiter_triton_fp8_bmm, @@ -83,7 +87,8 @@ def __init__( self.o_proj = mla_modules.o_proj self.kv_b_proj = mla_modules.kv_b_proj self.kv_cache = torch.tensor([]) - self._k_scale = torch.tensor(1.0, dtype=torch.float32) + self.one_scale = torch.tensor(1.0, dtype=torch.float32) + self._k_scale = self.one_scale self.topk_indices_buffer = ( mla_modules.indexer.topk_indices_buffer if mla_modules.indexer is not None @@ -159,32 +164,6 @@ def process_weights_after_loading(self): # Convert from (L, N, P) to (N, P, L) self.W_UK_T = W_UK.permute(1, 2, 0) - def _aiter_mla_decode_fwd( - self, - q: torch.Tensor, - kv_buffer: torch.Tensor, - o: torch.Tensor, - qo_indptr: torch.Tensor, - max_seqlen_qo: int, - kv_indptr: Optional[torch.Tensor] = None, - kv_indices: Optional[torch.Tensor] = None, - kv_last_page_lens: Optional[torch.Tensor] = None, - sm_scale: float = 1.0, - logit_cap: float = 0.0, - ) -> None: - mla_decode_fwd( - q, - kv_buffer.view(-1, 1, 1, q.shape[-1]), - o, - qo_indptr, - kv_indptr, - kv_indices, - kv_last_page_lens, - max_seqlen_qo, - sm_scale=sm_scale, - logit_cap=logit_cap, - ) - def _v_up_proj_and_o_proj(self, x): # Convert from (B, N, L) to (N, B, L) x = x.view(-1, self.num_heads, self.kv_lora_rank).transpose(0, 1) @@ -277,16 +256,31 @@ def _forward_decode( self.topk_indices_buffer[:B], ) - self._aiter_mla_decode_fwd( + q_scale = kv_scale = None + if self.kv_cache_dtype.startswith("fp8"): + q = q.to(dtypes.fp8) + q_scale = kv_scale = self.one_scale + mla_decode_fwd( q, - kv_buffer, + kv_buffer.view(-1, 1, 1, q.shape[-1]), o, attn_metadata.cu_seqlens_q, - attn_metadata.max_q_len, paged_kv_indptr, paged_kv_indices, attn_metadata.kv_last_page_lens, + attn_metadata.max_q_len, self.scale, + 0.0, + None, + None, + attn_metadata.work_meta_data, + attn_metadata.work_indptr, + attn_metadata.work_info_set, + attn_metadata.reduce_indptr, + attn_metadata.reduce_final_map, + attn_metadata.reduce_partial_map, + q_scale, + kv_scale, ) return self._v_up_proj_and_o_proj(o) diff --git a/atom/model_ops/attentions/aiter_attention.py b/atom/model_ops/attentions/aiter_attention.py index 3e46a7943b..6c9d35cce7 100644 --- a/atom/model_ops/attentions/aiter_attention.py +++ b/atom/model_ops/attentions/aiter_attention.py @@ -26,8 +26,8 @@ def get_impl_cls() -> Type["Attention"]: class AiterAttentionMetadataBuilder(CommonAttentionBuilder): BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] - def __init__(self, block_size: int): - super().__init__(block_size) # Call parent __init__ to initialize _cached_kv_cache_data + def __init__(self, block_size: int, device: torch.device): + super().__init__(block_size, device) # Call parent __init__ to initialize _cached_kv_cache_data def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): scheduled_bs = batch.total_seqs_num_decode diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index ce149adc85..0dbf920fbc 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -12,6 +12,8 @@ from atom.model_engine.scheduler import ScheduledBatch from atom.model_ops.attention_mla import MLAAttention from atom.config import get_current_atom_config +from aiter import get_mla_metadata_v1, get_mla_metadata_info_v1, dtypes +from aiter.dist.parallel_state import get_tp_group from .backends import AttentionBackend, CommonAttentionBuilder @@ -32,11 +34,91 @@ def get_impl_cls() -> Type["MLAAttention"]: class AiterMLAMetadataBuilder(CommonAttentionBuilder): - def __init__(self, block_size: int): + def __init__(self, block_size: int, device: torch.device): super().__init__( - block_size + block_size, + device, ) # Call parent __init__ to initialize _cached_kv_cache_data assert self.block_size == 1, "AITER MLA requires only block size 1." + config = get_current_atom_config() + hf_config = config.hf_config + self.num_attention_heads = ( + hf_config.num_attention_heads // get_tp_group().world_size + ) + self.is_sparse = hasattr(hf_config, "index_topk") + self.index_topk = hf_config.index_topk if self.is_sparse else -1 + + ( + (work_meta_data_size, work_meta_data_type), + (work_indptr_size, work_indptr_type), + (work_info_set_size, work_info_set_type), + (reduce_indptr_size, reduce_indptr_type), + (reduce_final_map_size, reduce_final_map_type), + (reduce_partial_map_size, reduce_partial_map_type), + ) = get_mla_metadata_info_v1( + config.max_num_seqs, + 1, + self.num_attention_heads, + torch.bfloat16, + dtypes.d_dtypes[config.kv_cache_dtype], + is_sparse=self.is_sparse, + fast_mode=True, + ) + # AITER MLA specific persistent buffers + self.work_meta_data = torch.empty( + work_meta_data_size, dtype=work_meta_data_type, device=device + ) + self.work_indptr = torch.empty( + work_indptr_size, dtype=work_indptr_type, device=device + ) + self.work_info_set = torch.empty( + work_info_set_size, dtype=work_info_set_type, device=device + ) + + self.reduce_indptr = torch.empty( + reduce_indptr_size, dtype=reduce_indptr_type, device=device + ) + self.reduce_final_map = torch.empty( + reduce_final_map_size, dtype=reduce_final_map_type, device=device + ) + self.reduce_partial_map = torch.empty( + reduce_partial_map_size, dtype=reduce_partial_map_type, device=device + ) + + def set_mla_persistent_worker_buffers(self, forward_vars, bs: int): + split_params = { + "kv_granularity": max(self.block_size, 16), + "max_seqlen_qo": 1, + "uni_seqlen_qo": 1, + "fast_mode": 1, + "topk": -1, + } + get_mla_metadata_v1( + forward_vars["cu_seqlens_q"].gpu[: bs + 1], + ( + forward_vars["sparse_kv_indptr"].gpu[: bs + 1] + if self.is_sparse + else forward_vars["kv_indptr"].gpu[: bs + 1] + ), + self.num_attention_heads, + 1, # nhead_kv, + True, + self.work_meta_data, + self.work_info_set, + self.work_indptr, + self.reduce_indptr, + self.reduce_final_map, + self.reduce_partial_map, + **split_params, + ) + return { + "work_meta_data": self.work_meta_data, + "work_info_set": self.work_info_set, + "work_indptr": self.work_indptr, + "reduce_indptr": self.reduce_indptr, + "reduce_final_map": self.reduce_final_map, + "reduce_partial_map": self.reduce_partial_map, + } def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): scheduled_bs = batch.total_seqs_num_decode @@ -86,9 +168,8 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): ("kv_indices", sum_blocks), ("kv_last_page_lens", bs), ] - config = get_current_atom_config() - if hasattr(config.hf_config, "index_topk"): - index_topk = config.hf_config.index_topk + if self.is_sparse: + index_topk = self.index_topk sparse_context_lens = np.clip(var["context_lens"].np[:bs], None, index_topk) var["sparse_kv_indptr"].np[1 : bs + 1] = np.cumsum( sparse_context_lens, dtype=np.int32 @@ -99,6 +180,8 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): vars_used.append(("sparse_kv_indptr", bs + 1)) ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} + ctx_mla_ps = self.set_mla_persistent_worker_buffers(forward_vars, bs) + ctx.update(ctx_mla_ps) attn_metadata = AttentionMetaData( dropout_p=dropout_p, max_q_len=max_q_len, @@ -112,10 +195,9 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: sparse_kv_indptr = ( - forward_vars["sparse_kv_indptr"].gpu - if "sparse_kv_indptr" in forward_vars - else None + forward_vars["sparse_kv_indptr"].gpu if self.is_sparse else None ) + ctx_mla_ps = self.set_mla_persistent_worker_buffers(forward_vars, bs) attn_matadata = AttentionMetaData( slot_mapping=forward_vars["slot_mapping"].gpu[:bs], context_lens=forward_vars["context_lens"].gpu[:bs], @@ -126,6 +208,7 @@ def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaDat kv_indices=forward_vars["kv_indices"].gpu[:], kv_last_page_lens=forward_vars["kv_last_page_lens"].gpu[:bs], sparse_kv_indptr=sparse_kv_indptr, + **ctx_mla_ps, ) positions = forward_vars["positions"].copy_to_gpu(bs) context = Context( diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py index 988f2c53c7..0b77a19e8b 100644 --- a/atom/model_ops/attentions/backends.py +++ b/atom/model_ops/attentions/backends.py @@ -87,8 +87,9 @@ def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaDat class CommonAttentionBuilder(AttentionMetadataBuilder[T], Generic[T]): - def __init__(self, block_size: int): + def __init__(self, block_size: int, device: torch.device): self.block_size = block_size + self.device = device def prepare_prefill(self, batch: ScheduledBatch, forward_vars): bs = batch.total_seqs_num_prefill diff --git a/atom/model_ops/base_attention.py b/atom/model_ops/base_attention.py index 052d1b1648..437c5d4b50 100644 --- a/atom/model_ops/base_attention.py +++ b/atom/model_ops/base_attention.py @@ -51,7 +51,7 @@ def __init__( kv_cache_dtype="bf16", layer_num=0, use_mla: bool = False, - mla_modules: MLAModules=None, + mla_modules: MLAModules = None, **kwargs, ): super().__init__() diff --git a/atom/utils/forward_context.py b/atom/utils/forward_context.py index 2c0bf4b237..f1cc75ce4d 100644 --- a/atom/utils/forward_context.py +++ b/atom/utils/forward_context.py @@ -50,6 +50,13 @@ class AttentionMetaData: cu_seqlen_ke: Optional[torch.Tensor] = None sparse_kv_indptr: Optional[torch.Tensor] = None + work_meta_data: Optional[torch.Tensor] = None + work_indptr: Optional[torch.Tensor] = None + work_info_set: Optional[torch.Tensor] = None + reduce_indptr: Optional[torch.Tensor] = None + reduce_final_map: Optional[torch.Tensor] = None + reduce_partial_map: Optional[torch.Tensor] = None + def __init__( self, cu_seqlens_q: Optional[torch.Tensor] = None, @@ -68,6 +75,12 @@ def __init__( cu_seqlen_ks: Optional[torch.Tensor] = None, cu_seqlen_ke: Optional[torch.Tensor] = None, sparse_kv_indptr: Optional[torch.Tensor] = None, + work_meta_data: Optional[torch.Tensor] = None, + work_indptr: Optional[torch.Tensor] = None, + work_info_set: Optional[torch.Tensor] = None, + reduce_indptr: Optional[torch.Tensor] = None, + reduce_final_map: Optional[torch.Tensor] = None, + reduce_partial_map: Optional[torch.Tensor] = None, ): self.cu_seqlens_q = cu_seqlens_q self.cu_seqlens_k = cu_seqlens_k @@ -85,6 +98,12 @@ def __init__( self.cu_seqlen_ks = cu_seqlen_ks self.cu_seqlen_ke = cu_seqlen_ke self.sparse_kv_indptr = sparse_kv_indptr + self.work_meta_data = work_meta_data + self.work_indptr = work_indptr + self.work_info_set = work_info_set + self.reduce_indptr = reduce_indptr + self.reduce_final_map = reduce_final_map + self.reduce_partial_map = reduce_partial_map def asdict_zerocopy(self, skip_fields: Optional[Set[str]] = None) -> Dict[str, Any]: """Similar to dataclasses.asdict, but avoids deepcopying.""" From 7096453c4bc44d8c81fea608ab0fbda40688035e Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:39:35 +0800 Subject: [PATCH 109/119] broadcast rank 0 sampling result when using deferred_out (#33) * broadcast rank 0 sampling result when using deferred_out * fix readme * use aiter new topk name --- README.md | 2 +- atom/model_engine/model_runner.py | 13 ++++++------- atom/models/deepseek_v2.py | 8 ++++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a2a2a9de79..b26c5361fc 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ docker run -it --network=host \ ```bash pip install aiter -i https://mkmartifactory.amd.com/artifactory/api/pypi/hw-orc3pypi-prod-local/simple git clone https://github.com/valarLip/atom.git -cd ../atom +cd ./atom pip install . ``` diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 188062415b..94aca7f2e5 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -8,7 +8,7 @@ import torch.profiler as torch_profiler import tqdm from aiter import destroy_dist_env, dtypes, init_dist_env -from aiter.dist.parallel_state import graph_capture +from aiter.dist.parallel_state import graph_capture, get_tp_group from atom.config import Config, set_current_atom_config, KVCacheTensor from atom.model_engine.scheduler import ScheduledBatch from atom.model_engine.sequence import Sequence @@ -38,9 +38,9 @@ "DeepseekV3ForCausalLM": DeepseekV2ForCausalLM, "DeepseekV32ForCausalLM": DeepseekV2ForCausalLM, } -seed = 34567 -np.random.seed(seed) -torch.cuda.manual_seed_all(seed) +# seed = 34567 +# np.random.seed(seed) +# torch.cuda.manual_seed_all(seed) class tokenIDProcessor: @@ -703,9 +703,8 @@ def postprocess( temperatures: torch.Tensor, ) -> dict[int, int]: sampled_tokens = self.sampler(logits, temperatures) - # if self.rank == 0: - # print(f"{logits=}") - # print(f"{sampled_tokens=}") + if get_tp_group().world_size > 1 and self.tokenID_processor.is_deferred_out: + sampled_tokens = get_tp_group().broadcast(sampled_tokens, src=0) token_ids = self.tokenID_processor.prepare_sampled_ids( batch, sampled_tokens, diff --git a/atom/models/deepseek_v2.py b/atom/models/deepseek_v2.py index c2200779f2..36337f2711 100644 --- a/atom/models/deepseek_v2.py +++ b/atom/models/deepseek_v2.py @@ -34,8 +34,8 @@ gemm_a8w8_blockscale, get_hip_quant, indexer_k_quant_and_cache, - topk_per_row, - topk_per_row_decode, + top_k_per_row_prefill, + top_k_per_row_decode, ) from aiter.dist.communication_op import tensor_model_parallel_all_reduce from aiter.dist.parallel_state import ( @@ -291,7 +291,7 @@ def sparse_attn_indexer( topk_indices = topk_indices_buffer[ num_decode_tokens:num_tokens, :topk_tokens ] - topk_per_row( + top_k_per_row_prefill( logits, cu_seqlen_ks, cu_seqlen_ke, @@ -319,7 +319,7 @@ def sparse_attn_indexer( topk_indices = topk_indices_buffer[ :num_decode_tokens, :topk_tokens ] - topk_per_row_decode( + top_k_per_row_decode( logits, next_n, decode_metadata.context_lens, From 0d471e5e859f9396110c9b853d765d3b035cc466 Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Thu, 13 Nov 2025 22:27:41 +0800 Subject: [PATCH 110/119] enable fused_qk_rope_cat_and_cache_mla (#35) --- atom/examples/simple_inference.py | 9 ++++++ atom/model_ops/attention_mla.py | 51 ++++++++++++++++--------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/atom/examples/simple_inference.py b/atom/examples/simple_inference.py index 598436b5c2..5635ec7fc6 100644 --- a/atom/examples/simple_inference.py +++ b/atom/examples/simple_inference.py @@ -48,8 +48,17 @@ def main(): for prompt in prompts ] print("This is prompts:", prompts) + print("Warming up...") + _ = llm.generate(["warmup"], sampling_params) + print("Warm up done") + print("\n" + "=" * 70) + print("Starting profiling...") + print("=" * 70) + + llm.start_profile() outputs = llm.generate(prompts, sampling_params) + llm.stop_profile() for prompt, output in zip(prompts, outputs): print("\n") diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index ff082fc870..05dac69e14 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -23,6 +23,7 @@ from aiter.ops.triton.batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant import ( # noqa: E501 # isort: skip batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as _aiter_triton_fp8_bmm, ) +from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla torch.set_printoptions(threshold=10_000) @@ -196,7 +197,7 @@ def _forward_prefill( self, q: torch.Tensor, kv_c_normed: torch.Tensor, - k_pe: torch.Tensor, + k_rope: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, attn_metadata: AttentionMetaData, ) -> torch.Tensor: @@ -207,9 +208,9 @@ def _forward_prefill( ) k_nope, v = kv_nope.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1) - if k_pe.dim() == 2: - k_pe = k_pe.unsqueeze(1) - k = torch.cat((k_nope, k_pe.expand((*k_nope.shape[:-1], -1))), dim=-1) + if k_rope.dim() == 2: + k_rope = k_rope.unsqueeze(1) + k = torch.cat((k_nope, k_rope.expand((*k_nope.shape[:-1], -1))), dim=-1) output = flash_attn_varlen_func( q=q, @@ -229,16 +230,14 @@ def _forward_prefill( def _forward_decode( self, - q_nope: torch.Tensor, - q_pe: torch.Tensor, + q: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, attn_metadata: AttentionMetaData, ) -> torch.Tensor: assert kv_c_and_k_pe_cache.numel() > 0 assert attn_metadata is not None - B = q_nope.shape[0] + B = q.shape[0] - q = torch.cat([q_nope, q_pe], dim=-1) o = torch.empty( B, self.num_heads, self.kv_lora_rank, dtype=q.dtype, device=q.device ) @@ -288,8 +287,8 @@ def _forward_decode( def forward( self, q: torch.Tensor, # query in unified attn - k_c_normed: torch.Tensor, # key in unified attn - k_pe: torch.Tensor, # value in unified attn + k_nope: torch.Tensor, + k_rope: torch.Tensor, positions: torch.Tensor, ) -> torch.Tensor: # kv_cache = self.kv_cache @@ -310,12 +309,12 @@ def forward( if context.is_prefill: prefill_q = self.q_proj(q).view(-1, self.num_heads, self.qk_head_dim) prefill_q_pe = prefill_q[..., self.qk_nope_head_dim :] - self.rotary_emb(positions, prefill_q_pe, k_pe) + self.rotary_emb(positions, prefill_q_pe, k_rope) if kv_cache.numel() > 0: concat_and_cache_mla( - k_c_normed, - k_pe.squeeze(1), + k_nope, + k_rope.squeeze(1), kv_cache, attn_metadata.slot_mapping.flatten(), kv_cache_dtype=self.kv_cache_dtype, @@ -323,25 +322,27 @@ def forward( ) output = self._forward_prefill( - prefill_q, k_c_normed, k_pe, kv_cache, attn_metadata + prefill_q, k_nope, k_rope, kv_cache, attn_metadata ) else: - decode_ql_nope, decode_q_pe = self._q_proj_and_k_up_proj(q) - self.rotary_emb(positions, decode_q_pe, k_pe) + q_nope, q_rope = self._q_proj_and_k_up_proj(q) if kv_cache.numel() > 0: - concat_and_cache_mla( - k_c_normed, - k_pe.squeeze(1), + decode_q, _, _, _ = fused_qk_rope_cat_and_cache_mla( + q_nope, + q_rope, + k_nope.view(-1, self.num_kv_heads, self.kv_lora_rank), + k_rope.view(-1, self.num_kv_heads, self.qk_rope_head_dim), kv_cache, - attn_metadata.slot_mapping.flatten(), - kv_cache_dtype=self.kv_cache_dtype, - scale=self._k_scale, + attn_metadata.slot_mapping, + positions, + self.rotary_emb.cos_cache, + self.rotary_emb.sin_cache, + k_scale=self._k_scale, + is_neox=self.rotary_emb.is_neox_style, ) - output = self._forward_decode( - decode_ql_nope, decode_q_pe, kv_cache, attn_metadata - ) + output = self._forward_decode(decode_q, kv_cache, attn_metadata) return output From 821b310cf6b0ad390b3f81843f91bf74682939ce Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Fri, 14 Nov 2025 10:12:37 +0800 Subject: [PATCH 111/119] move attention meta data init to backend (#34) * move attention meta data init to backend * move sparse data to subclass mla backend --- atom/model_engine/model_runner.py | 57 +----- atom/model_ops/attentions/aiter_attention.py | 30 ++-- atom/model_ops/attentions/aiter_mla.py | 179 ++++++++++++------- atom/model_ops/attentions/backends.py | 88 +++++---- 4 files changed, 191 insertions(+), 163 deletions(-) diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 94aca7f2e5..b449b725d4 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -237,10 +237,6 @@ def __init__(self, rank: int, config: Config): self.block_size, use_mla=self.use_mla, ) - self.attn_metadata_builder = self.attn_backend.get_builder_cls()( - self.block_size, - self.device, - ) self.tokenID_processor = tokenIDProcessor( self.config.max_num_batched_tokens, self.device ) @@ -262,6 +258,7 @@ def __init__(self, rank: int, config: Config): self.use_kv_indptr = True torch.set_default_device("cuda") self.allocate_forward_vars() + self.attn_metadata_builder = self.attn_backend.get_builder_cls()(self) self.warmup_model() logger.info(f"Model warmup done: {config.model}") torch.set_default_device("cpu") @@ -391,9 +388,8 @@ def allocate_forward_vars(self): config = self.config hidden_size = config.hf_config.hidden_size hidden_type = config.hf_config.torch_dtype - max_bs = min(self.config.max_num_seqs, 512) - max_num_blocks = (config.max_model_len + self.block_size - 1) // self.block_size - max_num_batched_tokens = config.max_num_batched_tokens + self.max_bs = min(self.config.max_num_seqs, 512) + self.max_num_batched_tokens = config.max_num_batched_tokens i64_kwargs = {"dtype": torch.int64, "device": self.device} i32_kwargs = {"dtype": torch.int32, "device": self.device} f32_kwargs = {"dtype": torch.float, "device": self.device} @@ -401,34 +397,10 @@ def allocate_forward_vars(self): # TODO: remove it in forward_context self.forward_vars = { "input_ids": self.tokenID_processor.input_ids, - "positions": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), - "slot_mapping": CpuGpuBuffer(max_num_batched_tokens, **i64_kwargs), - "context_lens": CpuGpuBuffer(max_bs, **i32_kwargs), - "block_tables": CpuGpuBuffer(max_bs, max_num_blocks, **i32_kwargs), - "temperatures": CpuGpuBuffer(max_bs, **f32_kwargs), - "cu_seqlens_q": CpuGpuBuffer(max_bs + 1, **i32_kwargs), - "kv_indptr": CpuGpuBuffer(max_bs + 1, **i32_kwargs), - "kv_indices": CpuGpuBuffer(max_bs * max_num_blocks, **i32_kwargs), - "kv_last_page_lens": CpuGpuBuffer(max_bs, **i32_kwargs), - "outputs": torch.empty(max_bs, hidden_size, dtype=hidden_type), + "positions": CpuGpuBuffer(self.max_num_batched_tokens, **i64_kwargs), + "temperatures": CpuGpuBuffer(self.max_bs, **f32_kwargs), + "outputs": torch.empty(self.max_bs, hidden_size, dtype=hidden_type), } - if self.is_deepseek_v32: - self.forward_vars["cu_seqlen_ke"] = CpuGpuBuffer( - max_num_batched_tokens, **i32_kwargs - ) - self.forward_vars["cu_seqlen_ks"] = CpuGpuBuffer( - max_num_batched_tokens, **i32_kwargs - ) - self.forward_vars["sparse_kv_indptr"] = CpuGpuBuffer( - max_bs + 1, **i32_kwargs - ) - self.forward_vars["cu_seqlens_q"].cpu.copy_( - torch.arange(0, max_bs + 1, step=1, dtype=torch.int32) - ) - self.forward_vars["cu_seqlens_q"].copy_to_gpu() - - self.forward_vars["kv_last_page_lens"].cpu.fill_(1) - self.forward_vars["kv_last_page_lens"].copy_to_gpu() def get_num_blocks(self): torch.set_default_device("cuda") @@ -611,15 +583,6 @@ def allocate_kv_cache(self, num_kvcache_blocks): torch.distributed.barrier() return True - def prepare_block_tables(self, seqs: list[Sequence]): - block_tables = self.forward_vars["block_tables"].np - for i, seq in enumerate(seqs): - block_tables[i] = 0 - block_tables[i, : seq.num_blocks] = seq.block_table - - def prepare_prefill(self, batch: ScheduledBatch): - return self.attn_metadata_builder.prepare_prefill(batch, self.forward_vars) - def prepare_intputs(self, batch: ScheduledBatch): is_prefill = batch.total_tokens_num_prefill > 0 bs = batch.total_seqs_num @@ -643,9 +606,7 @@ def prepare_intputs(self, batch: ScheduledBatch): self.forward_vars["cu_seqlens_q"].np[scheduled_bs + 1 : bs + 1] = ( self.forward_vars["cu_seqlens_q"].np[scheduled_bs] ) - attn_metadata, positions = self.attn_metadata_builder.build( - batch, self.forward_vars, bs - ) + attn_metadata, positions = self.attn_metadata_builder.build(batch, bs) context_bs = ( batch.total_seqs_num_prefill if is_prefill else batch.total_seqs_num_decode ) @@ -755,9 +716,7 @@ def capture_cudagraph(self): graph = torch.cuda.CUDAGraph() attn_metadata, context = ( - self.attn_metadata_builder.build_for_cudagraph_capture( - forward_vars=self.forward_vars, bs=bs - ) + self.attn_metadata_builder.build_for_cudagraph_capture(bs) ) set_forward_context( attn_metadata=attn_metadata, diff --git a/atom/model_ops/attentions/aiter_attention.py b/atom/model_ops/attentions/aiter_attention.py index 6c9d35cce7..8360d61b07 100644 --- a/atom/model_ops/attentions/aiter_attention.py +++ b/atom/model_ops/attentions/aiter_attention.py @@ -23,13 +23,14 @@ def get_builder_cls() -> Type["AiterAttentionMetadataBuilder"]: def get_impl_cls() -> Type["Attention"]: return Attention + class AiterAttentionMetadataBuilder(CommonAttentionBuilder): BLOCK_TABLE_EXTENDER: list[list[int]] = [[]] - def __init__(self, block_size: int, device: torch.device): - super().__init__(block_size, device) # Call parent __init__ to initialize _cached_kv_cache_data + def __init__(self, model_runner): + super().__init__(model_runner) - def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): + def prepare_decode(self, batch: ScheduledBatch, bs: int): scheduled_bs = batch.total_seqs_num_decode seqs = list(batch.seqs.values()) self.total_blocks = 0 @@ -44,13 +45,13 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): ] slot_mapping.extend([-1] * (bs - scheduled_bs)) - block_tables = forward_vars["block_tables"].np + var = self.model_runner.forward_vars + block_tables = var["block_tables"].np for i, seq in enumerate(seqs): block_tables[i] = 0 block_tables[i, : seq.num_blocks] = seq.block_table sum_scheduled_tokens = batch.total_tokens_num_decode - var = forward_vars var["slot_mapping"].np[:bs] = slot_mapping var["positions"].np[:sum_scheduled_tokens] = positions var["context_lens"].np[:scheduled_bs] = context_lens @@ -67,17 +68,20 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): **ctx, ) - positions=var["positions"].copy_to_gpu(sum_scheduled_tokens) + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) return attn_metadata, positions - def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: + def build_for_cudagraph_capture(self, bs: int) -> AttentionMetaData: + var = self.model_runner.forward_vars attn_matadata = AttentionMetaData( - slot_mapping=forward_vars["slot_mapping"].gpu[:bs], - context_lens=forward_vars["context_lens"].gpu[:bs], - block_tables=forward_vars["block_tables"].gpu[:bs], + slot_mapping=var["slot_mapping"].gpu[:bs], + context_lens=var["context_lens"].gpu[:bs], + block_tables=var["block_tables"].gpu[:bs], max_q_len=1, - cu_seqlens_q=forward_vars["cu_seqlens_q"].gpu[: bs + 1], + cu_seqlens_q=var["cu_seqlens_q"].gpu[: bs + 1], + ) + positions = var["positions"].copy_to_gpu(bs) + context = Context( + positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs ) - positions=forward_vars["positions"].copy_to_gpu(bs) - context = Context(positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs) return attn_matadata, context diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index 0dbf920fbc..7a7ed127b5 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -11,9 +11,9 @@ import torch from atom.model_engine.scheduler import ScheduledBatch from atom.model_ops.attention_mla import MLAAttention -from atom.config import get_current_atom_config from aiter import get_mla_metadata_v1, get_mla_metadata_info_v1, dtypes from aiter.dist.parallel_state import get_tp_group +from atom.utils import CpuGpuBuffer from .backends import AttentionBackend, CommonAttentionBuilder @@ -34,18 +34,17 @@ def get_impl_cls() -> Type["MLAAttention"]: class AiterMLAMetadataBuilder(CommonAttentionBuilder): - def __init__(self, block_size: int, device: torch.device): + def __init__(self, model_runner): super().__init__( - block_size, - device, - ) # Call parent __init__ to initialize _cached_kv_cache_data + model_runner, + ) assert self.block_size == 1, "AITER MLA requires only block size 1." - config = get_current_atom_config() + config = model_runner.config hf_config = config.hf_config self.num_attention_heads = ( hf_config.num_attention_heads // get_tp_group().world_size ) - self.is_sparse = hasattr(hf_config, "index_topk") + self.is_sparse = model_runner.is_deepseek_v32 self.index_topk = hf_config.index_topk if self.is_sparse else -1 ( @@ -56,7 +55,7 @@ def __init__(self, block_size: int, device: torch.device): (reduce_final_map_size, reduce_final_map_type), (reduce_partial_map_size, reduce_partial_map_type), ) = get_mla_metadata_info_v1( - config.max_num_seqs, + self.max_bs, 1, self.num_attention_heads, torch.bfloat16, @@ -64,28 +63,51 @@ def __init__(self, block_size: int, device: torch.device): is_sparse=self.is_sparse, fast_mode=True, ) - # AITER MLA specific persistent buffers - self.work_meta_data = torch.empty( - work_meta_data_size, dtype=work_meta_data_type, device=device - ) - self.work_indptr = torch.empty( - work_indptr_size, dtype=work_indptr_type, device=device - ) - self.work_info_set = torch.empty( - work_info_set_size, dtype=work_info_set_type, device=device - ) + i32_kwargs = {"dtype": torch.int32, "device": self.device} - self.reduce_indptr = torch.empty( - reduce_indptr_size, dtype=reduce_indptr_type, device=device - ) - self.reduce_final_map = torch.empty( - reduce_final_map_size, dtype=reduce_final_map_type, device=device - ) - self.reduce_partial_map = torch.empty( - reduce_partial_map_size, dtype=reduce_partial_map_type, device=device - ) + mla_metadata = { + # AITER MLA specific persistent buffers + "work_meta_data": torch.empty( + work_meta_data_size, dtype=work_meta_data_type, device=self.device + ), + "work_indptr": torch.empty( + work_indptr_size, dtype=work_indptr_type, device=self.device + ), + "work_info_set": torch.empty( + work_info_set_size, dtype=work_info_set_type, device=self.device + ), + "reduce_indptr": torch.empty( + reduce_indptr_size, dtype=reduce_indptr_type, device=self.device + ), + "reduce_final_map": torch.empty( + reduce_final_map_size, dtype=reduce_final_map_type, device=self.device + ), + "reduce_partial_map": torch.empty( + reduce_partial_map_size, + dtype=reduce_partial_map_type, + device=self.device, + ), + "kv_indptr": CpuGpuBuffer(self.max_bs + 1, **i32_kwargs), + "kv_indices": CpuGpuBuffer( + self.max_bs * self.max_num_blocks_per_seq, **i32_kwargs + ), + "kv_last_page_lens": CpuGpuBuffer(self.max_bs, **i32_kwargs), + } + mla_metadata["kv_last_page_lens"].cpu.fill_(1) + mla_metadata["kv_last_page_lens"].copy_to_gpu() + if self.is_sparse: + mla_metadata["cu_seqlen_ke"] = CpuGpuBuffer( + self.max_num_batched_tokens, **i32_kwargs + ) + mla_metadata["cu_seqlen_ks"] = CpuGpuBuffer( + self.max_num_batched_tokens, **i32_kwargs + ) + mla_metadata["sparse_kv_indptr"] = CpuGpuBuffer( + self.max_bs + 1, **i32_kwargs + ) + self.model_runner.forward_vars.update(mla_metadata) - def set_mla_persistent_worker_buffers(self, forward_vars, bs: int): + def set_mla_persistent_worker_buffers(self, bs: int): split_params = { "kv_granularity": max(self.block_size, 16), "max_seqlen_qo": 1, @@ -93,34 +115,68 @@ def set_mla_persistent_worker_buffers(self, forward_vars, bs: int): "fast_mode": 1, "topk": -1, } + var = self.model_runner.forward_vars + work_meta_data = var["work_meta_data"] + work_info_set = var["work_info_set"] + work_indptr = var["work_indptr"] + reduce_indptr = var["reduce_indptr"] + reduce_final_map = var["reduce_final_map"] + reduce_partial_map = var["reduce_partial_map"] get_mla_metadata_v1( - forward_vars["cu_seqlens_q"].gpu[: bs + 1], + var["cu_seqlens_q"].gpu[: bs + 1], ( - forward_vars["sparse_kv_indptr"].gpu[: bs + 1] + var["sparse_kv_indptr"].gpu[: bs + 1] if self.is_sparse - else forward_vars["kv_indptr"].gpu[: bs + 1] + else var["kv_indptr"].gpu[: bs + 1] ), self.num_attention_heads, 1, # nhead_kv, True, - self.work_meta_data, - self.work_info_set, - self.work_indptr, - self.reduce_indptr, - self.reduce_final_map, - self.reduce_partial_map, + work_meta_data, + work_info_set, + work_indptr, + reduce_indptr, + reduce_final_map, + reduce_partial_map, **split_params, ) return { - "work_meta_data": self.work_meta_data, - "work_info_set": self.work_info_set, - "work_indptr": self.work_indptr, - "reduce_indptr": self.reduce_indptr, - "reduce_final_map": self.reduce_final_map, - "reduce_partial_map": self.reduce_partial_map, + "work_meta_data": work_meta_data, + "work_info_set": work_info_set, + "work_indptr": work_indptr, + "reduce_indptr": reduce_indptr, + "reduce_final_map": reduce_final_map, + "reduce_partial_map": reduce_partial_map, } - def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): + def prepare_prefill(self, batch: ScheduledBatch): + attn_metadata, positions = super().prepare_prefill(batch) + bs = batch.total_seqs_num_prefill + seqs = list(batch.seqs.values()) + sum_scheduled_tokens = batch.total_tokens_num_prefill + var = self.model_runner.forward_vars + if self.is_sparse: + block_tables = var["block_tables"].np + for i, seq in enumerate(seqs): + block_tables[i] = 0 + if len(seq.block_table) > 0: + block_tables[i, : seq.num_blocks] = seq.block_table + var["cu_seqlen_ke"].np[:sum_scheduled_tokens] = ( + np.arange(sum_scheduled_tokens, dtype=np.int32) + 1 + ) + counts = var["cu_seqlens_q"].np[1 : bs + 1] - var["cu_seqlens_q"].np[:bs] + var["cu_seqlen_ks"].np[:sum_scheduled_tokens] = np.repeat( + var["cu_seqlens_q"].np[:bs], counts + ) + attn_metadata.cu_seqlen_ks = var["cu_seqlen_ks"].copy_to_gpu( + sum_scheduled_tokens + ) + attn_metadata.cu_seqlen_ke = var["cu_seqlen_ke"].copy_to_gpu( + sum_scheduled_tokens + ) + return attn_metadata, positions + + def prepare_decode(self, batch: ScheduledBatch, bs: int): scheduled_bs = batch.total_seqs_num_decode seqs = list(batch.seqs.values()) dropout_p = 0.0 @@ -133,12 +189,8 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): for seq in seqs ] - var = forward_vars - # prepare_block_tables - block_tables = var["block_tables"].np - for i, seq in enumerate(seqs): - block_tables[i] = 0 - block_tables[i, : seq.num_blocks] = seq.block_table + var = self.model_runner.forward_vars + self.prepare_block_tables(seqs) sum_scheduled_tokens = batch.total_tokens_num_decode var["slot_mapping"].np[:scheduled_bs] = slot_mapping @@ -180,7 +232,7 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): vars_used.append(("sparse_kv_indptr", bs + 1)) ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} - ctx_mla_ps = self.set_mla_persistent_worker_buffers(forward_vars, bs) + ctx_mla_ps = self.set_mla_persistent_worker_buffers(bs) ctx.update(ctx_mla_ps) attn_metadata = AttentionMetaData( dropout_p=dropout_p, @@ -193,24 +245,23 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): # print(f"{el}: {var}") return attn_metadata, positions - def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: - sparse_kv_indptr = ( - forward_vars["sparse_kv_indptr"].gpu if self.is_sparse else None - ) - ctx_mla_ps = self.set_mla_persistent_worker_buffers(forward_vars, bs) + def build_for_cudagraph_capture(self, bs: int) -> AttentionMetaData: + var = self.model_runner.forward_vars + sparse_kv_indptr = var["sparse_kv_indptr"].gpu if self.is_sparse else None + ctx_mla_ps = self.set_mla_persistent_worker_buffers(bs) attn_matadata = AttentionMetaData( - slot_mapping=forward_vars["slot_mapping"].gpu[:bs], - context_lens=forward_vars["context_lens"].gpu[:bs], - block_tables=forward_vars["block_tables"].gpu[:bs], + slot_mapping=var["slot_mapping"].gpu[:bs], + context_lens=var["context_lens"].gpu[:bs], + block_tables=var["block_tables"].gpu[:bs], max_q_len=1, - cu_seqlens_q=forward_vars["cu_seqlens_q"].gpu[: bs + 1], - kv_indptr=forward_vars["kv_indptr"].gpu[: bs + 1], - kv_indices=forward_vars["kv_indices"].gpu[:], - kv_last_page_lens=forward_vars["kv_last_page_lens"].gpu[:bs], + cu_seqlens_q=var["cu_seqlens_q"].gpu[: bs + 1], + kv_indptr=var["kv_indptr"].gpu[: bs + 1], + kv_indices=var["kv_indices"].gpu[:], + kv_last_page_lens=var["kv_last_page_lens"].gpu[:bs], sparse_kv_indptr=sparse_kv_indptr, **ctx_mla_ps, ) - positions = forward_vars["positions"].copy_to_gpu(bs) + positions = var["positions"].copy_to_gpu(bs) context = Context( positions=positions, is_prefill=False, batch_size=bs, graph_bs=bs ) diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py index 0b77a19e8b..4916bb793e 100644 --- a/atom/model_ops/attentions/backends.py +++ b/atom/model_ops/attentions/backends.py @@ -5,6 +5,8 @@ import numpy as np from atom.utils.forward_context import AttentionMetaData +from atom.utils import CpuGpuBuffer +from atom.model_engine.sequence import Sequence import torch from torch import nn @@ -67,34 +69,64 @@ def __init__(self, block_size: int) -> None: raise NotImplementedError @abstractmethod - def prepare_decode(self, batch: ScheduledBatch, bs: int, forward_vars): + def prepare_decode(self, batch: ScheduledBatch, bs: int): raise NotImplementedError @abstractmethod - def prepare_prefill(self, batch: ScheduledBatch, forward_vars): + def prepare_prefill(self, batch: ScheduledBatch): raise NotImplementedError @abstractmethod - def build(self, - batch: ScheduledBatch, - forward_vars, - bs: int): + def build(self, batch: ScheduledBatch, bs: int): raise NotImplementedError - + @abstractmethod - def build_for_cudagraph_capture(self, forward_vars, bs: int) -> AttentionMetaData: + def build_for_cudagraph_capture(self, bs: int) -> AttentionMetaData: raise NotImplementedError class CommonAttentionBuilder(AttentionMetadataBuilder[T], Generic[T]): - def __init__(self, block_size: int, device: torch.device): - self.block_size = block_size - self.device = device + def __init__(self, model_runner): + self.model_runner = model_runner + self.block_size = model_runner.block_size + self.device = model_runner.device + config = model_runner.config + self.max_num_batched_tokens = model_runner.max_num_batched_tokens + self.max_bs = model_runner.max_bs + self.max_num_blocks_per_seq = ( + config.max_model_len + self.block_size - 1 + ) // self.block_size + + i64_kwargs = {"dtype": torch.int64, "device": self.device} + i32_kwargs = {"dtype": torch.int32, "device": self.device} + + attn_metadata = { + "slot_mapping": CpuGpuBuffer(self.max_num_batched_tokens, **i64_kwargs), + "context_lens": CpuGpuBuffer(self.max_bs, **i32_kwargs), + "block_tables": CpuGpuBuffer( + self.max_bs, self.max_num_blocks_per_seq, **i32_kwargs + ), + "cu_seqlens_q": CpuGpuBuffer(self.max_bs + 1, **i32_kwargs), + "cu_seqlens_k": CpuGpuBuffer(self.max_bs + 1, **i32_kwargs), + } + + attn_metadata["cu_seqlens_q"].cpu.copy_( + torch.arange(0, self.max_bs + 1, step=1, dtype=torch.int32) + ) + attn_metadata["cu_seqlens_q"].copy_to_gpu() + self.model_runner.forward_vars.update(attn_metadata) + + def prepare_block_tables(self, seqs: list[Sequence]): + var = self.model_runner.forward_vars + block_tables = var["block_tables"].np + for i, seq in enumerate(seqs): + block_tables[i] = 0 + block_tables[i, : seq.num_blocks] = seq.block_table - def prepare_prefill(self, batch: ScheduledBatch, forward_vars): + def prepare_prefill(self, batch: ScheduledBatch): bs = batch.total_seqs_num_prefill sum_scheduled_tokens = batch.total_tokens_num_prefill - var = forward_vars + var = self.model_runner.forward_vars positions = [] cu_seqlens_k = [0] max_seqlen_q = 0 @@ -132,21 +164,6 @@ def prepare_prefill(self, batch: ScheduledBatch, forward_vars): ("cu_seqlens_q", bs + 1), ("slot_mapping", len(slot_mapping)), ] - if "cu_seqlen_ke" in var and "cu_seqlen_ks" in var: - block_tables = forward_vars["block_tables"].np - for i, seq in enumerate(seqs): - block_tables[i] = 0 - if len(seq.block_table) > 0: - block_tables[i, : seq.num_blocks] = seq.block_table - var["cu_seqlen_ke"].np[:sum_scheduled_tokens] = ( - np.arange(sum_scheduled_tokens, dtype=np.int32) + 1 - ) - counts = var["cu_seqlens_q"].np[1 : bs + 1] - var["cu_seqlens_q"].np[:bs] - var["cu_seqlen_ks"].np[:sum_scheduled_tokens] = np.repeat( - var["cu_seqlens_q"].np[:bs], counts - ) - vars_used.append(("cu_seqlen_ke", sum_scheduled_tokens)) - vars_used.append(("cu_seqlen_ks", sum_scheduled_tokens)) ctx = {el: var[el].copy_to_gpu(num) for el, num in vars_used} attn_metadata = AttentionMetaData( cu_seqlens_k=cu_seqlens_k.cuda(non_blocking=True), @@ -156,19 +173,16 @@ def prepare_prefill(self, batch: ScheduledBatch, forward_vars): dropout_p=dropout_p, **ctx, ) - positions=var["positions"].copy_to_gpu(sum_scheduled_tokens) + positions = var["positions"].copy_to_gpu(sum_scheduled_tokens) return attn_metadata, positions # return var["positions"].copy_to_gpu(sum_scheduled_tokens) - def build(self, - batch: ScheduledBatch, - forward_vars, - bs: int): + def build(self, batch: ScheduledBatch, bs: int): if batch.total_tokens_num_prefill > 0: - return self.prepare_prefill(batch, forward_vars) + return self.prepare_prefill(batch) else: - return self.prepare_decode(batch, bs, forward_vars) + return self.prepare_decode(batch, bs) class AttentionImpl(nn.Module): @@ -181,7 +195,7 @@ def __init__( num_kv_heads: Optional[int] = None, kv_cache_dtype: str = "auto", layer_num: int = 0, - mla_modules: MLAModules=None, + mla_modules: MLAModules = None, ) -> None: raise NotImplementedError @@ -191,6 +205,6 @@ def forward( query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, - position: torch.Tensor=None, + position: torch.Tensor = None, ) -> torch.Tensor: raise NotImplementedError From 7315f88f5da30690c0fd8af5a215de5c4c405103 Mon Sep 17 00:00:00 2001 From: la <46212055+junhaha666@users.noreply.github.com> Date: Fri, 14 Nov 2025 22:51:52 +0800 Subject: [PATCH 112/119] move block table (#37) --- atom/model_ops/attentions/aiter_mla.py | 12 ++++-------- atom/model_ops/attentions/backends.py | 4 ++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/atom/model_ops/attentions/aiter_mla.py b/atom/model_ops/attentions/aiter_mla.py index 7a7ed127b5..9509a2e3ba 100644 --- a/atom/model_ops/attentions/aiter_mla.py +++ b/atom/model_ops/attentions/aiter_mla.py @@ -156,11 +156,8 @@ def prepare_prefill(self, batch: ScheduledBatch): sum_scheduled_tokens = batch.total_tokens_num_prefill var = self.model_runner.forward_vars if self.is_sparse: - block_tables = var["block_tables"].np - for i, seq in enumerate(seqs): - block_tables[i] = 0 - if len(seq.block_table) > 0: - block_tables[i, : seq.num_blocks] = seq.block_table + self.prepare_block_tables(seqs) + attn_metadata.block_tables = var["block_tables"].copy_to_gpu(bs) var["cu_seqlen_ke"].np[:sum_scheduled_tokens] = ( np.arange(sum_scheduled_tokens, dtype=np.int32) + 1 ) @@ -190,8 +187,6 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int): ] var = self.model_runner.forward_vars - self.prepare_block_tables(seqs) - sum_scheduled_tokens = batch.total_tokens_num_decode var["slot_mapping"].np[:scheduled_bs] = slot_mapping var["slot_mapping"].np[scheduled_bs:bs] = -1 @@ -214,13 +209,14 @@ def prepare_decode(self, batch: ScheduledBatch, bs: int): vars_used = [ ("slot_mapping", bs), # TODO: MTP support ("context_lens", bs), - ("block_tables", bs), ("cu_seqlens_q", bs + 1), ("kv_indptr", bs + 1), ("kv_indices", sum_blocks), ("kv_last_page_lens", bs), ] if self.is_sparse: + self.prepare_block_tables(seqs) + vars_used.append(("block_tables", bs)) index_topk = self.index_topk sparse_context_lens = np.clip(var["context_lens"].np[:bs], None, index_topk) var["sparse_kv_indptr"].np[1 : bs + 1] = np.cumsum( diff --git a/atom/model_ops/attentions/backends.py b/atom/model_ops/attentions/backends.py index 4916bb793e..1471b04b4f 100644 --- a/atom/model_ops/attentions/backends.py +++ b/atom/model_ops/attentions/backends.py @@ -121,7 +121,8 @@ def prepare_block_tables(self, seqs: list[Sequence]): block_tables = var["block_tables"].np for i, seq in enumerate(seqs): block_tables[i] = 0 - block_tables[i, : seq.num_blocks] = seq.block_table + if len(seq.block_table) > 0: + block_tables[i, : seq.num_blocks] = seq.block_table def prepare_prefill(self, batch: ScheduledBatch): bs = batch.total_seqs_num_prefill @@ -160,7 +161,6 @@ def prepare_prefill(self, batch: ScheduledBatch): min_seqlen_q = 0 dropout_p = 0.0 vars_used = [ - ("block_tables", bs), ("cu_seqlens_q", bs + 1), ("slot_mapping", len(slot_mapping)), ] From b5fdf42420f7ec73487853cb8a375dfe2d0e0b16 Mon Sep 17 00:00:00 2001 From: HaonanWang98 Date: Tue, 18 Nov 2025 22:30:49 +0800 Subject: [PATCH 113/119] Add async engine and stream mode (#38) * add async engine 1st version * add stream mode * fix * remove reduntant and fix memory leak --- atom/__init__.py | 2 +- atom/entrypoints/openai_server.py | 414 ++++++++++++++++++++------- atom/examples/profile_offline.py | 19 +- atom/examples/profile_online.py | 52 +++- atom/model_engine/arg_utils.py | 22 +- atom/model_engine/engine_core.py | 30 +- atom/model_engine/engine_core_mgr.py | 76 ++++- atom/model_engine/llm_engine.py | 227 +++++++++++---- atom/model_engine/request.py | 12 + atom/model_engine/scheduler.py | 35 ++- atom/model_engine/sequence.py | 13 +- 11 files changed, 730 insertions(+), 172 deletions(-) create mode 100644 atom/model_engine/request.py diff --git a/atom/__init__.py b/atom/__init__.py index 0ad0f2999a..f9bcb6e444 100644 --- a/atom/__init__.py +++ b/atom/__init__.py @@ -1,2 +1,2 @@ -from atom.model_engine.llm_engine import LLMEngine +from atom.model_engine.llm_engine import LLMEngine, AsyncLLMEngine from atom.sampling_params import SamplingParams diff --git a/atom/entrypoints/openai_server.py b/atom/entrypoints/openai_server.py index de582f44f6..2aef6b702e 100644 --- a/atom/entrypoints/openai_server.py +++ b/atom/entrypoints/openai_server.py @@ -1,16 +1,24 @@ import argparse import asyncio +import json import time -from typing import List, Optional, Dict, Any, Union +import queue +from typing import List, Optional, Dict, Any, AsyncGenerator import uuid +from collections import defaultdict from fastapi import FastAPI, HTTPException +from fastapi.responses import StreamingResponse from pydantic import BaseModel import uvicorn from transformers import AutoTokenizer from atom import SamplingParams from atom.model_engine.arg_utils import EngineArgs +from atom.model_engine.request import RequestOutput +import logging + +logger = logging.getLogger("atom") class ChatMessage(BaseModel): @@ -26,16 +34,18 @@ class ChatCompletionRequest(BaseModel): max_tokens: Optional[int] = 256 stop: Optional[List[str]] = None ignore_eos: Optional[bool] = False + stream: Optional[bool] = False class CompletionRequest(BaseModel): model: str - prompt: Union[str, List[str]] + prompt: str temperature: Optional[float] = 1.0 top_p: Optional[float] = 1.0 max_tokens: Optional[int] = 256 stop: Optional[List[str]] = None ignore_eos: Optional[bool] = False + stream: Optional[bool] = False class ChatCompletionResponse(BaseModel): @@ -56,79 +66,256 @@ class CompletionResponse(BaseModel): usage: Dict[str, Any] -# Global engine engine = None tokenizer: Optional[AutoTokenizer] = None model_name: str = "" +_stream_queues: Dict[str, queue.Queue] = {} +_seq_id_to_request_id: Dict[int, str] = {} + + +def create_chat_completion_chunk( + request_id: str, + model: str, + content: str = "", + finish_reason: Optional[str] = None, + usage: Optional[Dict] = None +) -> str: + chunk = { + "id": request_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [{"index": 0, "delta": {"content": content} if content else {}, "finish_reason": finish_reason, "logprobs": None}], + } + if usage is not None: + chunk["usage"] = usage + return f"data: {json.dumps(chunk)}\n\n" + + +def create_chat_usage_chunk( + request_id: str, + model: str, + usage: Dict +) -> str: + """Create a chunk containing only usage information for chat completions (no choices).""" + chunk = { + "id": request_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "usage": usage, + } + return f"data: {json.dumps(chunk)}\n\n" + + +def create_completion_chunk( + request_id: str, + model: str, + text: str, + finish_reason: Optional[str] = None, + usage: Optional[Dict] = None +) -> str: + chunk = { + "id": request_id, + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [{"index": 0, "text": text, "finish_reason": finish_reason, "logprobs": None}], + } + if usage is not None: + chunk["usage"] = usage + return f"data: {json.dumps(chunk)}\n\n" + + +def create_usage_chunk( + request_id: str, + model: str, + usage: Dict +) -> str: + """Create a chunk containing only usage information (no choices).""" + chunk = { + "id": request_id, + "object": "text_completion", + "created": int(time.time()), + "model": model, + "usage": usage, + } + return f"data: {json.dumps(chunk)}\n\n" -async def generate_async(prompt: str, sampling_params: SamplingParams) -> Dict[str, Any]: - """Async wrapper for engine generation.""" - global engine +def send_stream_chunk(request_output: RequestOutput): + global tokenizer, _stream_queues, _seq_id_to_request_id - async for output in engine.generate_async(prompt, sampling_params): - return output + # Get request_id from sequence ID + request_id = _seq_id_to_request_id.get(request_output.request_id) + if request_id is None: + logger.warning(f"send_stream_chunk: No request_id found for sequence {request_output.request_id}") + return + + # Decode the new tokens + new_text = tokenizer.decode(request_output.output_tokens, skip_special_tokens=True) + logger.debug(f"send_stream_chunk: seq_id={request_output.request_id}, request_id={request_id}, tokens={request_output.output_tokens}, text='{new_text}'") + + # Get the queue for this request + stream_queue = _stream_queues.get(request_id) + if stream_queue is None: + logger.warning(f"send_stream_chunk: No queue found for request_id {request_id}") + return + + # Prepare chunk data + chunk_data = { + "text": new_text, + "token_ids": request_output.output_tokens, + "finished": request_output.finished, + "finish_reason": request_output.finish_reason, + } + + try: + stream_queue.put_nowait(chunk_data) + logger.debug(f"send_stream_chunk: Successfully put chunk data into queue for request_id {request_id}") + except queue.Full: + logger.warning(f"send_stream_chunk: Queue full for request_id {request_id}, skipping chunk") + pass + + +async def generate_async( + prompt: str, + sampling_params: SamplingParams, + request_id: str +) -> AsyncGenerator[Dict[str, Any], None]: + global engine, tokenizer + async for output in engine.generate_async(prompt, sampling_params, request_id): + yield { + "text": output["text"], + "token_ids": output["token_ids"], + "finish_reason": output.get("finish_reason"), + "num_tokens_input": len(tokenizer.encode(prompt)), + "num_tokens_output": len(output["token_ids"]), + "ttft": output.get("ttft", 0.0), + "tpot": output.get("tpot", 0.0), + "latency": output.get("latency", 0.0), + } app = FastAPI(title="Atom OpenAI API Server") -@app.post("/v1/chat/completions", response_model=ChatCompletionResponse) +@app.post("/v1/chat/completions") async def chat_completions(request: ChatCompletionRequest): - """Handle chat completion requests.""" global engine, tokenizer, model_name - try: prompt = tokenizer.apply_chat_template( [{"role": msg.role, "content": msg.content} for msg in request.messages], tokenize=False, add_generation_prompt=True, ) - + sampling_params = SamplingParams( temperature=request.temperature, max_tokens=request.max_tokens, stop_strings=request.stop, ignore_eos=request.ignore_eos, ) - - output = await generate_async(prompt, sampling_params) - - response = ChatCompletionResponse( - id=f"chatcmpl-{uuid.uuid4().hex}", - created=int(time.time()), + + request_id = f"chatcmpl-{uuid.uuid4().hex}" + created = int(time.time()) + if request.stream: + async def generate_stream(): + stream_queue = queue.Queue() + _stream_queues[request_id] = stream_queue + + def stream_callback(request_output: RequestOutput): + send_stream_chunk(request_output) + + seq = engine.io_processor.preprocess(prompt, sampling_params, stream_callback=stream_callback) + seq_id = seq.id + _seq_id_to_request_id[seq_id] = request_id + logger.info(f"API: Created request_id={request_id}, seq_id={seq_id}, queue={stream_queue is not None}") + engine.core_mgr.add_request([seq]) + logger.info(f"API: Added request to engine, callback registered: {seq.stream_callback is not None}") + + prev_text = "" + num_tokens_input = len(tokenizer.encode(prompt)) + num_tokens_output = 0 + yield create_chat_completion_chunk(request_id, request.model, "") + + # Consume chunks from queue using executor to avoid blocking + finished = False + loop = asyncio.get_event_loop() + while not finished: + try: + chunk_data = await asyncio.wait_for( + loop.run_in_executor(None, stream_queue.get), + timeout=30.0 + ) + new_text = chunk_data["text"] + current_text = prev_text + new_text + new_content = new_text + prev_text = current_text + + chunk_token_ids = chunk_data.get("token_ids", []) + num_tokens_output += len(chunk_token_ids) + + yield create_chat_completion_chunk( + request_id, + request.model, + new_content, + finish_reason=chunk_data.get("finish_reason") + ) + + if chunk_data.get("finished", False): + finished = True + except asyncio.TimeoutError: + break + + _stream_queues.pop(request_id, None) + _seq_id_to_request_id.pop(seq_id, None) + + yield create_chat_completion_chunk(request_id, request.model, "", "stop") + usage = { + "prompt_tokens": num_tokens_input, + "completion_tokens": num_tokens_output, + "total_tokens": num_tokens_input + num_tokens_output, + } + yield create_chat_usage_chunk(request_id, request.model, usage) + yield "data: [DONE]\n\n" + + return StreamingResponse(generate_stream(), media_type="text/event-stream") + + # non-streaming + final_output = None + async for output in generate_async(prompt, sampling_params, request_id): + final_output = output + if final_output is None: + raise RuntimeError("No output generated") + + return ChatCompletionResponse( + id=request_id, + created=created, model=request.model, choices=[ { "index": 0, - "message": { - "role": "assistant", - "content": output["text"], - }, - "finish_reason": output["finish_reason"], + "message": {"role": "assistant", "content": final_output["text"]}, + "finish_reason": final_output["finish_reason"], } ], usage={ - "prompt_tokens": output["num_tokens_input"], - "completion_tokens": output["num_tokens_output"], - "total_tokens": output["num_tokens_input"] + output["num_tokens_output"], - "ttft_s": output.get("ttft", 0.0), - "tpot_s": output.get("tpot", 0.0), - "latency_s": output.get("latency", 0.0), + "prompt_tokens": final_output["num_tokens_input"], + "completion_tokens": final_output["num_tokens_output"], + "total_tokens": final_output["num_tokens_input"] + final_output["num_tokens_output"], + "ttft_s": round(final_output.get("ttft", 0.0), 4), + "tpot_s": round(final_output.get("tpot", 0.0), 4), + "latency_s": round(final_output.get("latency", 0.0), 4), }, ) - - return response - except Exception as e: raise HTTPException(status_code=500, detail=str(e)) -@app.post("/v1/completions", response_model=CompletionResponse) +@app.post("/v1/completions") async def completions(request: CompletionRequest): - """Handle text completion requests.""" - global engine, model_name - + global engine, tokenizer, model_name try: sampling_params = SamplingParams( temperature=request.temperature, @@ -136,69 +323,103 @@ async def completions(request: CompletionRequest): stop_strings=request.stop, ignore_eos=request.ignore_eos, ) - - if isinstance(request.prompt, str): - output = await generate_async(request.prompt, sampling_params) - choices = [ - { - "index": 0, - "text": output["text"], - "finish_reason": output["finish_reason"], - } - ] - total_prompt_tokens = output["num_tokens_input"] - total_completion_tokens = output["num_tokens_output"] - else: - tasks = [ - generate_async(prompt, sampling_params) - for prompt in request.prompt - ] - outputs = await asyncio.gather(*tasks) - - choices = [ - { - "index": i, - "text": output["text"], - "finish_reason": output["finish_reason"], + + if request.stream: + request_id = f"cmpl-{uuid.uuid4().hex}" + + async def generate_stream(): + stream_queue = queue.Queue() + _stream_queues[request_id] = stream_queue + + def stream_callback(request_output: RequestOutput): + send_stream_chunk(request_output) + + seq = engine.io_processor.preprocess(request.prompt, sampling_params, stream_callback=stream_callback) + seq_id = seq.id + _seq_id_to_request_id[seq_id] = request_id + engine.core_mgr.add_request([seq]) + + prev_text = "" + num_tokens_input = len(tokenizer.encode(request.prompt)) + num_tokens_output = 0 + + finished = False + loop = asyncio.get_event_loop() + while not finished: + try: + chunk_data = await asyncio.wait_for( + loop.run_in_executor(None, stream_queue.get), + timeout=30.0 + ) + new_text = chunk_data["text"] + current_text = prev_text + new_text + new_content = new_text + prev_text = current_text + + chunk_token_ids = chunk_data.get("token_ids", []) + num_tokens_output += len(chunk_token_ids) + + yield create_completion_chunk( + request_id, + request.model, + new_content, + finish_reason=chunk_data.get("finish_reason") + ) + + if chunk_data.get("finished", False): + finished = True + except asyncio.TimeoutError: + break + + # Cleanup + _stream_queues.pop(request_id, None) + _seq_id_to_request_id.pop(seq_id, None) + + yield create_completion_chunk(request_id, request.model, "", "stop") + usage = { + "prompt_tokens": num_tokens_input, + "completion_tokens": num_tokens_output, + "total_tokens": num_tokens_input + num_tokens_output, } - for i, output in enumerate(outputs) - ] - total_prompt_tokens = sum(output["num_tokens_input"] for output in outputs) - total_completion_tokens = sum(output["num_tokens_output"] for output in outputs) - - if isinstance(request.prompt, str): - ttft = output.get("ttft", 0.0) - tpot = output.get("tpot", 0.0) - latency = output.get("latency", 0.0) - else: - ttft = sum(output.get("ttft", 0.0) for output in outputs) / len(outputs) if outputs else 0.0 - tpot = sum(output.get("tpot", 0.0) for output in outputs) / len(outputs) if outputs else 0.0 - latency = sum(output.get("latency", 0.0) for output in outputs) / len(outputs) if outputs else 0.0 - - response = CompletionResponse( - id=f"cmpl-{uuid.uuid4().hex}", + yield create_usage_chunk(request_id, request.model, usage) + yield "data: [DONE]\n\n" + + return StreamingResponse(generate_stream(), media_type="text/event-stream") + + # non-streaming + request_id = f"cmpl-{uuid.uuid4().hex}" + final_output = None + async for output in generate_async(request.prompt, sampling_params, request_id): + final_output = output + if final_output is None: + raise RuntimeError("No output generated") + + return CompletionResponse( + id=request_id, created=int(time.time()), model=request.model, - choices=choices, + choices=[ + { + "index": 0, + "text": final_output["text"], + "finish_reason": final_output["finish_reason"], + } + ], usage={ - "prompt_tokens": total_prompt_tokens, - "completion_tokens": total_completion_tokens, - "total_tokens": total_prompt_tokens + total_completion_tokens, - "ttft_s": ttft, - "tpot_s": tpot, - "latency_s": latency, + "prompt_tokens": final_output["num_tokens_input"], + "completion_tokens": final_output["num_tokens_output"], + "total_tokens": final_output["num_tokens_input"] + final_output["num_tokens_output"], + "ttft_s": round(final_output.get("ttft", 0.0), 4), + "tpot_s": round(final_output.get("tpot", 0.0), 4), + "latency_s": round(final_output.get("latency", 0.0), 4), }, ) - - return response - except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @app.get("/v1/models") async def list_models(): - """List available models.""" global model_name return { "object": "list", @@ -215,13 +436,11 @@ async def list_models(): @app.get("/health") async def health(): - """Health check endpoint.""" return {"status": "ok"} @app.post("/start_profile") async def start_profile(): - """Start profiling.""" global engine try: engine.start_profile() @@ -232,7 +451,6 @@ async def start_profile(): @app.post("/stop_profile") async def stop_profile(): - """Stop profiling and generate trace files.""" global engine try: engine.stop_profile() @@ -243,30 +461,26 @@ async def stop_profile(): def main(): global engine, tokenizer, model_name - parser = argparse.ArgumentParser(description="Atom OpenAI API Server") - EngineArgs.add_cli_args(parser) - parser.add_argument("--host", type=str, default="0.0.0.0", help="Server host") parser.add_argument( - "--server-port", type=int, default=8000, + "--server-port", type=int, default=8000, help="Server port (note: --port is used for internal engine communication)" ) - args = parser.parse_args() print(f"Loading tokenizer from {args.model}...") tokenizer = AutoTokenizer.from_pretrained(args.model) model_name = args.model - - print(f"Initializing engine with model {args.model}...") + + print(f"Initializing async engine with model {args.model}...") engine_args = EngineArgs.from_cli_args(args) - engine = engine_args.create_engine() - + engine = engine_args.create_async_engine() + print(f"Starting server on {args.host}:{args.server_port}...") uvicorn.run(app, host=args.host, port=args.server_port) if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/atom/examples/profile_offline.py b/atom/examples/profile_offline.py index 97dfcccb7a..3a75171c9a 100644 --- a/atom/examples/profile_offline.py +++ b/atom/examples/profile_offline.py @@ -24,6 +24,12 @@ default=32, help="Output generation length in tokens", ) +parser.add_argument( + "--bs", + type=int, + default=1, + help="Batch size (number of requests to process in parallel)", +) parser.add_argument( "--random-input", action="store_true", @@ -74,7 +80,11 @@ def main(): llm.start_profile() - outputs = llm.generate([prompt], sampling_params) + # Create batch of prompts based on batch size + prompts = [prompt] * args.bs + print(f"Processing batch of {args.bs} requests...") + + outputs = llm.generate(prompts, sampling_params) llm.stop_profile() @@ -85,9 +95,12 @@ def main(): if not args.random_input: print("Generated Output:") - for output in outputs: + for i, output in enumerate(outputs): generated_text = output['text'] - print(f"Output: {generated_text}\n") + if args.bs > 1: + print(f"Output [{i+1}/{args.bs}]: {generated_text}\n") + else: + print(f"Output: {generated_text}\n") if __name__ == "__main__": diff --git a/atom/examples/profile_online.py b/atom/examples/profile_online.py index f9a94655c9..befff5fadb 100644 --- a/atom/examples/profile_online.py +++ b/atom/examples/profile_online.py @@ -2,6 +2,7 @@ import time import argparse import random +from concurrent.futures import ThreadPoolExecutor, as_completed from transformers import AutoTokenizer @@ -75,7 +76,7 @@ def main(): help="Input prompt length in tokens (used with --random-input)", ) parser.add_argument( - "--output-len", + "--output-length", type=int, default=32, help="Number of output tokens to generate (max_tokens) (default: 32)" @@ -102,6 +103,12 @@ def main(): type=str, help="Model name or path for tokenizer (required when using --random-input)", ) + parser.add_argument( + "--bs", + type=int, + default=1, + help="Batch size (number of concurrent requests to send)", + ) args = parser.parse_args() @@ -131,9 +138,34 @@ def main(): time.sleep(2) - result = send_completion_request(base_url, input_prompt, args.output_len) - if result is None: - print("Warning: Request failed or returned no result") + # Send concurrent requests based on batch size + print(f"\nSending {args.bs} concurrent request(s)...") + results = [] + + if args.bs == 1: + # Single request + result = send_completion_request(base_url, input_prompt, args.output_length) + if result is None: + print("Warning: Request failed or returned no result") + results = [result] + else: + # Concurrent requests using ThreadPoolExecutor + with ThreadPoolExecutor(max_workers=args.bs) as executor: + # Submit all requests and track their indices + future_to_index = { + executor.submit(send_completion_request, base_url, input_prompt, args.output_length): i + 1 + for i in range(args.bs) + } + + # Process results as they complete + for future in as_completed(future_to_index): + index = future_to_index[future] + result = future.result() + if result is None: + print(f"Warning: Request {index}/{args.bs} failed or returned no result") + else: + print(f"Request {index}/{args.bs} completed") + results.append(result) time.sleep(2) @@ -144,13 +176,17 @@ def main(): return # Print output results for non-random input - if not args.random_input and result: + if not args.random_input and results: print("\n" + "=" * 60) print("Generated Output:") print("=" * 60) - if "choices" in result and len(result["choices"]) > 0: - generated_text = result["choices"][0]["text"] - print(f"Output: {generated_text}\n") + for i, result in enumerate(results): + if result and "choices" in result and len(result["choices"]) > 0: + generated_text = result["choices"][0]["text"] + if args.bs > 1: + print(f"Output [{i+1}/{args.bs}]: {generated_text}\n") + else: + print(f"Output: {generated_text}\n") print("=" * 60) diff --git a/atom/model_engine/arg_utils.py b/atom/model_engine/arg_utils.py index afe5d2ab2b..f93035718d 100644 --- a/atom/model_engine/arg_utils.py +++ b/atom/model_engine/arg_utils.py @@ -1,7 +1,7 @@ import argparse from typing import List -from atom import LLMEngine +from atom import LLMEngine, AsyncLLMEngine from atom.config import CompilationConfig @@ -167,4 +167,24 @@ def create_engine(self) -> LLMEngine: cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes) if self.cudagraph_capture_sizes else None, ), ) + + def create_async_engine(self) -> AsyncLLMEngine: + """Create and return an AsyncLLMEngine instance with the configured parameters.""" + return AsyncLLMEngine( + self.model, + enforce_eager=self.enforce_eager, + tensor_parallel_size=self.tensor_parallel_size, + kv_cache_dtype=self.kv_cache_dtype, + kv_cache_block_size=self.block_size, + max_model_len=self.max_model_len, + port=self.port, + load_dummy=self.load_dummy, + enable_expert_parallel=self.enable_expert_parallel, + torch_profiler_dir=self.torch_profiler_dir, + asyncio_mode=True, + compilation_config=CompilationConfig( + level=self.level, + cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes), + ), + ) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index ade23a6530..6e5cca5483 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -44,6 +44,8 @@ class EngineCoreRequestType(enum.Enum): EXECUTOR_FAILED = b"\x04" # Sentinel used within EngineCore. SHUTDOWN = b"\x05" + # Stream output for callbacks + STREAM = b"\x06" class EngineCore: @@ -51,6 +53,7 @@ def __init__(self, config: Config, input_address: str, output_address: str): self.label = "Engine Core" self.input_queue = queue.Queue[Sequence]() self.output_queue = queue.Queue[List[Sequence]]() + self.stream_output_queue = queue.Queue() # Queue for streaming intermediate outputs self.input_address = input_address self.output_address = output_address self.input_thread = threading.Thread( @@ -138,8 +141,20 @@ def _process_engine_step(self): scheduled_batch = self.scheduler.schedule() out = self.runner_mgr.call_func("forward", scheduled_batch, wait_out=True) seqs = scheduled_batch.seqs.values() - self.scheduler.postprocess(seqs, out) - self.output_queue.put_nowait([seq for seq in seqs if seq.is_finished]) + # Pass stream_output_queue to postprocess for streaming callbacks + finished_seqs = self.scheduler.postprocess(seqs, out, stream_output_queue=self.stream_output_queue) + + # Send stream outputs to main process via output_queue + try: + while not self.stream_output_queue.empty(): + stream_outputs = self.stream_output_queue.get_nowait() + # Send stream outputs as intermediate results + self.output_queue.put_nowait(("STREAM", stream_outputs)) + except queue.Empty: + pass + + if finished_seqs: + self.output_queue.put_nowait(finished_seqs) def pull_and_process_input_queue(self): recv_reqs = [] @@ -203,7 +218,16 @@ def process_output_sockets(self, output_address: str): logger.debug(f"{self.label}: output socket connected") while True: - seqs = self.output_queue.get() + item = self.output_queue.get() + if isinstance(item, tuple) and item[0] == "STREAM": + # Send stream outputs + stream_outputs = item[1] + serialized_obj = pickle.dumps((EngineCoreRequestType.STREAM, stream_outputs)) + socket.send(serialized_obj) + continue + + # Regular finished sequences + seqs = item valid_seqs = [ seq for seq in seqs if seq.status != SequenceStatus.EXIT_ENGINE ] diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 2975035d53..0f67cb6c71 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -1,3 +1,4 @@ +import asyncio import logging import multiprocessing import pickle @@ -14,6 +15,7 @@ from atom.config import Config from atom.model_engine.engine_core import EngineCore, EngineCoreRequestType from atom.model_engine.sequence import Sequence +from atom.model_engine.request import RequestOutput from atom.utils import ( get_open_zmq_inproc_path, get_open_zmq_ipc_path, @@ -27,9 +29,10 @@ class CoreManager: def __init__(self, config: Config): self.label = "Engine Core Mgr" - sync_ctx = zmq.Context(io_threads=2) - self.ctx = zmq.asyncio.Context(sync_ctx) if config.asyncio_mode else sync_ctx + self.ctx = zmq.Context(io_threads=2) self.outputs_queue = queue.Queue[List[Sequence]]() + self.stream_outputs_queue = queue.Queue() + self._seq_id_to_callback = {} engine_core_process, addresses = launch_engine_core(config) self.engine_core_process = engine_core_process @@ -66,7 +69,7 @@ def process_outputs_socket(): break obj = self.output_socket.recv(copy=False) - request_type, seqs = pickle.loads(obj) + request_type, data = pickle.loads(obj) if request_type == EngineCoreRequestType.SHUTDOWN: logger.debug( f"{self.label}: output thread receive SHUTDOWN request" @@ -76,8 +79,26 @@ def process_outputs_socket(): SystemExit(f"{self.label}: shutdown") ) break + elif request_type == EngineCoreRequestType.STREAM: + stream_outputs = data # List of (seq_id, RequestOutput) tuples + logger.debug(f"{self.label}: Received STREAM message with {len(stream_outputs)} outputs") + self.stream_outputs_queue.put_nowait(stream_outputs) + # Also call callbacks if registered + for seq_id, request_output in stream_outputs: + callback = self._seq_id_to_callback.get(seq_id) + logger.debug(f"{self.label}: seq_id={seq_id}, callback={'found' if callback is not None else 'NOT FOUND'}, tokens={request_output.output_tokens}") + if callback is not None: + try: + callback(request_output) + logger.debug(f"{self.label}: Successfully called callback for seq_id={seq_id}") + except Exception as e: + logger.warning(f"Error calling stream_callback for sequence {seq_id}: {e}", exc_info=True) + if request_output.finished: + self._seq_id_to_callback.pop(seq_id, None) + logger.debug(f"{self.label}: Cleaned up callback for finished sequence {seq_id}") elif request_type == EngineCoreRequestType.ADD: # logger.info(f"Engine core output sequence id: {seq.id}") + seqs = data self.outputs_queue.put_nowait(seqs) finally: # Close sockets. @@ -91,6 +112,44 @@ def process_outputs_socket(): ) self.output_queue_thread.start() self._finalizer = weakref.finalize(self, self.close) + self.async_output_queue = asyncio.Queue() if config.asyncio_mode else None + self._output_handler_task = None + self._asyncio_mode = config.asyncio_mode + + def _ensure_output_handler_task(self): + if self._asyncio_mode and self._output_handler_task is None: + try: + loop = asyncio.get_running_loop() + self._output_handler_task = loop.create_task(self._async_output_handler()) + except RuntimeError: + # If no running event loop, try to get/create one + loop = asyncio.get_event_loop() + if loop.is_running(): + self._output_handler_task = loop.create_task(self._async_output_handler()) + else: + raise RuntimeError("CoreManager with asyncio_mode requires a running event loop") + + async def _async_output_handler(self): + loop = asyncio.get_event_loop() + while True: + # Use run_in_executor to avoid blocking event loop + seqs = await loop.run_in_executor(None, self.outputs_queue.get) + if isinstance(seqs, BaseException): + await self.async_output_queue.put(seqs) + break + await self.async_output_queue.put(seqs) + + async def get_output_async(self) -> List[Sequence]: + if not self.async_output_queue: + raise RuntimeError("Engine async mode not enabled") + + # Ensure output handler task is started + self._ensure_output_handler_task() + + seqs = await self.async_output_queue.get() + if isinstance(seqs, BaseException): + raise seqs + return seqs def close(self): self._shutdown_engine_core() @@ -104,11 +163,22 @@ def close(self): def add_request(self, seqs: List[Sequence]): logger.debug(f"{self.label}: Add request, sequence ids: {[seq.id for seq in seqs]}") + # Register callbacks before sending to engine core + for seq in seqs: + if seq.stream_callback is not None: + self._seq_id_to_callback[seq.id] = seq.stream_callback + seq.stream_callback = None self.input_socket.send_multipart( [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.ADD, seqs))], copy=False, ) + def get_stream_outputs(self): + try: + return self.stream_outputs_queue.get_nowait() + except queue.Empty: + return None + def send_utility_command(self, cmd: str): logger.debug(f"{self.label}: Send utility command: {cmd}") self.input_socket.send_multipart( diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 36f8bdb3c1..882f0b8d20 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -2,8 +2,9 @@ import itertools import logging import time +import asyncio from dataclasses import fields -from typing import List, Union +from typing import List, Union, AsyncGenerator, Dict, Optional import torch.multiprocessing as mp from tqdm.auto import tqdm @@ -39,7 +40,8 @@ def __init__(self, model, **kwargs): def add_request( self, prompt_or_tokens_list: List[Union[str, List[int]]], - sampling_params_list: SamplingParams | List[SamplingParams] + sampling_params_list: SamplingParams | List[SamplingParams], + stream_callback=None ): # if sampling params is not list, use it for all prompts if not isinstance(sampling_params_list, list): @@ -53,9 +55,22 @@ def add_request( ) sampling_params_iter = sampling_params_list + # Handle stream_callback + if stream_callback is not None and not isinstance(stream_callback, list): + stream_callback_iter = itertools.repeat(stream_callback) + elif isinstance(stream_callback, list): + if len(stream_callback) != len(prompt_or_tokens_list): + raise ValueError( + f"number of elements in prompt_or_tokens_list and stream_callback is different: " + f"{len(prompt_or_tokens_list)=} vs {len(stream_callback)=}" + ) + stream_callback_iter = stream_callback + else: + stream_callback_iter = itertools.repeat(None) + reqs = [] - for prompt, sampling_param in zip(prompt_or_tokens_list, sampling_params_iter): - req = self.io_processor.preprocess(prompt, sampling_param) + for prompt, sampling_param, callback in zip(prompt_or_tokens_list, sampling_params_iter, stream_callback_iter): + req = self.io_processor.preprocess(prompt, sampling_param, stream_callback=callback) reqs.append(req) self.core_mgr.add_request(reqs) @@ -84,49 +99,6 @@ def generate( outputs = [outputs[seq_id] for seq_id in sorted(outputs)] return outputs - async def generate_async( - self, - prompt: str, - sampling_params: SamplingParams, - ): - # Initialize lock on first use - if self._step_lock is None: - self._step_lock = asyncio.Lock() - - # Add the request and get its sequence ID - req = self.io_processor.preprocess(prompt, sampling_params) - seq_id = req.id - self.core_mgr.add_request([req]) - - while True: - # Check if result is already available - if seq_id in self._pending_results: - result = self._pending_results.pop(seq_id) - yield result - return - - if seq_id not in self.io_processor.requests: - break - - if not (self.core_mgr.is_alive() or self.core_mgr.is_rest()): - break - - # Coordinate step() calls across all concurrent requests - async with self._step_lock: - seqs = self.step() - outs = self.io_processor.postprocess(seqs) - - # Store all results so other waiting tasks can find them - self._pending_results.update(outs) - - if seq_id in self._pending_results: - result = self._pending_results.pop(seq_id) - yield result - return - - # Let other tasks run before trying again - await asyncio.sleep(0) - def start_profile(self): self.core_mgr.send_utility_command("start_profile") logger.info("Profiling started") @@ -144,7 +116,7 @@ def __init__(self, tokenizer, block_size): self.requests = {} def preprocess( - self, prompt_or_tokens: str | list[int], sampling_params: SamplingParams + self, prompt_or_tokens: str | list[int], sampling_params: SamplingParams, stream_callback=None ): """responsible for: 1) Tokenize @@ -164,7 +136,7 @@ def preprocess( if stop_tokens: stop_token_sequences.append(stop_tokens) - seq = Sequence(tokens, self.block_size, sampling_params, stop_token_sequences) + seq = Sequence(tokens, self.block_size, sampling_params, stop_token_sequences, stream_callback=stream_callback) seq.arrive_time = time.time() self.requests[seq.id] = seq print( @@ -211,3 +183,160 @@ def postprocess(self, reqs: List[Sequence]): def has_pending_requests(self): return len(self.requests) > 0 + + +class AsyncLLMEngine(LLMEngine): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._request_queues: Dict[str, asyncio.Queue] = {} + self._seq_id_to_request_id: Dict[int, str] = {} + self._running = True + self._output_task = None + + def _ensure_output_task(self): + if self._output_task is None: + try: + loop = asyncio.get_running_loop() + self._output_task = loop.create_task(self._process_outputs_loop()) + except RuntimeError: + loop = asyncio.get_event_loop() + if loop.is_running(): + self._output_task = loop.create_task(self._process_outputs_loop()) + else: + raise RuntimeError("AsyncLLMEngine requires a running event loop") + + async def _process_outputs_loop(self): + while self._running: + try: + try: + seqs = await asyncio.wait_for( + self.core_mgr.get_output_async(), + timeout=1.0 + ) + except asyncio.TimeoutError: + continue + + for seq in seqs: + request_id = self._seq_id_to_request_id.get(seq.id) + if request_id and request_id in self._request_queues: + try: + self._request_queues[request_id].put_nowait(seq) + except asyncio.QueueFull: + try: + old_seq = self._request_queues[request_id].get_nowait() + logger.debug(f"Request {request_id} queue full, dropping old result and adding new one") + except asyncio.QueueEmpty: + pass + try: + self._request_queues[request_id].put_nowait(seq) + except asyncio.QueueFull: + logger.warning(f"Request {request_id} queue still full, result lost") + else: + logger.debug(f"Sequence {seq.id} request queue does not exist (request_id={request_id}), ignoring result") + + except Exception as e: + logger.error(f"Error in output processing loop: {e}", exc_info=True) + await asyncio.sleep(0.1) + + async def generate_async( + self, + prompt: str, + sampling_params: SamplingParams, + request_id: str + ) -> AsyncGenerator[Dict, None]: + self._ensure_output_task() + + output_queue = asyncio.Queue(maxsize=1) + self._request_queues[request_id] = output_queue + + seq = None + seq_id = None + try: + req = self.io_processor.preprocess(prompt, sampling_params) + seq_id = req.id + if seq_id in self._seq_id_to_request_id: + logger.warning(f"Sequence ID {seq_id} already exists in mapping, overwriting old mapping") + self._seq_id_to_request_id[seq_id] = request_id + self.core_mgr.add_request([req]) + + while True: + seq = await output_queue.get() + + if seq.id != seq_id: + logger.error(f"Received wrong sequence ID: expected {seq_id}, got {seq.id}, request_id={request_id}") + continue + + result = self._postprocess_sequence(seq) + + if seq.is_finished: + yield result + break + + yield result + + finally: + self._request_queues.pop(request_id, None) + if seq_id is not None: + self._seq_id_to_request_id.pop(seq_id, None) + # Clean up pending request if sequence didn't finish normally + if seq is None or not seq.is_finished: + self.io_processor.requests.pop(seq_id, None) + + #TODO add tokenizer process to handle tokens combination for utf8 decoding + def _postprocess_sequence(self, seq: Sequence) -> Dict: + # Set leave_time when sequence is finished + if seq.is_finished and seq.leave_time == 0.0: + seq.leave_time = time.time() + + # Remove from pending requests when sequence is finished + if seq.is_finished: + self.io_processor.requests.pop(seq.id, None) + + ttft = 0.0 + tpot = 0.0 + latency = 0.0 + + if seq.first_token_time > 0: + ttft = seq.first_token_time - seq.arrive_time + if seq.num_completion_tokens > 1: + leave_time = seq.leave_time if seq.leave_time > 0 else time.time() + tpot = (leave_time - seq.first_token_time) / (seq.num_completion_tokens - 1) + + # Calculate latency if sequence is finished + if seq.is_finished and seq.leave_time > 0: + latency = seq.leave_time - seq.arrive_time + + return { + "text": self.tokenizer.decode(seq.completion_token_ids, skip_special_tokens=True), + "token_ids": seq.completion_token_ids.copy(), + "finished": seq.is_finished, + "finish_reason": getattr(seq, "leave_reason", None), + "ttft": ttft, + "tpot": tpot, + "latency": latency, + } + + #TODO enable abort request + # def abort_request(self, request_id: str): + # if request_id in self._request_queues: + # # Find seq_id from request_id mapping + # seq_id = None + # for sid, rid in self._seq_id_to_request_id.items(): + # if rid == request_id: + # seq_id = sid + # break + # # Remove from pending requests using seq_id + # if seq_id is not None: + # self.io_processor.requests.pop(seq_id, None) + # self._request_queues.pop(request_id, None) + # logger.info(f"Request {request_id} aborted") + + async def close(self): + self._running = False + if self._output_task is not None: + self._output_task.cancel() + try: + await self._output_task + except asyncio.CancelledError: + pass + self.core_mgr.close() \ No newline at end of file diff --git a/atom/model_engine/request.py b/atom/model_engine/request.py new file mode 100644 index 0000000000..1fcaabe3ff --- /dev/null +++ b/atom/model_engine/request.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class RequestOutput: + """Output structure passed to stream callback.""" + request_id: int # Sequence ID + output_tokens: List[int] # Newly generated tokens since last callback + finished: bool # Whether the sequence is finished + finish_reason: Optional[str] = None # Reason for finishing (eos, max_tokens, stop_sequence, etc.) + diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index ffd3fb1fca..50c6ffcbe2 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -2,13 +2,14 @@ import time from collections import deque from dataclasses import dataclass, field -from typing import Any, Optional, cast +from typing import Any, Optional, cast, List import numpy as np import torch from atom.config import Config from atom.model_engine.block_manager import BlockManager from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType +from atom.model_engine.request import RequestOutput logger = logging.getLogger("atom") @@ -149,18 +150,29 @@ def preempt(self, seq: Sequence): self.block_manager.deallocate(seq) self.waiting.appendleft(seq) - def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): - is_deferred_out = prev_token_ids[-1] + def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int], stream_output_queue=None) -> list[Sequence]: + is_deferred_out = prev_token_ids.get(-1, False) # update token_ids with the actual sampled token ids finished_seqs = [] + stream_outputs = [] + for seq in self.running: if seq.id not in prev_token_ids: continue token_id = prev_token_ids[seq.id] + new_tokens = [] if is_deferred_out: seq.token_ids[-1] = token_id + + if seq.output_tokens: + seq.output_tokens[-1] = token_id + new_tokens = [token_id] + else: + seq.output_tokens.append(token_id) + new_tokens = [token_id] else: seq.append_token(token_id) + new_tokens = [token_id] if seq.num_completion_tokens == 1 and seq.first_token_time == 0.0: seq.first_token_time = time.time() @@ -177,10 +189,26 @@ def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): leave_reason = "eos" elif seq.num_completion_tokens == seq.max_tokens: leave_reason = "max_tokens" + # Prepare stream output + if stream_output_queue is not None and new_tokens: + request_output = RequestOutput( + request_id=seq.id, + output_tokens=new_tokens.copy(), + finished=(leave_reason is not None), + finish_reason=leave_reason + ) + # Store sequence ID instead of sequence object to avoid pickling issues + stream_outputs.append((seq.id, request_output)) + logger.debug(f"Scheduler: Created stream output for seq_id={seq.id}, tokens={new_tokens}, finished={leave_reason is not None}") + if leave_reason is not None: seq.leave_reason = leave_reason seq.status = SequenceStatus.FINISHED finished_seqs.append(seq) + + if stream_output_queue is not None and stream_outputs: + stream_output_queue.put_nowait(stream_outputs) + if is_deferred_out: # placeholder for the each decode step for seq in seqs: @@ -189,3 +217,4 @@ def postprocess(self, seqs: list[Sequence], prev_token_ids: dict[int, int]): for seq in finished_seqs: self.block_manager.deallocate(seq) self.running.remove(seq) + return finished_seqs diff --git a/atom/model_engine/sequence.py b/atom/model_engine/sequence.py index 1949554050..a30359bf84 100644 --- a/atom/model_engine/sequence.py +++ b/atom/model_engine/sequence.py @@ -1,6 +1,7 @@ from copy import copy from enum import Enum, auto from itertools import count +from typing import Optional, Callable, Any from atom.sampling_params import SamplingParams @@ -28,7 +29,12 @@ class Sequence: counter = count() def __init__( - self, token_ids: list[int], block_size: int, sampling_params=SamplingParams(), stop_token_sequences: list[list[int]] = None + self, + token_ids: list[int], + block_size: int, + sampling_params=SamplingParams(), + stop_token_sequences: list[list[int]] = None, + stream_callback: Optional[Callable[[Any], None]] = None ): self.block_size = block_size self.id = next(Sequence.counter) @@ -46,6 +52,10 @@ def __init__( self.stop_strings = sampling_params.stop_strings self.stop_token_sequences = stop_token_sequences or [] + # stream callback + self.stream_callback = stream_callback + self.output_tokens = [] # cache for newly generate tokens + # statistics fields self.arrive_time = 0.0 self.first_token_time = 0.0 @@ -104,6 +114,7 @@ def block(self, i): def append_token(self, token_id: int): self.token_ids.append(token_id) self.last_token = token_id + self.output_tokens.append(token_id) self.num_tokens += 1 # def __getstate__(self): From e1a306c9ff6835819d2564315c8b6310835984aa Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 19 Nov 2025 13:43:59 +0800 Subject: [PATCH 114/119] support dp in atom and dp attention+tp/ep moe --- atom/config.py | 123 +++++++++ atom/model_engine/arg_utils.py | 21 ++ atom/model_engine/engine_core.py | 105 ++++++- atom/model_engine/engine_core_mgr.py | 253 +++++++++++++---- atom/model_engine/llm_engine.py | 7 +- atom/model_engine/model_runner.py | 150 +++++++++- atom/model_engine/scheduler.py | 3 +- atom/model_ops/attention_mla.py | 34 ++- atom/model_ops/embed_head.py | 2 +- atom/model_ops/linear.py | 4 +- atom/model_ops/moe.py | 394 +++++++++++++++++++++++++-- atom/utils/__init__.py | 71 +++-- atom/utils/backends.py | 2 +- atom/utils/cuda_piecewise_backend.py | 2 +- atom/utils/decorators.py | 36 ++- atom/utils/distributed/utils.py | 136 +++++++++ atom/utils/envs.py | 18 ++ atom/utils/forward_context.py | 117 +++++++- 18 files changed, 1335 insertions(+), 143 deletions(-) create mode 100644 atom/utils/distributed/utils.py create mode 100644 atom/utils/envs.py diff --git a/atom/config.py b/atom/config.py index bb8af34905..b47c1d40b6 100644 --- a/atom/config.py +++ b/atom/config.py @@ -1,14 +1,22 @@ +import logging import enum import hashlib import os import re from dataclasses import dataclass, field from typing import Any, ClassVar, Optional, Union +from atom.utils import envs +from torch.distributed import ProcessGroup, ReduceOp +from atom.utils.distributed.utils import stateless_init_torch_distributed_process_group import torch from aiter import QuantType from aiter.utility.dtypes import d_dtypes from transformers import AutoConfig, PretrainedConfig +from aiter.dist.parallel_state import get_dp_group + + +logger = logging.getLogger("atom") @dataclass class KVCacheTensor: @@ -359,6 +367,112 @@ def _get_hf_token() -> str | None: return AutoConfig.from_pretrained(model) +@dataclass +class ParallelConfig: + data_parallel_size: int = 1 + """Number of data parallel groups. MoE layers will be sharded according to + the product of the tensor parallel size and data parallel size.""" + data_parallel_size_local: int = 1 + """Number of local data parallel groups.""" + data_parallel_rank: int = 0 + """Rank of the data parallel group.""" + data_parallel_rank_local: Optional[int] = None + """Local rank of the data parallel group, + set only in SPMD mode.""" + world_size: int = field(init=False) + """world_size is TPxPP, it affects the number of workers we create.""" + data_parallel_master_port: int = 29500 + """Port of the data parallel master.""" + + data_parallel_base_port: int = 29400 + + data_parallel_master_ip: str = "127.0.0.1" + + + @property + def world_size_across_dp(self) -> int: + """world_size_across_dp is TPxPPxDP, it is the size of the world + including data parallelism.""" + return self.world_size * self.data_parallel_size + + def get_next_dp_init_port(self) -> int: + """ + We might need to initialize process groups in multiple + processes that is related to data parallelism, + e.g. both in the worker and in the engine, which + can live in different processes. To avoid port conflicts, we + pop a new port from the prepared port list each time we need to + initialize a new process group related to data parallelism. + """ + answer = self.data_parallel_master_port + self.data_parallel_master_port += self.data_parallel_rank + + return answer + + def stateless_init_dp_group(self): + # NOTE: In high-concurrency scenarios multiple processes + # can pick the same (currently free) port through a race + # condition when calling `get_open_port()`. When the first + # process binds the port the others will subsequently fail + # with `torch.distributed.DistNetworkError: EADDRINUSE`. + # To make the initialization more robust we retry a few times + # with a fresh port whenever this specific error is observed. + dp_group = stateless_init_torch_distributed_process_group( + self.data_parallel_master_ip, + self.get_next_dp_init_port(), + self.data_parallel_rank, + self.data_parallel_size, + backend="gloo") + return dp_group + + + @staticmethod + def has_unfinished_dp(dp_group: ProcessGroup, + has_unfinished: bool) -> bool: + tensor = torch.tensor([has_unfinished], + dtype=torch.int32, + device="cpu") + # dp rank 0: has_unfinished_seqs=True + # dp rank 1: has_unfinished_seqs=False + # aggregated: has_unfinished_seqs=True + # so this is an OR operation, i.e. MAX in integers + # torch.distributed.all_reduce(tensor, op=ReduceOp.MAX, group=dp_group) + # from aiter.dist.parallel_state import get_dp_group + torch.distributed.all_reduce(tensor, op=ReduceOp.MAX, group=dp_group) + aggregated_has_unfinished = bool(tensor.item()) + return aggregated_has_unfinished + + def compute_hash(self): + """ + Provide a hash that uniquely identifies all the configs + that affect the structure of the computation + graph from input ids/embeddings to the final hidden states, + excluding anything before input ids/embeddings and after + the final hidden states. + """ + factors: list[Any] = [] + factors.append(self.data_parallel_size) + factors.append(self.data_parallel_rank) + factors.append(self.data_parallel_rank_local) + factors.append(self.data_parallel_master_ip) + factors.append(self.data_parallel_master_port) + return hashlib.sha256(str(factors).encode()).hexdigest() + + def __post_init__(self) -> None: + # Only override with env vars if not already set to non-default values + # This allows programmatic configuration to take precedence + import os + if os.getenv("ATOM_DP_SIZE") is not None: + self.data_parallel_size = envs.ATOM_DP_SIZE + if os.getenv("ATOM_DP_RANK") is not None: + self.data_parallel_rank = envs.ATOM_DP_RANK + if os.getenv("ATOM_DP_RANK_LOCAL") is not None: + self.data_parallel_rank_local = envs.ATOM_DP_RANK_LOCAL + # self.data_parallel_master_ip = envs.ATOM_DP_MASTER_IP + # self.data_parallel_master_port = envs.ATOM_DP_MASTER_PORT + + + @dataclass class Config: model: str @@ -369,6 +483,7 @@ class Config: tensor_parallel_size: int = 1 enforce_eager: bool = False hf_config: PretrainedConfig = field(init=False) + parallel_config: ParallelConfig = field(default_factory=ParallelConfig) bos_token_id: int = -1 eos_token_id: int = -1 kv_cache_block_size: int = 16 @@ -386,6 +501,7 @@ class Config: enable_expert_parallel: bool = False master_addr: str = "127.0.0.1" graph_bs: Optional[list[int]] = None + enable_dp_attention: bool = False def _set_cudagraph_sizes(self): if self.compilation_config.cudagraph_capture_sizes: @@ -443,9 +559,13 @@ def compute_hash(self) -> str: if self.compilation_config: vllm_factors.append(self.compilation_config.compute_hash()) + + if self.parallel_config: + vllm_factors.append(self.parallel_config.compute_hash()) factors.append(vllm_factors) factors.append(self.tensor_parallel_size) + factors.append(self.enable_dp_attention) hash_str = hashlib.md5( str(factors).encode(), usedforsecurity=False @@ -459,6 +579,9 @@ def compute_hash(self) -> str: def set_current_atom_config(atom_config: Config): global _current_atom_config _current_atom_config = atom_config + # for MoE to check + import os + os.environ["ATOM_ENFORCE_EAGER"] = "1" if atom_config.enforce_eager else "0" def get_current_atom_config() -> Config: diff --git a/atom/model_engine/arg_utils.py b/atom/model_engine/arg_utils.py index f93035718d..8505f1c731 100644 --- a/atom/model_engine/arg_utils.py +++ b/atom/model_engine/arg_utils.py @@ -22,6 +22,7 @@ def __init__( self, model: str = "Qwen/Qwen3-0.6B", tensor_parallel_size: int = 1, + data_parallel_size: int = 1, enforce_eager: bool = False, enable_prefix_caching: bool = False, port: int = 8006, @@ -33,6 +34,7 @@ def __init__( load_dummy: bool = False, enable_expert_parallel: bool = False, torch_profiler_dir: str = None, + enable_dp_attention: bool = False, ): self.model = model self.tensor_parallel_size = tensor_parallel_size @@ -47,6 +49,8 @@ def __init__( self.load_dummy = load_dummy self.enable_expert_parallel = enable_expert_parallel self.torch_profiler_dir = torch_profiler_dir + self.enable_dp_attention = enable_dp_attention + self.data_parallel_size = data_parallel_size @staticmethod def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: @@ -128,6 +132,19 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: help="Directory to save torch profiler traces", ) + parser.add_argument( + "--data-parallel-size", + "-dp", + type=int, + default=1, + help="Data parallel size.", + ) + + parser.add_argument( + "--enable-dp-attention", + action="store_true", + help="Enable DP attention.", + ) return parser @classmethod @@ -136,6 +153,7 @@ def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs": return cls( model=args.model, tensor_parallel_size=args.tensor_parallel_size, + data_parallel_size=getattr(args, "data_parallel_size", 1), enforce_eager=args.enforce_eager, enable_prefix_caching=args.enable_prefix_caching, port=args.port, @@ -147,6 +165,7 @@ def from_cli_args(cls, args: argparse.Namespace) -> "EngineArgs": load_dummy=args.load_dummy, enable_expert_parallel=args.enable_expert_parallel, torch_profiler_dir=args.torch_profiler_dir, + enable_dp_attention=args.enable_dp_attention, ) def create_engine(self) -> LLMEngine: @@ -186,5 +205,7 @@ def create_async_engine(self) -> AsyncLLMEngine: level=self.level, cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes), ), + data_parallel_size=self.data_parallel_size, + enable_dp_attention=self.enable_dp_attention, ) diff --git a/atom/model_engine/engine_core.py b/atom/model_engine/engine_core.py index 6e5cca5483..a65705a098 100644 --- a/atom/model_engine/engine_core.py +++ b/atom/model_engine/engine_core.py @@ -5,9 +5,11 @@ import signal import threading import weakref +import time from contextlib import ExitStack from multiprocessing.shared_memory import SharedMemory from typing import Any, List, Optional +import torch import zmq import zmq.asyncio @@ -26,6 +28,8 @@ shutdown_all_processes, zmq_socket_ctx, ) +from atom.utils.distributed.utils import stateless_destroy_torch_distributed_process_group +from atom.config import Config, ParallelConfig logger = logging.getLogger("atom") @@ -67,6 +71,7 @@ def __init__(self, config: Config, input_address: str, output_address: str): self.profile_enbaled = config.torch_profiler_dir is not None init_exit_handler(self) + self._init_data_parallel(config) # Initialize model runner processes try: @@ -101,6 +106,9 @@ def __init__(self, config: Config, input_address: str, output_address: str): self.scheduler = Scheduler(config) + def _init_data_parallel(self, config: Config): + pass + def exit(self): if not self.still_running: return @@ -119,7 +127,10 @@ def _send_engine_dead(self): def run_engine(config: Config, input_address: str, output_address: str): engine: EngineCore = None try: - engine = EngineCore(config, input_address, output_address) + if config.parallel_config.data_parallel_size > 1: + engine = DPEngineCoreProc(config, input_address, output_address) + else: + engine = EngineCore(config, input_address, output_address) engine.busy_loop() except Exception as e: logger.error(f"run_engine: exception: {e}", exc_info=True) @@ -139,6 +150,9 @@ def busy_loop(self): def _process_engine_step(self): scheduled_batch = self.scheduler.schedule() + if scheduled_batch is None: + logger.debug(f"{self.label}: No sequences to schedule, skipping forward") + return False out = self.runner_mgr.call_func("forward", scheduled_batch, wait_out=True) seqs = scheduled_batch.seqs.values() # Pass stream_output_queue to postprocess for streaming callbacks @@ -155,6 +169,7 @@ def _process_engine_step(self): if finished_seqs: self.output_queue.put_nowait(finished_seqs) + return True def pull_and_process_input_queue(self): recv_reqs = [] @@ -252,3 +267,91 @@ def stop_profiler(self): print("Stopping profiler...") self.runner_mgr.call_func("stop_profiler", wait_out=True) print("Profiler stopped.") + + +class DPEngineCoreProc(EngineCore): + def __init__(self, config: Config, input_address: str, output_address: str): + # self.dp_group = config.parallel_config.dp_group + self.dp_rank = config.parallel_config.data_parallel_rank + # self.dp_group = config.parallel_config.stateless_init_dp_group() + super().__init__(config, input_address, output_address) + # Initialize to True so first iteration reaches all_reduce + self.engines_running = True + self._shutting_down = False + + def _init_data_parallel(self, config: Config): + dp_rank = config.parallel_config.data_parallel_rank + dp_size = config.parallel_config.data_parallel_size + local_dp_rank = config.parallel_config.data_parallel_rank_local + + assert dp_size > 1 + assert local_dp_rank is not None + assert 0 <= local_dp_rank <= dp_rank < dp_size + + self.dp_rank = dp_rank + self.dp_group = config.parallel_config.stateless_init_dp_group() + + def exit(self): + super().exit() + if dp_group := getattr(self, "dp_group", None): + stateless_destroy_torch_distributed_process_group(dp_group) + + def busy_loop(self): + shutdown = False + while True: + local_is_unfinished = not self.scheduler.is_finished() + + # Synchronize shutdown state across all DP ranks + # This ensures all ranks know when any rank wants to shutdown + if not self._shutting_down: + global_should_shutdown = self._sync_shutdown_state(shutdown and not local_is_unfinished) + if global_should_shutdown: + self._shutting_down = True + logger.debug(f"{self.label}: Entering shutdown phase (synchronized across DP)") + + self.engines_running = self._has_global_unfinished_reqs(local_is_unfinished) + logger.debug(f"{self.label}: [Sync] engines_running={self.engines_running}") + + if not self.engines_running: + logger.debug(f"{self.label}: All DP ranks finished") + if shutdown: + break + shutdown = shutdown or self.pull_and_process_input_queue() + continue + + shutdown = shutdown or self.pull_and_process_input_queue() + + executed = self._process_engine_step() + + if not executed: + logger.debug(f"{self.label}: Executing dummy batch for DP sync") + # we must have dummy execution to avoid deadlock in other ranks + self._execute_dummy_batch() + + def _execute_dummy_batch(self): + return self.runner_mgr.call_func("dummy_execution", wait_out=True) + + def _sync_shutdown_state(self, local_should_shutdown: bool) -> bool: + try: + tensor = torch.tensor([local_should_shutdown], + dtype=torch.int32, + device="cpu") + torch.distributed.all_reduce(tensor, op=torch.distributed.ReduceOp.MAX, group=self.dp_group) + global_should_shutdown = bool(tensor.item()) + return global_should_shutdown + except RuntimeError as e: + # If all_reduce fails, it means other ranks are shutting down + logger.warning(f"{self.label}: Shutdown sync failed, assuming shutdown: {e}") + return True + + def _has_global_unfinished_reqs(self, local_unfinished: bool) -> bool: + if self._shutting_down: + logger.info(f"{self.label}: Skipping DP sync during shutdown") + return local_unfinished + try: + return ParallelConfig.has_unfinished_dp(self.dp_group, + local_unfinished) + except RuntimeError as e: + # Handle case where other ranks have already shut down + logger.warning(f"{self.label}: DP sync failed during shutdown: {e}") + return local_unfinished \ No newline at end of file diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 0f67cb6c71..3e7748b02c 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -20,6 +20,7 @@ get_open_zmq_inproc_path, get_open_zmq_ipc_path, make_zmq_socket, + set_device_control_env_var, shutdown_all_processes, ) @@ -29,34 +30,113 @@ class CoreManager: def __init__(self, config: Config): self.label = "Engine Core Mgr" + self._closed = False # Track whether already closed + if config.enable_dp_attention: + self.local_engine_count = config.tensor_parallel_size * config.parallel_config.data_parallel_size + logger.info(f"Enable dp attention, using {self.local_engine_count} data parallel ranks") + config.parallel_config.data_parallel_size = self.local_engine_count + config.tensor_parallel_size = 1 + else: + self.local_engine_count = config.parallel_config.data_parallel_size self.ctx = zmq.Context(io_threads=2) self.outputs_queue = queue.Queue[List[Sequence]]() self.stream_outputs_queue = queue.Queue() self._seq_id_to_callback = {} - engine_core_process, addresses = launch_engine_core(config) - self.engine_core_process = engine_core_process + self.engine_core_processes = [] + self.input_sockets = [] + self.output_sockets = [] + self.engine_core_identities = [] + self.shutdown_paths = [] + self.output_threads = [] - input_address = addresses["input_address"] - output_address = addresses["output_address"] - self.input_socket = make_zmq_socket( - self.ctx, input_address, zmq.ROUTER, bind=True - ) - identity, _ = self.input_socket.recv_multipart() - self.engine_core_identity = identity - self.output_socket = make_zmq_socket(self.ctx, output_address, zmq.PULL) + import torch + if torch.multiprocessing.get_start_method(allow_none=True) is None: + torch.multiprocessing.set_start_method("spawn", force=False) + + processes_info = [] + local_dp_ranks = [] + + try: + for dp_rank in range(self.local_engine_count): + logger.info(f"{self.label}: Creating EngineCore for DP rank {dp_rank}/{self.local_engine_count}") + + # Create config for this DP rank + import copy + rank_config = copy.deepcopy(config) + rank_config.parallel_config.data_parallel_rank = dp_rank + + engine_core_process, addresses, local_dp_rank = launch_engine_core(rank_config, dp_rank) + + processes_info.append({ + 'process': engine_core_process, + 'addresses': addresses, + 'dp_rank': dp_rank, + 'config': rank_config + }) + local_dp_ranks.append(local_dp_rank) + + data_parallel = config.parallel_config.data_parallel_size > 1 + try: + for info, local_dp_rank in zip(processes_info, local_dp_ranks): + dp_rank = info['dp_rank'] + logger.info(f"{self.label}: Starting EngineCore for DP rank {dp_rank}/{self.local_engine_count}") + + if data_parallel: + with set_device_control_env_var(info['config'], local_dp_rank): + info['process'].start() + else: + info['process'].start() + + self.engine_core_processes.append(info['process']) + + input_address = info['addresses']["input_address"] + input_socket = make_zmq_socket( + self.ctx, input_address, zmq.ROUTER, bind=True + ) + identity, _ = input_socket.recv_multipart() + self.input_sockets.append(input_socket) + self.engine_core_identities.append(identity) + + output_address = info['addresses']["output_address"] + output_socket = make_zmq_socket(self.ctx, output_address, zmq.PULL) + self.output_sockets.append(output_socket) - shutdown_path = get_open_zmq_inproc_path() - self.shutdown_path = shutdown_path + shutdown_path = get_open_zmq_inproc_path() + self.shutdown_paths.append(shutdown_path) + output_thread = self._create_output_thread(dp_rank, output_socket, shutdown_path) + output_thread.start() + self.output_threads.append(output_thread) + + logger.info(f"{self.label}: DP rank {dp_rank} initialized successfully") + finally: + if self.finished_procs(): + logger.error(f"{self.label}: Some processes failed to start, shutting down all") + self.close() + raise RuntimeError("Failed to start all EngineCore processes") + + except Exception as e: + logger.error(f"{self.label}: Failed to initialize all EngineCores, cleaning up: {e}") + self.close() + raise + + logger.info(f"{self.label}: All {self.local_engine_count} EngineCores initialized") + self._finalizer = weakref.finalize(self, self.close) + self.async_output_queue = asyncio.Queue() if config.asyncio_mode else None + self._output_handler_task = None + self._asyncio_mode = config.asyncio_mode + + + def _create_output_thread(self, dp_rank: int, output_socket: zmq.Socket, shutdown_path: str) -> Thread: def process_outputs_socket(): - assert isinstance(self.output_socket, zmq.Socket) + assert isinstance(output_socket, zmq.Socket) shutdown_socket = self.ctx.socket(zmq.PAIR) try: shutdown_socket.bind(shutdown_path) poller = zmq.Poller() poller.register(shutdown_socket, zmq.POLLIN) - poller.register(self.output_socket, zmq.POLLIN) - logger.debug(f"{self.label}: output thread started") + poller.register(output_socket, zmq.POLLIN) + logger.debug(f"{self.label} (DP {dp_rank}): output thread started") while True: socks = poller.poll() if not socks: @@ -64,20 +144,17 @@ def process_outputs_socket(): if len(socks) == 2 or socks[0][0] == shutdown_socket: # shutdown signal, exit thread. logger.debug( - f"{self.label}: output thread receive shutdown signal" + f"{self.label} (DP {dp_rank}): output thread receive shutdown signal" ) break - obj = self.output_socket.recv(copy=False) + obj = output_socket.recv(copy=False) request_type, data = pickle.loads(obj) if request_type == EngineCoreRequestType.SHUTDOWN: logger.debug( - f"{self.label}: output thread receive SHUTDOWN request" - ) - self._shutdown_engine_core() - self.outputs_queue.put_nowait( - SystemExit(f"{self.label}: shutdown") + f"{self.label} (DP {dp_rank}): output thread receive SHUTDOWN request" ) + self._shutdown_engine_core_rank(dp_rank) break elif request_type == EngineCoreRequestType.STREAM: stream_outputs = data # List of (seq_id, RequestOutput) tuples @@ -103,18 +180,15 @@ def process_outputs_socket(): finally: # Close sockets. shutdown_socket.close(linger=0) - self.output_socket.close(linger=0) + output_socket.close(linger=0) + - self.output_queue_thread = Thread( + return Thread( target=process_outputs_socket, - name="EngineCoreOutputQueueThread", + name=f"EngineCoreOutputThread-DP{dp_rank}", daemon=True, ) - self.output_queue_thread.start() - self._finalizer = weakref.finalize(self, self.close) - self.async_output_queue = asyncio.Queue() if config.asyncio_mode else None - self._output_handler_task = None - self._asyncio_mode = config.asyncio_mode + def _ensure_output_handler_task(self): if self._asyncio_mode and self._output_handler_task is None: @@ -152,14 +226,34 @@ async def get_output_async(self) -> List[Sequence]: return seqs def close(self): - self._shutdown_engine_core() - self.input_socket.close() - if self.shutdown_path and self.output_queue_thread: - with self.ctx.socket(zmq.PAIR) as shutdown_sender: - shutdown_sender.connect(self.shutdown_path) - # Send shutdown signal. - shutdown_sender.send(b"") - self.output_queue_thread.join() + if self._closed: + return + self._closed = True + + logger.info(f"{self.label}: Shutting down all {self.local_engine_count} EngineCores") + + for dp_rank in range(self.local_engine_count): + self._shutdown_engine_core_rank(dp_rank) + + for input_socket in self.input_sockets: + if not input_socket.closed: + input_socket.close() + + for shutdown_path in self.shutdown_paths: + if shutdown_path: + try: + with self.ctx.socket(zmq.PAIR) as shutdown_sender: + shutdown_sender.connect(shutdown_path) + shutdown_sender.send(b"") + except Exception as e: + logger.debug(f"{self.label}: Error sending shutdown signal: {e}") + + for thread in self.output_threads: + if thread and thread.is_alive(): + thread.join(timeout=5.0) + + logger.info(f"{self.label}: All EngineCores shut down") + def add_request(self, seqs: List[Sequence]): logger.debug(f"{self.label}: Add request, sequence ids: {[seq.id for seq in seqs]}") @@ -168,10 +262,28 @@ def add_request(self, seqs: List[Sequence]): if seq.stream_callback is not None: self._seq_id_to_callback[seq.id] = seq.stream_callback seq.stream_callback = None - self.input_socket.send_multipart( - [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.ADD, seqs))], - copy=False, - ) + if self.local_engine_count == 1: + # Single DP rank, send all requests + logger.debug(f"{self.label}: Add {len(seqs)} requests to DP rank 0") + self.input_sockets[0].send_multipart( + [self.engine_core_identities[0], pickle.dumps((EngineCoreRequestType.ADD, seqs))], + copy=False, + ) + else: + # DP ranks, round-robin + dp_seqs = [[] for _ in range(self.local_engine_count)] + for i, seq in enumerate(seqs): + dp_rank = i % self.local_engine_count + dp_seqs[dp_rank].append(seq) + + for dp_rank, rank_seqs in enumerate(dp_seqs): + if rank_seqs: + logger.debug(f"{self.label}: Add {len(rank_seqs)} requests to DP rank {dp_rank}") + self.input_sockets[dp_rank].send_multipart( + [self.engine_core_identities[dp_rank], + pickle.dumps((EngineCoreRequestType.ADD, rank_seqs))], + copy=False, + ) def get_stream_outputs(self): try: @@ -179,22 +291,33 @@ def get_stream_outputs(self): except queue.Empty: return None - def send_utility_command(self, cmd: str): - logger.debug(f"{self.label}: Send utility command: {cmd}") - self.input_socket.send_multipart( - [self.engine_core_identity, pickle.dumps((EngineCoreRequestType.UTILITY, {"cmd": cmd}))], + def send_utility_command(self, cmd: str, dp_rank: int = 0): + logger.debug(f"{self.label}: Send utility command '{cmd}' to DP rank {dp_rank}") + self.input_sockets[dp_rank].send_multipart( + [self.engine_core_identities[dp_rank], + pickle.dumps((EngineCoreRequestType.UTILITY, {"cmd": cmd}))], copy=False, ) - def _shutdown_engine_core(self): - if self.engine_core_process is not None and self.engine_core_process.is_alive(): - self.input_socket.send_multipart( - [ - self.engine_core_identity, - pickle.dumps((EngineCoreRequestType.SHUTDOWN, None)), - ], - copy=False, - ) + def _shutdown_engine_core_rank(self, dp_rank: int): + if dp_rank >= len(self.engine_core_processes): + return + + process = self.engine_core_processes[dp_rank] + if process is not None and process.is_alive(): + try: + input_socket = self.input_sockets[dp_rank] + if not input_socket.closed: + input_socket.send_multipart( + [ + self.engine_core_identities[dp_rank], + pickle.dumps((EngineCoreRequestType.SHUTDOWN, None)), + ], + copy=False, + ) + logger.debug(f"{self.label}: Sent shutdown to DP rank {dp_rank}") + except Exception as e: + logger.debug(f"{self.label}: Error sending shutdown to DP rank {dp_rank}: {e}") def get_output(self) -> List[Sequence]: seqs = self.outputs_queue.get() @@ -206,23 +329,33 @@ def is_rest(self): return not self.outputs_queue.empty() def is_alive(self): - return self.engine_core_process is not None + return any(proc is not None and proc.is_alive() + for proc in self.engine_core_processes) + def finished_procs(self): + return any(proc is not None and not proc.is_alive() + for proc in self.engine_core_processes) -def launch_engine_core(config: Config): +def launch_engine_core(config: Config, dp_rank: int = 0): input_address = get_open_zmq_ipc_path() output_address = get_open_zmq_ipc_path() import torch if torch.multiprocessing.get_start_method(allow_none=True) is None: torch.multiprocessing.set_start_method("spawn", force=False) + + config.parallel_config.data_parallel_rank = dp_rank + config.parallel_config.data_parallel_rank_local = dp_rank + + logger.info(f"Creating EngineCore process: DP rank {dp_rank}, will use GPUs {dp_rank * config.tensor_parallel_size} to {(dp_rank + 1) * config.tensor_parallel_size - 1}") + process = multiprocessing.Process( target=EngineCore.run_engine, - name=f"EngineCore", + name=f"EngineCore-DP{dp_rank}", kwargs={ "config": config, "input_address": input_address, "output_address": output_address, }, ) - process.start() - return process, {"input_address": input_address, "output_address": output_address} + + return process, {"input_address": input_address, "output_address": output_address}, dp_rank diff --git a/atom/model_engine/llm_engine.py b/atom/model_engine/llm_engine.py index 882f0b8d20..3162f0cdc2 100644 --- a/atom/model_engine/llm_engine.py +++ b/atom/model_engine/llm_engine.py @@ -23,11 +23,14 @@ class LLMEngine: def __init__(self, model, **kwargs): config_fields = {field.name for field in fields(Config)} config_kwargs = {k: v for k, v in kwargs.items() if k in config_fields} + data_parallel_size = kwargs.get('data_parallel_size', 1) config = Config(model, **config_kwargs) self.tokenizer = AutoTokenizer.from_pretrained(config.model, use_fast=True) config.bos_token_id = self.tokenizer.bos_token_id config.eos_token_id = self.tokenizer.eos_token_id - + # Set data parallel size in config + config.parallel_config.data_parallel_size = data_parallel_size + self.data_parallel_size = data_parallel_size self.rquest_ids = set() self.io_processor = InputOutputProcessor( self.tokenizer, config.kv_cache_block_size @@ -35,7 +38,7 @@ def __init__(self, model, **kwargs): self.core_mgr = CoreManager(config) self._step_lock = None self._pending_results = {} - logger.info("LLMEngine init") + logger.info(f"LLMEngine init with {self.data_parallel_size} data parallel ranks") def add_request( self, diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index b449b725d4..6175fae2dd 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -7,6 +7,7 @@ import torch import torch.profiler as torch_profiler import tqdm +from aiter.dist.utils import get_distributed_init_method from aiter import destroy_dist_env, dtypes, init_dist_env from aiter.dist.parallel_state import graph_capture, get_tp_group from atom.config import Config, set_current_atom_config, KVCacheTensor @@ -20,6 +21,9 @@ from atom.models.qwen3 import Qwen3ForCausalLM from atom.utils import CpuGpuBuffer, init_exit_handler, get_hf_text_config from atom.utils.selector import get_attn_backend +from atom.model_engine.sequence import Sequence, SequenceStatus, SequenceType +from atom.utils import CpuGpuBuffer, envs, init_exit_handler, get_hf_text_config +from aiter.dist.parallel_state import get_dp_group, get_tp_group logger = logging.getLogger("atom") from atom.utils.forward_context import ( @@ -29,6 +33,7 @@ reset_forward_context, set_forward_context, set_kv_cache_data, + DPMetadata, ) suppot_model_arch_dict = { @@ -207,14 +212,31 @@ def __init__(self, rank: int, config: Config): self.enforce_eager = config.enforce_eager self.world_size = config.tensor_parallel_size self.rank = rank - self.device = torch.device(f"cuda:{rank}") self.label = f"Model Runner{rank}/{self.world_size}" self.hf_text_config = get_hf_text_config(hf_config) self.use_mla = self.is_deepseek_mla() self.is_deepseek_v32 = ( hasattr(hf_config, "index_topk") if self.use_mla else False ) + # Calculate local device rank considering both TP and DP + # When data parallelism is enabled on the same node, different DP ranks + # need to use different sets of GPUs + dp_rank_local = config.parallel_config.data_parallel_rank_local + if dp_rank_local is None: + dp_rank_local = 0 + local_device_rank = dp_rank_local * config.tensor_parallel_size + rank + num_gpus = torch.cuda.device_count() + if local_device_rank >= num_gpus: + raise ValueError( + f"Calculated local_device_rank={local_device_rank} exceeds available GPUs ({num_gpus}). " + ) + device = torch.device(f"cuda:{local_device_rank}") + print( + f"ModelRunner rank={rank}, dp_rank_local={dp_rank_local}, local_device_rank={local_device_rank}, device={device}" + ) + self.device = device + # Initialize profiler for this rank self.profiler = None self.profiler_dir = None @@ -228,7 +250,18 @@ def __init__(self, rank: int, config: Config): torch.cuda.set_device(self.device) os.environ["MASTER_ADDR"] = self.config.master_addr os.environ["MASTER_PORT"] = str(self.config.port) - init_dist_env(self.world_size, rankID=rank) + distributed_init_method = get_distributed_init_method( + config.parallel_config.data_parallel_master_ip, + config.parallel_config.data_parallel_base_port, + ) + init_dist_env( + config.tensor_parallel_size, + rankID=rank, + backend="nccl", + distributed_init_method=distributed_init_method, + data_parallel_size=config.parallel_config.data_parallel_size, + data_parallel_rank=config.parallel_config.data_parallel_rank, + ) init_exit_handler(self) default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) @@ -349,6 +382,56 @@ def stop_profiler(self): self.profiler = None return True + def dummy_execution(self): + """Execute dummy decode batch for DP synchronization. """ + num_tokens_original = 1 + + seq = Sequence([0] * num_tokens_original, block_size=self.block_size) + seq.status = SequenceStatus.RUNNING + seq.type = SequenceType.DECODE + seq.block_table = [0] + bs = 1 + + dummy_batch = ScheduledBatch( + seqs={seq.id: seq}, + num_scheduled_tokens=np.array([num_tokens_original], dtype=np.int32), + total_tokens_num=num_tokens_original, # original value + total_tokens_num_decode=num_tokens_original, + total_seqs_num=1, + total_seqs_num_decode=1, + ) + + attn_metadata, positions = self.attn_metadata_builder.build(dummy_batch, bs) + context_bs = dummy_batch.total_seqs_num_decode + + num_input_tokens, num_tokens_across_dp = self._preprocess(dummy_batch) + + context = Context( + positions=positions, + is_prefill=False, + batch_size=context_bs, + graph_bs=bs, + ) + + actual_num_tokens = dummy_batch.total_tokens_num + set_forward_context( + attn_metadata=attn_metadata, + atom_config=self.config, + context=context, + num_tokens=actual_num_tokens, # original value, not with padding + num_tokens_across_dp=num_tokens_across_dp, + ) + + self.tokenID_processor.input_ids.np[:num_input_tokens] = [0] * num_input_tokens + self.tokenID_processor.input_ids.copy_to_gpu(num_input_tokens) + input_ids = self.tokenID_processor.input_ids.gpu[:num_input_tokens] + + logits = self.run_model(input_ids) + + reset_forward_context() + logger.debug(f"{self.label}: dummy batch executed with {num_input_tokens} tokens") + return True + def warmup_model(self): start_time = time.time() torch.cuda.empty_cache() @@ -357,16 +440,32 @@ def warmup_model(self): self.config.max_num_batched_tokens, self.config.max_model_len, ) + dp_size = get_dp_group().world_size + warmup_max_tokens = max_num_batched_tokens // dp_size + num_seqs = min( - max_num_batched_tokens // max_model_len, self.config.max_num_seqs + warmup_max_tokens // max_model_len, self.config.max_num_seqs ) + + if num_seqs == 0: + num_seqs = 1 + seq_len = min(warmup_max_tokens, max_model_len) + if seq_len == 0: + seq_len = 1 + logger.warning( + f"{self.label}: DP size={dp_size} too large, warmup_max_tokens={warmup_max_tokens} < max_model_len={max_model_len}. " + f"Using {num_seqs} seq with length {seq_len} for warmup." + ) + else: + seq_len = max_model_len + seqs = [ - Sequence([0] * max_model_len, block_size=self.block_size) + Sequence([0] * seq_len, block_size=self.block_size) for _ in range(num_seqs) ] seqs = {seq.id: seq for seq in seqs} - num_scheduled_tokens = np.array([max_model_len] * num_seqs, dtype=np.int32) + num_scheduled_tokens = np.array([seq_len] * num_seqs, dtype=np.int32) total_tokens_num = num_scheduled_tokens.sum() dummy_batch = ScheduledBatch( @@ -583,6 +682,33 @@ def allocate_kv_cache(self, num_kvcache_blocks): torch.distributed.barrier() return True + def get_dp_padding(self, num_tokens: int) -> tuple[int, Optional[torch.Tensor]]: + dp_size = self.config.parallel_config.data_parallel_size + dp_rank = self.config.parallel_config.data_parallel_rank + + # For DP: Don't pad when setting enforce_eager. + # This lets us set enforce_eager on the prefiller in a P/D setup and + # still use CUDA graphs (enabled by this padding) on the decoder. + # + # TODO(tms) : There are many cases where padding is enabled for + # prefills, causing unnecessary and excessive padding of activations. + + if dp_size == 1 or self.enforce_eager: + # Early exit. + return 0, None + num_tokens_across_dp = DPMetadata.num_tokens_across_dp( + num_tokens, dp_size, dp_rank + ) + max_tokens_across_dp_cpu = torch.max(num_tokens_across_dp).item() + + return max_tokens_across_dp_cpu - num_tokens, num_tokens_across_dp + + def _preprocess(self, batch: ScheduledBatch): + num_input_tokens = batch.total_tokens_num + num_pad, num_tokens_across_dp = self.get_dp_padding(num_input_tokens) + num_input_tokens += num_pad + return num_input_tokens, num_tokens_across_dp + def prepare_intputs(self, batch: ScheduledBatch): is_prefill = batch.total_tokens_num_prefill > 0 bs = batch.total_seqs_num @@ -617,9 +743,16 @@ def prepare_intputs(self, batch: ScheduledBatch): batch_size=context_bs, graph_bs=bs, ) + num_input_tokens, num_tokens_across_dp = self._preprocess(batch) + actual_num_tokens = batch.total_tokens_num set_forward_context( - attn_metadata=attn_metadata, atom_config=self.config, context=context + attn_metadata=attn_metadata, + atom_config=self.config, + context=context, + num_tokens=actual_num_tokens, + num_tokens_across_dp=num_tokens_across_dp, ) + return num_input_tokens def prepare_sample(self, batch: ScheduledBatch) -> torch.Tensor: temperatures = [seq.temperature for seq in batch.seqs.values()] @@ -718,10 +851,15 @@ def capture_cudagraph(self): attn_metadata, context = ( self.attn_metadata_builder.build_for_cudagraph_capture(bs) ) + num_tokens = bs + num_pad, num_tokens_across_dp = self.get_dp_padding(num_tokens) + num_tokens += num_pad set_forward_context( attn_metadata=attn_metadata, atom_config=self.config, context=context, + num_tokens=num_tokens, + num_tokens_across_dp=num_tokens_across_dp, ) outputs[:bs] = self.model(input_ids[:bs], positions[:bs]) # warmup diff --git a/atom/model_engine/scheduler.py b/atom/model_engine/scheduler.py index 50c6ffcbe2..850e0bf97b 100644 --- a/atom/model_engine/scheduler.py +++ b/atom/model_engine/scheduler.py @@ -72,7 +72,8 @@ def schedule(self) -> ScheduledBatch: num_scheduled_tokens: list[int] = [] if not self.running and not self.waiting: - self.block_manager.reset() + # self.block_manager.reset() + return None while self.waiting and num_seqs_prefill < self.max_num_seqs: seq = self.waiting[0] diff --git a/atom/model_ops/attention_mla.py b/atom/model_ops/attention_mla.py index 05dac69e14..5309fcc951 100644 --- a/atom/model_ops/attention_mla.py +++ b/atom/model_ops/attention_mla.py @@ -24,6 +24,7 @@ batched_gemm_a8w8_a_per_token_group_prequant_w_per_batched_tensor_quant as _aiter_triton_fp8_bmm, ) from aiter.ops.triton.fused_kv_cache import fused_qk_rope_cat_and_cache_mla +from aiter.dist.parallel_state import get_dp_group torch.set_printoptions(threshold=10_000) @@ -259,6 +260,27 @@ def _forward_decode( if self.kv_cache_dtype.startswith("fp8"): q = q.to(dtypes.fp8) q_scale = kv_scale = self.one_scale + + dp_size = get_dp_group().world_size + is_fp8 = self.kv_cache_dtype.startswith("fp8") + use_persistent_mode = not (dp_size == 8 and not is_fp8) + + if not use_persistent_mode: + # DP + bf16: disable persistent mode to avoid overflow + work_meta_data = None + work_indptr = None + work_info_set = None + reduce_indptr = None + reduce_final_map = None + reduce_partial_map = None + else: + work_meta_data = attn_metadata.work_meta_data + work_indptr = attn_metadata.work_indptr + work_info_set = attn_metadata.work_info_set + reduce_indptr = attn_metadata.reduce_indptr + reduce_final_map = attn_metadata.reduce_final_map + reduce_partial_map = attn_metadata.reduce_partial_map + mla_decode_fwd( q, kv_buffer.view(-1, 1, 1, q.shape[-1]), @@ -272,12 +294,12 @@ def _forward_decode( 0.0, None, None, - attn_metadata.work_meta_data, - attn_metadata.work_indptr, - attn_metadata.work_info_set, - attn_metadata.reduce_indptr, - attn_metadata.reduce_final_map, - attn_metadata.reduce_partial_map, + work_meta_data, + work_indptr, + work_info_set, + reduce_indptr, + reduce_final_map, + reduce_partial_map, q_scale, kv_scale, ) diff --git a/atom/model_ops/embed_head.py b/atom/model_ops/embed_head.py index 09020cc225..0d7c51591e 100644 --- a/atom/model_ops/embed_head.py +++ b/atom/model_ops/embed_head.py @@ -17,7 +17,7 @@ def __init__( embedding_dim: int, ): super().__init__() - self.tp_rank = get_tp_group().rank + self.tp_rank = get_tp_group().rank_in_group self.tp_size = get_tp_group().world_size assert num_embeddings % self.tp_size == 0 self.num_embeddings = num_embeddings diff --git a/atom/model_ops/linear.py b/atom/model_ops/linear.py index 3bf7d4e526..eb28c2a744 100644 --- a/atom/model_ops/linear.py +++ b/atom/model_ops/linear.py @@ -51,7 +51,7 @@ def __init__( output_size if isinstance(output_size, int) else sum(output_size) ) self.tp_dim = tp_dim - self.tp_rank = get_tp_group().rank + self.tp_rank = get_tp_group().rank_in_group self.tp_size = get_tp_group().world_size self.output_partition_sizes = ( output_size if isinstance(output_size, list) else [output_size] @@ -407,7 +407,7 @@ def __init__( reduce_results: bool = True, **kwargs, ): - self.tp_rank = get_tp_group().rank + self.tp_rank = get_tp_group().rank_in_group super().__init__( input_size, output_size, diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 38fea53ea1..094caa1080 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -2,14 +2,17 @@ from functools import lru_cache from typing import Callable, List, Optional, Tuple +from dataclasses import dataclass + +from atom.utils.forward_context import ForwardContext, get_forward_context import torch from aiter import ActivationType, QuantType, dtypes, get_hip_quant -from aiter.dist.parallel_state import get_tp_group +from aiter.dist.parallel_state import get_tp_group, get_dp_group from aiter.fused_moe import fused_moe from torch import nn from transformers import PretrainedConfig -from atom.config import QuantizationConfig, get_current_atom_config +from atom.config import Config, QuantizationConfig, get_current_atom_config from atom.model_loader.weight_utils import set_weight_attrs from atom.model_ops.base_config import QuantizeMethodBase from atom.model_ops.topK import ( @@ -25,6 +28,134 @@ shuffle_weights, ) from atom.utils.custom_register import direct_register_custom_op +from aiter.jit.utils.torch_guard import torch_compile_guard +from atom.utils import envs + + +@dataclass +class FusedMoEParallelConfig: + tp_size: int + dp_size: int + ep_size: int + tp_rank: int + dp_rank: int + ep_rank: int + + use_ep: bool # whether to use EP or not + @property + def use_all2all_kernels(self): + return self.dp_size > 1 and self.use_ep + + @staticmethod + def make(tp_size_: int, dp_size_: int, + parallel_config: Config) -> "FusedMoEParallelConfig": + def flatten_tp_across_dp(dp_rank: int): + tp_rank = 0 if tp_size_ == 1 else get_tp_group().rank_in_group + # There are actually dp_size_ * tp_size_ devices. Update tp_size + # and tp_rank so we shard across all devices. + tp_size = dp_size_ * tp_size_ + tp_rank = dp_rank * tp_size_ + tp_rank + return tp_size, tp_rank + + use_ep = (dp_size_ * tp_size_ > 1 + and parallel_config.enable_expert_parallel) + + dp_size = dp_size_ + dp_rank = get_dp_group().rank_in_group if dp_size > 1 else 0 + tp_size, tp_rank = flatten_tp_across_dp(dp_rank) + + if not use_ep: + return FusedMoEParallelConfig(tp_size=tp_size, + tp_rank=tp_rank, + dp_size=dp_size, + dp_rank=dp_rank, + ep_size=1, + ep_rank=0, + use_ep=False) + # DP + EP / TP + EP / DP + TP + EP + assert use_ep + # In EP, each device owns a set of experts fully. There is no tensor + # parallel update tp_size, tp_rank, ep_size and ep_rank to reflect that. + ep_size = tp_size + ep_rank = tp_rank + return FusedMoEParallelConfig(tp_size=1, + tp_rank=0, + dp_size=dp_size, + dp_rank=dp_rank, + ep_size=ep_size, + ep_rank=ep_rank, + use_ep=True) + + +def naive_multicast_fake(x: torch.Tensor, + cu_tokens_across_dp_cpu: torch.Tensor)->torch.Tensor: + assert (len(x.shape) == 2) + # print(f"cu_tokens_across_dp_cpu: {cu_tokens_across_dp_cpu}") + buffer = torch.empty((cu_tokens_across_dp_cpu[-1], x.size(1)), + device=x.device, + dtype=x.dtype) + return buffer + + + +@torch_compile_guard() +def naive_multicast(x: torch.Tensor, + cu_tokens_across_dp_cpu: torch.Tensor)->torch.Tensor: + dp_rank = get_dp_group().rank_in_group + assert (len(x.shape) == 2) + # print(f"cu_tokens_across_dp_cpu: {cu_tokens_across_dp_cpu}") + buffer = torch.empty((cu_tokens_across_dp_cpu[-1], x.size(1)), + device=x.device, + dtype=x.dtype) + + start = 0 if dp_rank == 0 else cu_tokens_across_dp_cpu[ + dp_rank - 1] + end = cu_tokens_across_dp_cpu[dp_rank] + buffer[start:end, :].copy_(x) + for idx in range(get_dp_group().world_size): + start = 0 if idx == 0 else cu_tokens_across_dp_cpu[idx - 1] + end = cu_tokens_across_dp_cpu[idx] + get_dp_group().broadcast(buffer[start:end, :], idx) + return buffer + + +def all_gather_with_padding(x: torch.Tensor): + max_batch_size = 512 + dim = 0 + original_batch_size = x.shape[dim] + padded_x = x + if original_batch_size < max_batch_size: + padding_size = max_batch_size - original_batch_size + + padding_shape = list(x.shape) + padding_shape[dim] = padding_size + + padding = torch.empty( + padding_shape, dtype=x.dtype, device=x.device + ) + padding.zero_() + padded_x = torch.cat([x, padding], dim=dim) + + gathered_hidden_states = get_dp_group().all_gather(padded_x, dim=dim) + return gathered_hidden_states, original_batch_size + +def reduce_scatter_with_unpadding(x: torch.Tensor, original_batch_size: int)->torch.Tensor: + dim = 0 + dp_group = get_dp_group() + + scattered_output = dp_group.reduce_scatter(x, dim=dim) + + if scattered_output.shape[dim] > original_batch_size: + slices = [slice(None)] * scattered_output.ndim + slices[dim] = slice(0, original_batch_size) + scattered_output = scattered_output[slices] + + return scattered_output + + +@torch_compile_guard() +def get_max_tokens_across_dispatchers(input: torch.Tensor) -> int: + return input.item() class FusedMoEMethodBase(QuantizeMethodBase): @@ -591,6 +722,32 @@ def determine_expert_map( return (local_num_experts, expert_map) +def moe_forward( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + layer_name: str, +) -> torch.Tensor: + atom_config = get_current_atom_config() + self = atom_config.compilation_config.static_forward_context[layer_name] + return self.forward_impl(hidden_states, router_logits) + + +def moe_forward_fake( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + layer_name: str, +) -> torch.Tensor: + return torch.empty_like(hidden_states) + +direct_register_custom_op( + op_name="moe_forward", + op_func=moe_forward, + mutates_args=["hidden_states"], + fake_impl=moe_forward_fake, + tags=(torch.Tag.needs_fixed_stride_order, ), +) + + class FusedMoE(torch.nn.Module): """FusedMoE layer for MoE models. @@ -644,31 +801,21 @@ def __init__( self.quant_config = quant_config # Note: here we guard against accessing the TP and DP groups when # uninitialized (this happens when testing) - self.tp_size = tp_size if tp_size is not None else get_tp_group().world_size - tp_rank = 0 if self.tp_size == 1 else get_tp_group().rank_in_group - self.global_num_experts = num_experts + # self.tp_size = 1 + tp_size = tp_size if tp_size is not None else get_tp_group().world_size + dp_size = dp_size if dp_size is not None else get_dp_group().world_size atom_config = get_current_atom_config() - use_ep = self.tp_size > 1 and atom_config.enable_expert_parallel - - if use_ep: - # Set TP size to 1 to adjust for EP and adjust EP size and rank - self.ep_rank = tp_rank - self.tp_rank = 0 - self.ep_size = self.tp_size - self.tp_size = 1 - + self.moe_parallel_config = FusedMoEParallelConfig.make(tp_size, dp_size, atom_config) + tp_rank = 0 if self.tp_size == 1 else get_tp_group().rank_in_group + self.global_num_experts = num_experts + if self.use_ep: self.local_num_experts, self.expert_map = determine_expert_map( ep_size=self.ep_size, ep_rank=self.ep_rank, global_num_experts=self.global_num_experts, ) else: - # Adjust TP size for DP attention - self.tp_rank = tp_rank - self.ep_rank = 0 - self.tp_size = self.tp_size - self.ep_size = 1 self.local_num_experts = self.global_num_experts self.expert_map = None self.top_k = top_k @@ -687,7 +834,7 @@ def __init__( else 1.0 ) self.expert_mask = None - if use_ep: + if self.use_ep: expert_mask = torch.ones( (self.global_num_experts + self.num_fused_shared_experts + 1,), dtype=torch.int32, @@ -716,19 +863,18 @@ def __init__( n_routed_experts=self.global_num_experts, n_shared_experts=self.num_fused_shared_experts, top_k=self.top_k, - tp_rank=self.ep_rank if use_ep else tp_rank, - tp_size=self.ep_size if use_ep else tp_size, + tp_rank=self.ep_rank if self.use_ep else self.tp_rank, + tp_size=self.ep_size if self.use_ep else self.tp_size, shared_experts_score=( 1.0 if is_rocm_aiter_fuse_routed_scaling_factor() else 1 / self.routed_scaling_factor ), max_num_tokens=16384, # Default max tokens - is_EP=use_ep, + is_EP=self.use_ep, ) if is_rocm_aiter_fusion_shared_expert_enabled(): self.local_num_experts += self.num_fused_shared_experts - assert intermediate_size % self.tp_size == 0 self.hidden_size = hidden_size self.intermediate_size_per_partition = intermediate_size // self.tp_size @@ -744,6 +890,24 @@ def __init__( self.e_score_correction_bias = e_score_correction_bias self.activation = activation + self.use_chunked = (get_dp_group().world_size > 1) and (not envs.ATOM_ENFORCE_EAGER) + self.batched_hidden_states: Optional[torch.Tensor] = None + self.batched_router_logits: Optional[torch.Tensor] = None + + if self.use_chunked: + self.vllm_moe_dp_chunk_size = 256 + self.batched_hidden_states = torch.zeros( + (self.vllm_moe_dp_chunk_size, self.hidden_size), + dtype=torch.bfloat16, + device=torch.cuda.current_device()) + + # Note here we use `num_experts` which is logical expert count + self.batched_router_logits = torch.zeros( + (self.vllm_moe_dp_chunk_size, num_experts), + dtype=torch.bfloat16, + device=torch.cuda.current_device()) + + if self.scoring_func != "softmax" and not self.use_grouped_topk: raise ValueError( "Only softmax scoring function is supported for " "non-grouped topk." @@ -774,6 +938,42 @@ def __init__( "weight_loader": self.weight_loader, } self.quant_method.create_weights(layer=self, **moe_quant_params) + compilation_config = atom_config.compilation_config + if prefix in compilation_config.static_forward_context: + raise ValueError("Duplicate layer name: {}".format(prefix)) + compilation_config.static_forward_context[prefix] = self + self.layer_name = prefix + + + + @property + def tp_size(self): + return self.moe_parallel_config.tp_size + + @property + def dp_size(self): + return self.moe_parallel_config.dp_size + + @property + def ep_size(self): + return self.moe_parallel_config.ep_size + + @property + def tp_rank(self): + return self.moe_parallel_config.tp_rank + + @property + def dp_rank(self): + return self.moe_parallel_config.dp_rank + + @property + def ep_rank(self): + return self.moe_parallel_config.ep_rank + + @property + def use_ep(self): + return self.moe_parallel_config.use_ep + def _load_per_tensor_weight_scale( self, @@ -1102,10 +1302,143 @@ def select_experts( return topk_weights, topk_ids def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): - return self.forward_impl(hidden_states, router_logits) + return torch.ops.aiter.moe_forward(hidden_states, router_logits, self.layer_name) + + def forward_impl_chunked(self, full_hidden_states: torch.Tensor, full_router_logits: torch.Tensor): + """Chunked forward for DP to support CUDA Graph.""" + + from atom.utils.forward_context import get_forward_context + assert self.batched_hidden_states is not None + assert self.batched_router_logits is not None + assert self.batched_hidden_states.dtype == full_hidden_states.dtype + assert self.batched_router_logits.dtype == full_router_logits.dtype + # Check size compatibility. + assert ( + self.batched_hidden_states.size(-1) == full_hidden_states.size(-1)) + assert ( + self.batched_router_logits.size(-1) == full_router_logits.size(-1)) + + full_fused_final_hidden_states = torch.empty_like(full_hidden_states) + + + def process_chunk(chunk_start, chunk_end, skip_result_store=False): + chunk_size = chunk_end - chunk_start + hidden_states = full_hidden_states[chunk_start:chunk_end, :] + router_logits = full_router_logits[chunk_start:chunk_end, :] + + assert (self.batched_hidden_states.size(0) # type: ignore + >= chunk_size) + assert (self.batched_router_logits.size(0) # type: ignore + >= chunk_size) + staged_hidden_states = self.batched_hidden_states[: + chunk_size, :] # type: ignore + staged_router_logits = self.batched_router_logits[: + chunk_size, :] # type: ignore + staged_hidden_states.copy_(hidden_states, non_blocking=True) + staged_router_logits.copy_(router_logits, non_blocking=True) + + # Matrix multiply. + final_hidden_states = self.quant_method.apply( + layer=self, + x=staged_hidden_states, + router_logits=staged_router_logits, + top_k=self.top_k, + renormalize=self.renormalize, + use_grouped_topk=self.use_grouped_topk, + global_num_experts=self.global_num_experts, + expert_map=self.expert_mask, + topk_group=self.topk_group, + num_expert_group=self.num_expert_group, + custom_routing_function=self.custom_routing_function, + scoring_func=self.scoring_func, + e_score_correction_bias=self.e_score_correction_bias, + activation=self.activation, + apply_router_weight_on_input=self.apply_router_weight_on_input, + ) + + if not skip_result_store: + full_fused_final_hidden_states[ + chunk_start:chunk_end, :].copy_(final_hidden_states, + non_blocking=True) + + ctx = get_forward_context() + max_tokens_across_dispatchers = ctx.dp_metadata.max_tokens_across_dp_cpu + moe_dp_chunk_size_per_rank = self.vllm_moe_dp_chunk_size + + num_tokens = full_hidden_states.size(0) + for chunk_idx, chunk_start_ in enumerate( + range(0, get_max_tokens_across_dispatchers(max_tokens_across_dispatchers), + moe_dp_chunk_size_per_rank)): + chunk_start = chunk_start_ + chunk_end = min(chunk_start + moe_dp_chunk_size_per_rank, + max_tokens_across_dispatchers) + # clamp start and end + chunk_start = min(chunk_start, num_tokens - 1) + chunk_end = min(chunk_end, num_tokens) + with ctx.dp_metadata.chunked_sizes(moe_dp_chunk_size_per_rank, + chunk_idx): + process_chunk(chunk_start, + chunk_end, + skip_result_store=chunk_start_ >= num_tokens) + + + return full_fused_final_hidden_states + + def forward_impl_graph(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): + dp_group = get_dp_group() + if dp_group.world_size > 1: + hidden_states, original_hidden_size = all_gather_with_padding(hidden_states) + router_logits, _ = all_gather_with_padding(router_logits) + + # Matrix multiply. + final_hidden_states = self.quant_method.apply( + layer=self, + x=hidden_states, + router_logits=router_logits, + top_k=self.top_k, + renormalize=self.renormalize, + use_grouped_topk=self.use_grouped_topk, + global_num_experts=self.global_num_experts, + expert_map=self.expert_mask, + topk_group=self.topk_group, + num_expert_group=self.num_expert_group, + custom_routing_function=self.custom_routing_function, + scoring_func=self.scoring_func, + e_score_correction_bias=self.e_score_correction_bias, + activation=self.activation, + apply_router_weight_on_input=self.apply_router_weight_on_input, + ) + + dp_group = get_dp_group() + if dp_group.world_size > 1: + final_hidden_states = reduce_scatter_with_unpadding(final_hidden_states, original_hidden_size) + + + if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): + # Default set to False. (May have to add shared expert outputs.) + final_hidden_states = get_tp_group().all_reduce( + final_hidden_states, ca_fp8_quant=False + ) + + return final_hidden_states + + + def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): assert self.quant_method is not None + # cuda graph not supported forward with combine and dispatch + is_prefill = get_forward_context().context.is_prefill + if self.use_chunked: + return self.forward_impl_graph(hidden_states, router_logits) + # return self.forward_impl_chunked(hidden_states, router_logits) + + dp_group = get_dp_group() + if dp_group.world_size > 1: + cu_tokens_across_dp_cpu = get_forward_context().dp_metadata.cu_tokens_across_dp_cpu + + hidden_states = naive_multicast(hidden_states, cu_tokens_across_dp_cpu) + router_logits = naive_multicast(router_logits, cu_tokens_across_dp_cpu) # Matrix multiply. final_hidden_states = self.quant_method.apply( @@ -1125,6 +1458,17 @@ def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor) activation=self.activation, apply_router_weight_on_input=self.apply_router_weight_on_input, ) + + dp_group = get_dp_group() + if dp_group.world_size > 1: + dp_rank = dp_group.rank_in_group + start = 0 if dp_rank == 0 else cu_tokens_across_dp_cpu[ + dp_rank - 1] + end = cu_tokens_across_dp_cpu[dp_rank] + + all_hidden_states = get_dp_group().all_reduce(final_hidden_states) + final_hidden_states = all_hidden_states[start:end, :] + if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): # Default set to False. (May have to add shared expert outputs.) diff --git a/atom/utils/__init__.py b/atom/utils/__init__.py index e83cd5a867..3e89d62ba6 100644 --- a/atom/utils/__init__.py +++ b/atom/utils/__init__.py @@ -10,7 +10,7 @@ from functools import lru_cache from multiprocessing.context import ForkContext, SpawnContext from multiprocessing.process import BaseProcess -from typing import Any, Iterator, Optional, Sequence, Union +from typing import Any, Iterator, Optional, Sequence, Union, Callable, TYPE_CHECKING from urllib.parse import urlparse from uuid import uuid4 from transformers import PretrainedConfig @@ -21,10 +21,49 @@ import zmq import zmq.asyncio from atom.utils.custom_register import direct_register_custom_op -from typing import Any, Callable, Optional + +if TYPE_CHECKING: + from atom.config import Config +from unittest.mock import patch logger = logging.getLogger("atom") + +@contextlib.contextmanager +def set_device_control_env_var(config: "Config", local_dp_rank: int): + """ + Temporarily set CUDA_VISIBLE_DEVICES or equivalent + for engine subprocess. + """ + world_size = config.tensor_parallel_size + evar = "VLLM_DEVICE_CONTROL_ENV_VAR_PLACEHOLDER" + + value = get_device_indices(evar, local_dp_rank, world_size) + print(f"Setting DP rank {local_dp_rank} to {value}") + with patch.dict(os.environ, values=((evar, value), )): + yield + +def get_device_indices(device_control_env_var: str, local_dp_rank: int, + world_size: int): + """ + Returns a comma-separated string of device indices for the specified + data parallel rank. + For example, if world_size=2 and local_dp_rank=1, and there are 4 devices, + this will select devices 2 and 3 for local_dp_rank=1. + """ + try: + value = ",".join( + str(i) + for i in range(local_dp_rank * world_size, (local_dp_rank + 1) * + world_size)) + except IndexError as e: + raise Exception(f"Error setting {device_control_env_var}: " + f"local range: [{local_dp_rank * world_size}, " + f"{(local_dp_rank + 1) * world_size}) " + "base value: " + f"\"{os.getenv(device_control_env_var)}\"") from e + return value + def mark_spliting_op( is_custom: bool, gen_fake: Optional[Callable[..., Any]] = None, @@ -402,7 +441,6 @@ def copy_to_cpu(self, n: Optional[int] = None) -> torch.Tensor: from packaging import version from packaging.version import Version -from atom.config import CompilationConfig, CompilationLevel, Config context_manager = None torch_compile_start_time: float = 0.0 @@ -509,33 +547,6 @@ def resolve_obj_by_qualname(qualname: str) -> Any: return getattr(module, obj_name) -def start_monitoring_torch_compile(vllm_config: Config): - global torch_compile_start_time - torch_compile_start_time = time.time() - - compilation_config: CompilationConfig = vllm_config.compilation_config - if ( - compilation_config.level == CompilationLevel.PIECEWISE - and compilation_config.debug_dump_path - ): - import depyf - - path = os.path.join(compilation_config.debug_dump_path, "rank_0") - # f"rank_{vllm_config.parallel_config.rank}") - global context_manager - context_manager = depyf.prepare_debug(path) - context_manager.__enter__() - - -def end_monitoring_torch_compile(vllm_config: Config): - compilation_config: CompilationConfig = vllm_config.compilation_config - if compilation_config.level == CompilationLevel.PIECEWISE: - global context_manager - if context_manager is not None: - context_manager.__exit__(None, None, None) - context_manager = None - - def getLogger(): global logger if not logger.handlers: diff --git a/atom/utils/backends.py b/atom/utils/backends.py index 66909c7390..bfe15d4b27 100644 --- a/atom/utils/backends.py +++ b/atom/utils/backends.py @@ -558,7 +558,7 @@ def __call__(self, graph: fx.GraphModule, example_inputs) -> Callable: # when dynamo calls the backend, it means the bytecode # transform and analysis are done compilation_counter.num_graphs_seen += 1 - from atom.utils import torch_compile_start_time + from atom.utils.decorators import torch_compile_start_time dynamo_time = time.time() - torch_compile_start_time logger.info("Dynamo bytecode transform time: %.2f s", dynamo_time) self.compilation_config.compilation_time += dynamo_time diff --git a/atom/utils/cuda_piecewise_backend.py b/atom/utils/cuda_piecewise_backend.py index 0887443ea6..9ccbbf90b5 100644 --- a/atom/utils/cuda_piecewise_backend.py +++ b/atom/utils/cuda_piecewise_backend.py @@ -8,7 +8,7 @@ # import vllm.envs as envs from atom.utils.backends import VllmBackend -from atom.utils import end_monitoring_torch_compile +from atom.utils.decorators import end_monitoring_torch_compile from atom.config import Config from aiter import logger diff --git a/atom/utils/decorators.py b/atom/utils/decorators.py index bc3f7e1914..2cf51fe7f7 100644 --- a/atom/utils/decorators.py +++ b/atom/utils/decorators.py @@ -12,13 +12,43 @@ from torch._dynamo.symbolic_convert import InliningInstructionTranslator import torch import torch.nn as nn +import time -from atom.config import CompilationConfig, Config -from atom.utils import start_monitoring_torch_compile -from atom.config import CompilationLevel +from atom.config import CompilationConfig, Config, CompilationLevel +# from atom.utils import start_monitoring_torch_compile _T = TypeVar("_T", bound=type[nn.Module]) +context_manager = None +torch_compile_start_time: float = 0.0 + +# We remove it from utils/__init__.py to avoid circular import +def start_monitoring_torch_compile(vllm_config: Config): + global torch_compile_start_time + torch_compile_start_time = time.time() + + compilation_config: CompilationConfig = vllm_config.compilation_config + if ( + compilation_config.level == CompilationLevel.PIECEWISE + and compilation_config.debug_dump_path + ): + import depyf + + path = os.path.join(compilation_config.debug_dump_path, "rank_0") + # f"rank_{vllm_config.parallel_config.rank}") + global context_manager + context_manager = depyf.prepare_debug(path) + context_manager.__enter__() + + +def end_monitoring_torch_compile(vllm_config: Config): + compilation_config: CompilationConfig = vllm_config.compilation_config + if compilation_config.level == CompilationLevel.PIECEWISE: + global context_manager + if context_manager is not None: + context_manager.__exit__(None, None, None) + context_manager = None + def init_backend(config: Config): from .backends import VllmBackend diff --git a/atom/utils/distributed/utils.py b/atom/utils/distributed/utils.py new file mode 100644 index 0000000000..11a1704c6c --- /dev/null +++ b/atom/utils/distributed/utils.py @@ -0,0 +1,136 @@ +import ipaddress +from torch.distributed.distributed_c10d import (Backend, PrefixStore, + _get_default_timeout, + _unregister_process_group) +from torch.distributed import ProcessGroup +from torch.distributed.rendezvous import rendezvous +import torch +from datetime import timedelta + + +from atom.utils import is_torch_equal_or_newer + + +def is_valid_ipv6_address(address: str) -> bool: + try: + ipaddress.IPv6Address(address) + return True + except ValueError: + return False + + +def get_tcp_uri(ip: str, port: int) -> str: + if is_valid_ipv6_address(ip): + return f"tcp://[{ip}]:{port}" + else: + return f"tcp://{ip}:{port}" + + +def init_gloo_process_group(backend: Backend, prefix_store: PrefixStore, + group_rank: int, group_size: int, + timeout: timedelta) -> ProcessGroup: + """ + Stateless init ProcessGroup with gloo backend compatible with + different torch versions. + """ + if is_torch_equal_or_newer("2.6"): + pg = ProcessGroup( + prefix_store, + group_rank, + group_size, + ) + else: + options = ProcessGroup.Options(backend=backend) + pg = ProcessGroup( + prefix_store, + group_rank, + group_size, + options, + ) + from torch.distributed.distributed_c10d import ProcessGroupGloo + backend_class = ProcessGroupGloo(prefix_store, + group_rank, + group_size, + timeout=timeout) + backend_type = ProcessGroup.BackendType.GLOO + device = torch.device("cpu") + if is_torch_equal_or_newer("2.6"): + # _set_default_backend is supported in torch >= 2.6 + pg._set_default_backend(backend_type) + backend_class._set_sequence_number_for_group() + + pg._register_backend(device, backend_type, backend_class) + return pg + + +def stateless_init_torch_distributed_process_group( + host: str, port: int, rank: int, world_size: int, + backend: str) -> ProcessGroup: + """ + A replacement for `torch.distributed.init_process_group` that does not + pollute the global state. The created ProcessGroup object can be used for + some operations such as `allreduce`, because it does not depend on the + global rank. However, some operations such as `broadcast` cannot be used + because it depends on the global rank. + + # TODO: ask for help from PyTorch team if we need the `broadcast` operation. + + This function is useful when we are not sure about the total number of + processes in the process group. For example, we may have process + 1, 2, ..., 8 who want to communicate, and process 9 might be the same + process as process 1, or it might be a different process; process 10 + might be the same process as process 5, or it might be a different process. + In this case, how can we reliably form a communication channel within + process 9 and 10, without affecting the communication channel within + process 1, 2, ..., 8? + + One possible solution is to figure out if process 9 and 10 are the same + as process 1 and 5 beforehand, and then form a communication channel + based on the information, adjusting the ranks and world_size etc. However, + figuring out the information is not always easy, and it will interfere + with the main communication channel. + + Our solution is to always form a communication channel with process 1, 2, + ..., 8, and then use this function to form another communication channel + with process 9 and 10. This way, regardless of whether process 9 and 10 + are the same as process 1 and 5, the main communication channel is + always formed with process 1, 2, ..., 8, and the additional communication + channel is formed with process 9 and 10. + """ + init_method = get_tcp_uri(host, port) + backend = Backend(backend) # it is basically string + timeout = _get_default_timeout(backend) + + store, rank, world_size = next( + rendezvous(init_method, rank, world_size, timeout=timeout)) + store.set_timeout(timeout) + group_rank = rank + group_size = world_size + + # Use a PrefixStore to avoid accidental overrides of keys used by + # different systems (e.g. RPC) in case the store is multi-tenant. + prefix_store = PrefixStore(init_method, store) + if backend == "gloo": + return init_gloo_process_group(backend=backend, + prefix_store=prefix_store, + group_rank=group_rank, + group_size=group_size, + timeout=timeout) + else: + raise ValueError(f"Unsupported backend: {backend}") + + +def stateless_destroy_torch_distributed_process_group( + pg: ProcessGroup) -> None: + """ + Destroy ProcessGroup returned by + stateless_init_torch_distributed_process_group(). + """ + if is_torch_equal_or_newer("2.7"): + pg.shutdown() + else: + # Lazy import for non-CUDA backends. + from torch.distributed.distributed_c10d import _shutdown_backend + _shutdown_backend(pg) + + _unregister_process_group(pg.group_name) diff --git a/atom/utils/envs.py b/atom/utils/envs.py new file mode 100644 index 0000000000..c18fb00be7 --- /dev/null +++ b/atom/utils/envs.py @@ -0,0 +1,18 @@ +import os +from typing import Callable, Any +# Data parallel environment variables + +environment_variables: dict[str, Callable[[], Any]] = { + "ATOM_DP_RANK": lambda: int(os.getenv("ATOM_DP_RANK", "0")), + "ATOM_DP_RANK_LOCAL": lambda: int(os.getenv("ATOM_DP_RANK_LOCAL", "0")), + "ATOM_DP_SIZE": lambda: int(os.getenv("ATOM_DP_SIZE", "1")), + "ATOM_DP_MASTER_IP": lambda: os.getenv("ATOM_DP_MASTER_IP", "127.0.0.1"), + "ATOM_DP_MASTER_PORT": lambda: int(os.getenv("ATOM_DP_MASTER_PORT", "29500")), + "ATOM_ENFORCE_EAGER": lambda: os.getenv("ATOM_ENFORCE_EAGER", "0") == "1", +} + +def __getattr__(name: str): + # lazy evaluation of environment variables + if name in environment_variables: + return environment_variables[name]() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/atom/utils/forward_context.py b/atom/utils/forward_context.py index f1cc75ce4d..96a01e0436 100644 --- a/atom/utils/forward_context.py +++ b/atom/utils/forward_context.py @@ -5,7 +5,106 @@ from atom.config import Config, KVCacheTensor import torch from abc import ABC, abstractmethod +from atom.config import Config, ParallelConfig +def _compute_chunked_local_num_tokens(num_tokens_across_dp_cpu: list[int], + max_num_tokens: int, + chunk_idx: int) -> list[int]: + dp_size = len(num_tokens_across_dp_cpu) + + local_size = [-1] * dp_size + for i in range(dp_size): + dp_tokens = num_tokens_across_dp_cpu[i] + local_size[i] = min(max_num_tokens, + dp_tokens - (max_num_tokens * chunk_idx)) + if local_size[i] <= 0: + local_size[i] = 1 # ensure lockstep even if done + return local_size + + +@dataclass +class DPMetadata: + max_tokens_across_dp_cpu: torch.Tensor + cu_tokens_across_dp_cpu: torch.Tensor + local_sizes: Optional[list[int]] = None + + @staticmethod + def num_tokens_across_dp(num_tokens: int, dp_size: int, + dp_rank: int) -> torch.Tensor: + """ + Gather the num_tokens across all DP ranks and return results in a + CPU tensor of size dp_size. + """ + num_tokens_across_dp = [0] * dp_size + num_tokens_across_dp[dp_rank] = num_tokens + num_tokens_tensor = torch.tensor(num_tokens_across_dp, + device="cpu", + dtype=torch.int32) + from aiter.dist.parallel_state import get_dp_group + import torch.distributed as dist + dist.all_reduce(num_tokens_tensor, group=get_dp_group().cpu_group) + return num_tokens_tensor + + @staticmethod + def make( + parallel_config: ParallelConfig, + # attn_metadata: Any, + num_tokens: int, + num_tokens_across_dp: Optional[torch.Tensor] = None + ) -> "DPMetadata": + + assert parallel_config.data_parallel_size > 1 + dp_size = parallel_config.data_parallel_size + dp_rank = parallel_config.data_parallel_rank + batchsize = num_tokens + + # If num_tokens_across_dp is None, it will be computed by all_reduce + # Otherwise, num_tokens_across_dp[dp_rank] should be equal to batchsize + assert (num_tokens_across_dp is None + or num_tokens_across_dp[dp_rank] == batchsize) + if num_tokens_across_dp is None: + num_tokens_across_dp = DPMetadata.num_tokens_across_dp( + batchsize, dp_size, dp_rank) + max_tokens_across_dp_cpu = torch.max(num_tokens_across_dp) + cu_tokens_across_dp_cpu = torch.cumsum(num_tokens_across_dp, dim=0) + return DPMetadata(max_tokens_across_dp_cpu, cu_tokens_across_dp_cpu) + + @contextmanager + def chunked_sizes(self, max_chunk_size_per_rank: int, chunk_idx: int): + """ + Context manager to compute and temporarily set the per-rank local token + sizes for a specific chunk during chunked forward execution. + This is necessary to ensure each DP (data parallel) rank processes its + designated portion of tokens in lockstep with others, even when the + token counts are uneven or some ranks have completed their input early. + For chunked execution, we break up the total tokens on each rank into + multiple chunks (of at most `max_chunk_size_per_rank`), and for a given + `chunk_idx`, this context manager sets `self.local_sizes` to the number + of tokens to process in that chunk on each rank. + It uses cumulative sizes (`cu_tokens_across_dp_cpu`) to derive the + number of tokens per rank, and calls `_compute_chunked_local_num_tokens` + to determine the chunk-wise split. + `self.local_sizes` is only valid inside the context. + Args: + max_chunk_size_per_rank: The max number of tokens each rank is + allowed to process in this chunk. + chunk_idx: The index of the chunk to compute sizes for. + """ + cu_sizes = self.cu_tokens_across_dp_cpu + num_tokens_across_dp_cpu = [ + (cu_sizes[i] - + cu_sizes[i - 1]).item() if i > 0 else cu_sizes[0].item() + for i in range(len(cu_sizes)) + ] + self.local_sizes = _compute_chunked_local_num_tokens( + num_tokens_across_dp_cpu, max_chunk_size_per_rank, chunk_idx) + try: + yield self.local_sizes + finally: + self.local_sizes = None + + def get_chunk_sizes_across_dp_rank(self) -> Optional[list[int]]: + return self.local_sizes @dataclass class Context: @@ -131,6 +230,8 @@ class ForwardContext: context: Optional[Context] = None + dp_metadata: Optional[DPMetadata] = None + def __post_init__(self): if not hasattr(self, "no_compile_layers") or self.no_compile_layers is None: self.no_compile_layers = {} @@ -152,17 +253,25 @@ def get_forward_context() -> ForwardContext: def set_forward_context( - attn_metadata: AttentionMetaData, atom_config: Config, context: Context + attn_metadata: AttentionMetaData, atom_config: Config, context: Context, + num_tokens: Optional[int] = None, + num_tokens_across_dp: Optional[torch.Tensor] = None, ) -> None: - global _forward_context + dp_metadata: Optional[DPMetadata] = None + if atom_config.parallel_config.data_parallel_size > 1 and num_tokens is not None: + dp_metadata = DPMetadata.make(atom_config.parallel_config, + # attn_metadata, + num_tokens or 0, + num_tokens_across_dp) + _forward_context = ForwardContext( attn_metadata=attn_metadata, no_compile_layers=atom_config.compilation_config.static_forward_context, kv_cache_data=_forward_kv_cache_context.kv_cache_data, context=context, - ) - # _forward_context.attn_metadata = attn_metadata + dp_metadata=dp_metadata, + ) # _forward_context.attn_metadata = attn_metadata # _forward_context.no_compile_layers = atom_config.compilation_config.static_forward_context # _forward_context = ForwardContext(no_compile_layers=atom_config.compilation_config.static_forward_context, attn_metadata=attn_metadata) From 681e7744f70b5854c0eb89b4fb60ba46496ce884 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 19 Nov 2025 14:52:11 +0800 Subject: [PATCH 115/119] fix some bug --- atom/model_engine/arg_utils.py | 2 ++ atom/model_engine/engine_core_mgr.py | 8 +++++--- atom/model_engine/model_runner.py | 6 +++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/atom/model_engine/arg_utils.py b/atom/model_engine/arg_utils.py index 8505f1c731..b1e5438448 100644 --- a/atom/model_engine/arg_utils.py +++ b/atom/model_engine/arg_utils.py @@ -185,6 +185,8 @@ def create_engine(self) -> LLMEngine: level=self.level, cudagraph_capture_sizes=parse_size_list(self.cudagraph_capture_sizes) if self.cudagraph_capture_sizes else None, ), + data_parallel_size=self.data_parallel_size, + enable_dp_attention=self.enable_dp_attention, ) def create_async_engine(self) -> AsyncLLMEngine: diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 3e7748b02c..774a8d4db9 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -48,6 +48,7 @@ def __init__(self, config: Config): self.engine_core_identities = [] self.shutdown_paths = [] self.output_threads = [] + self._rr_counter = 0 import torch if torch.multiprocessing.get_start_method(allow_none=True) is None: @@ -270,11 +271,12 @@ def add_request(self, seqs: List[Sequence]): copy=False, ) else: - # DP ranks, round-robin + # DP ranks, round-robin with counter for load balancing for atom server dp_seqs = [[] for _ in range(self.local_engine_count)] - for i, seq in enumerate(seqs): - dp_rank = i % self.local_engine_count + for seq in seqs: + dp_rank = self._rr_counter % self.local_engine_count dp_seqs[dp_rank].append(seq) + self._rr_counter += 1 for dp_rank, rank_seqs in enumerate(dp_seqs): if rank_seqs: diff --git a/atom/model_engine/model_runner.py b/atom/model_engine/model_runner.py index 6175fae2dd..68270935be 100644 --- a/atom/model_engine/model_runner.py +++ b/atom/model_engine/model_runner.py @@ -265,7 +265,7 @@ def __init__(self, rank: int, config: Config): init_exit_handler(self) default_dtype = torch.get_default_dtype() torch.set_default_dtype(hf_config.torch_dtype) - torch.set_default_device("cuda") + torch.set_default_device(self.device) self.attn_backend = get_attn_backend( self.block_size, use_mla=self.use_mla, @@ -289,7 +289,7 @@ def __init__(self, rank: int, config: Config): load_model(self.model, config.model, config.hf_config, config.load_dummy) if isinstance(self.model, DeepseekV2ForCausalLM): self.use_kv_indptr = True - torch.set_default_device("cuda") + torch.set_default_device(self.device) self.allocate_forward_vars() self.attn_metadata_builder = self.attn_backend.get_builder_cls()(self) self.warmup_model() @@ -502,7 +502,7 @@ def allocate_forward_vars(self): } def get_num_blocks(self): - torch.set_default_device("cuda") + torch.set_default_device(self.device) config = self.config hf_config = config.hf_config if not hasattr(hf_config, "head_dim") or hf_config.head_dim is None: From 279a0208e2df1b69ab105aa59dce7a2f67dce411 Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 19 Nov 2025 15:08:38 +0800 Subject: [PATCH 116/119] use real cuda graph size and remove RR counter --- atom/examples/simple_inference.py | 13 ++++++++++++- atom/model_engine/engine_core_mgr.py | 8 +++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/atom/examples/simple_inference.py b/atom/examples/simple_inference.py index 5635ec7fc6..ca26077072 100644 --- a/atom/examples/simple_inference.py +++ b/atom/examples/simple_inference.py @@ -20,6 +20,16 @@ ) +def generate_cuda_graph_sizes(max_size): +# This is for DP split batch size + sizes = [] + power = 1 + while power <= max_size: + sizes.append(power) + power *= 2 + return sizes + + def main(): prompts = [ "introduce yourself", @@ -28,7 +38,8 @@ def main(): "ε¦‚δ½•εœ¨δΈ€δΈͺζœˆε†…ε’žθ‚Œ10ε…¬ζ–€", ] args = parser.parse_args() - args.cudagraph_capture_sizes = str([1, len(prompts)]) + # Generate power of 2 sizes for CUDA graph: [1, 2, 4, 8, ...] + args.cudagraph_capture_sizes = str(generate_cuda_graph_sizes(len(prompts))) # Create engine from args engine_args = EngineArgs.from_cli_args(args) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 774a8d4db9..3e7748b02c 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -48,7 +48,6 @@ def __init__(self, config: Config): self.engine_core_identities = [] self.shutdown_paths = [] self.output_threads = [] - self._rr_counter = 0 import torch if torch.multiprocessing.get_start_method(allow_none=True) is None: @@ -271,12 +270,11 @@ def add_request(self, seqs: List[Sequence]): copy=False, ) else: - # DP ranks, round-robin with counter for load balancing for atom server + # DP ranks, round-robin dp_seqs = [[] for _ in range(self.local_engine_count)] - for seq in seqs: - dp_rank = self._rr_counter % self.local_engine_count + for i, seq in enumerate(seqs): + dp_rank = i % self.local_engine_count dp_seqs[dp_rank].append(seq) - self._rr_counter += 1 for dp_rank, rank_seqs in enumerate(dp_seqs): if rank_seqs: From 99daae43450b359be8e54493da22a4c5de3d772b Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 19 Nov 2025 15:26:56 +0800 Subject: [PATCH 117/119] use rr count in add_request --- atom/model_engine/engine_core_mgr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/atom/model_engine/engine_core_mgr.py b/atom/model_engine/engine_core_mgr.py index 3e7748b02c..774a8d4db9 100644 --- a/atom/model_engine/engine_core_mgr.py +++ b/atom/model_engine/engine_core_mgr.py @@ -48,6 +48,7 @@ def __init__(self, config: Config): self.engine_core_identities = [] self.shutdown_paths = [] self.output_threads = [] + self._rr_counter = 0 import torch if torch.multiprocessing.get_start_method(allow_none=True) is None: @@ -270,11 +271,12 @@ def add_request(self, seqs: List[Sequence]): copy=False, ) else: - # DP ranks, round-robin + # DP ranks, round-robin with counter for load balancing for atom server dp_seqs = [[] for _ in range(self.local_engine_count)] - for i, seq in enumerate(seqs): - dp_rank = i % self.local_engine_count + for seq in seqs: + dp_rank = self._rr_counter % self.local_engine_count dp_seqs[dp_rank].append(seq) + self._rr_counter += 1 for dp_rank, rank_seqs in enumerate(dp_seqs): if rank_seqs: From e399df01b0205609f26cf6e5a4c8422908d2bfac Mon Sep 17 00:00:00 2001 From: ZhangLirong-amd Date: Wed, 19 Nov 2025 18:05:16 +0800 Subject: [PATCH 118/119] format moe --- atom/model_ops/moe.py | 259 +++++++++++++++--------------------------- 1 file changed, 93 insertions(+), 166 deletions(-) diff --git a/atom/model_ops/moe.py b/atom/model_ops/moe.py index 094caa1080..20532c2b75 100644 --- a/atom/model_ops/moe.py +++ b/atom/model_ops/moe.py @@ -42,13 +42,15 @@ class FusedMoEParallelConfig: ep_rank: int use_ep: bool # whether to use EP or not + @property def use_all2all_kernels(self): return self.dp_size > 1 and self.use_ep @staticmethod - def make(tp_size_: int, dp_size_: int, - parallel_config: Config) -> "FusedMoEParallelConfig": + def make( + tp_size_: int, dp_size_: int, parallel_config: Config + ) -> "FusedMoEParallelConfig": def flatten_tp_across_dp(dp_rank: int): tp_rank = 0 if tp_size_ == 1 else get_tp_group().rank_in_group # There are actually dp_size_ * tp_size_ devices. Update tp_size @@ -57,59 +59,62 @@ def flatten_tp_across_dp(dp_rank: int): tp_rank = dp_rank * tp_size_ + tp_rank return tp_size, tp_rank - use_ep = (dp_size_ * tp_size_ > 1 - and parallel_config.enable_expert_parallel) + use_ep = dp_size_ * tp_size_ > 1 and parallel_config.enable_expert_parallel dp_size = dp_size_ dp_rank = get_dp_group().rank_in_group if dp_size > 1 else 0 tp_size, tp_rank = flatten_tp_across_dp(dp_rank) if not use_ep: - return FusedMoEParallelConfig(tp_size=tp_size, - tp_rank=tp_rank, - dp_size=dp_size, - dp_rank=dp_rank, - ep_size=1, - ep_rank=0, - use_ep=False) + return FusedMoEParallelConfig( + tp_size=tp_size, + tp_rank=tp_rank, + dp_size=dp_size, + dp_rank=dp_rank, + ep_size=1, + ep_rank=0, + use_ep=False, + ) # DP + EP / TP + EP / DP + TP + EP assert use_ep # In EP, each device owns a set of experts fully. There is no tensor # parallel update tp_size, tp_rank, ep_size and ep_rank to reflect that. ep_size = tp_size ep_rank = tp_rank - return FusedMoEParallelConfig(tp_size=1, - tp_rank=0, - dp_size=dp_size, - dp_rank=dp_rank, - ep_size=ep_size, - ep_rank=ep_rank, - use_ep=True) - - -def naive_multicast_fake(x: torch.Tensor, - cu_tokens_across_dp_cpu: torch.Tensor)->torch.Tensor: - assert (len(x.shape) == 2) + return FusedMoEParallelConfig( + tp_size=1, + tp_rank=0, + dp_size=dp_size, + dp_rank=dp_rank, + ep_size=ep_size, + ep_rank=ep_rank, + use_ep=True, + ) + + +def naive_multicast_fake( + x: torch.Tensor, cu_tokens_across_dp_cpu: torch.Tensor +) -> torch.Tensor: + assert len(x.shape) == 2 # print(f"cu_tokens_across_dp_cpu: {cu_tokens_across_dp_cpu}") - buffer = torch.empty((cu_tokens_across_dp_cpu[-1], x.size(1)), - device=x.device, - dtype=x.dtype) + buffer = torch.empty( + (cu_tokens_across_dp_cpu[-1], x.size(1)), device=x.device, dtype=x.dtype + ) return buffer - @torch_compile_guard() -def naive_multicast(x: torch.Tensor, - cu_tokens_across_dp_cpu: torch.Tensor)->torch.Tensor: +def naive_multicast( + x: torch.Tensor, cu_tokens_across_dp_cpu: torch.Tensor +) -> torch.Tensor: dp_rank = get_dp_group().rank_in_group - assert (len(x.shape) == 2) + assert len(x.shape) == 2 # print(f"cu_tokens_across_dp_cpu: {cu_tokens_across_dp_cpu}") - buffer = torch.empty((cu_tokens_across_dp_cpu[-1], x.size(1)), - device=x.device, - dtype=x.dtype) + buffer = torch.empty( + (cu_tokens_across_dp_cpu[-1], x.size(1)), device=x.device, dtype=x.dtype + ) - start = 0 if dp_rank == 0 else cu_tokens_across_dp_cpu[ - dp_rank - 1] + start = 0 if dp_rank == 0 else cu_tokens_across_dp_cpu[dp_rank - 1] end = cu_tokens_across_dp_cpu[dp_rank] buffer[start:end, :].copy_(x) for idx in range(get_dp_group().world_size): @@ -130,26 +135,27 @@ def all_gather_with_padding(x: torch.Tensor): padding_shape = list(x.shape) padding_shape[dim] = padding_size - padding = torch.empty( - padding_shape, dtype=x.dtype, device=x.device - ) + padding = torch.empty(padding_shape, dtype=x.dtype, device=x.device) padding.zero_() padded_x = torch.cat([x, padding], dim=dim) gathered_hidden_states = get_dp_group().all_gather(padded_x, dim=dim) return gathered_hidden_states, original_batch_size -def reduce_scatter_with_unpadding(x: torch.Tensor, original_batch_size: int)->torch.Tensor: + +def reduce_scatter_with_unpadding( + x: torch.Tensor, original_batch_size: int +) -> torch.Tensor: dim = 0 dp_group = get_dp_group() - + scattered_output = dp_group.reduce_scatter(x, dim=dim) - + if scattered_output.shape[dim] > original_batch_size: slices = [slice(None)] * scattered_output.ndim slices[dim] = slice(0, original_batch_size) scattered_output = scattered_output[slices] - + return scattered_output @@ -322,9 +328,21 @@ def rocm_aiter_fused_moe_impl( activation_ = ActivationType(activation) quant_type_ = QuantType(quant_type) - return fused_moe(hidden_states, w1, w2, topk_weight, topk_ids, expert_mask, - activation_, quant_type_, doweight_stage1, w1_scale, - w2_scale, a1_scale, a2_scale) + return fused_moe( + hidden_states, + w1, + w2, + topk_weight, + topk_ids, + expert_mask, + activation_, + quant_type_, + doweight_stage1, + w1_scale, + w2_scale, + a1_scale, + a2_scale, + ) def rocm_aiter_fused_moe_fake( @@ -344,6 +362,7 @@ def rocm_aiter_fused_moe_fake( ) -> torch.Tensor: return torch.empty_like(hidden_states) + direct_register_custom_op( op_name="rocm_aiter_fused_moe", op_func=rocm_aiter_fused_moe_impl, @@ -351,6 +370,7 @@ def rocm_aiter_fused_moe_fake( fake_impl=rocm_aiter_fused_moe_fake, ) + class Fp8MoEMethod(FusedMoEMethodBase): """MoE method for FP8. Supports loading FP8 checkpoints with static weight scale and @@ -658,7 +678,7 @@ def apply( num_fused_shared_experts=layer.num_fused_shared_experts, routed_scaling_factor=layer.routed_scaling_factor, ) - + return torch.ops.aiter.rocm_aiter_fused_moe( x, layer.w13_weight, @@ -672,8 +692,8 @@ def apply( w2_scale=layer.w2_weight_scale, a1_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, - doweight_stage1=apply_router_weight_on_input) - + doweight_stage1=apply_router_weight_on_input, + ) def determine_expert_map( @@ -739,12 +759,13 @@ def moe_forward_fake( ) -> torch.Tensor: return torch.empty_like(hidden_states) + direct_register_custom_op( op_name="moe_forward", op_func=moe_forward, mutates_args=["hidden_states"], fake_impl=moe_forward_fake, - tags=(torch.Tag.needs_fixed_stride_order, ), + tags=(torch.Tag.needs_fixed_stride_order,), ) @@ -806,7 +827,9 @@ def __init__( dp_size = dp_size if dp_size is not None else get_dp_group().world_size atom_config = get_current_atom_config() - self.moe_parallel_config = FusedMoEParallelConfig.make(tp_size, dp_size, atom_config) + self.moe_parallel_config = FusedMoEParallelConfig.make( + tp_size, dp_size, atom_config + ) tp_rank = 0 if self.tp_size == 1 else get_tp_group().rank_in_group self.global_num_experts = num_experts if self.use_ep: @@ -890,23 +913,9 @@ def __init__( self.e_score_correction_bias = e_score_correction_bias self.activation = activation - self.use_chunked = (get_dp_group().world_size > 1) and (not envs.ATOM_ENFORCE_EAGER) - self.batched_hidden_states: Optional[torch.Tensor] = None - self.batched_router_logits: Optional[torch.Tensor] = None - - if self.use_chunked: - self.vllm_moe_dp_chunk_size = 256 - self.batched_hidden_states = torch.zeros( - (self.vllm_moe_dp_chunk_size, self.hidden_size), - dtype=torch.bfloat16, - device=torch.cuda.current_device()) - - # Note here we use `num_experts` which is logical expert count - self.batched_router_logits = torch.zeros( - (self.vllm_moe_dp_chunk_size, num_experts), - dtype=torch.bfloat16, - device=torch.cuda.current_device()) - + self.use_chunked = (get_dp_group().world_size > 1) and ( + not envs.ATOM_ENFORCE_EAGER + ) if self.scoring_func != "softmax" and not self.use_grouped_topk: raise ValueError( @@ -944,8 +953,6 @@ def __init__( compilation_config.static_forward_context[prefix] = self self.layer_name = prefix - - @property def tp_size(self): return self.moe_parallel_config.tp_size @@ -974,7 +981,6 @@ def ep_rank(self): def use_ep(self): return self.moe_parallel_config.use_ep - def _load_per_tensor_weight_scale( self, shard_id: str, @@ -1302,93 +1308,17 @@ def select_experts( return topk_weights, topk_ids def forward(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): - return torch.ops.aiter.moe_forward(hidden_states, router_logits, self.layer_name) - - def forward_impl_chunked(self, full_hidden_states: torch.Tensor, full_router_logits: torch.Tensor): - """Chunked forward for DP to support CUDA Graph.""" - - from atom.utils.forward_context import get_forward_context - assert self.batched_hidden_states is not None - assert self.batched_router_logits is not None - assert self.batched_hidden_states.dtype == full_hidden_states.dtype - assert self.batched_router_logits.dtype == full_router_logits.dtype - # Check size compatibility. - assert ( - self.batched_hidden_states.size(-1) == full_hidden_states.size(-1)) - assert ( - self.batched_router_logits.size(-1) == full_router_logits.size(-1)) - - full_fused_final_hidden_states = torch.empty_like(full_hidden_states) - - - def process_chunk(chunk_start, chunk_end, skip_result_store=False): - chunk_size = chunk_end - chunk_start - hidden_states = full_hidden_states[chunk_start:chunk_end, :] - router_logits = full_router_logits[chunk_start:chunk_end, :] - - assert (self.batched_hidden_states.size(0) # type: ignore - >= chunk_size) - assert (self.batched_router_logits.size(0) # type: ignore - >= chunk_size) - staged_hidden_states = self.batched_hidden_states[: - chunk_size, :] # type: ignore - staged_router_logits = self.batched_router_logits[: - chunk_size, :] # type: ignore - staged_hidden_states.copy_(hidden_states, non_blocking=True) - staged_router_logits.copy_(router_logits, non_blocking=True) - - # Matrix multiply. - final_hidden_states = self.quant_method.apply( - layer=self, - x=staged_hidden_states, - router_logits=staged_router_logits, - top_k=self.top_k, - renormalize=self.renormalize, - use_grouped_topk=self.use_grouped_topk, - global_num_experts=self.global_num_experts, - expert_map=self.expert_mask, - topk_group=self.topk_group, - num_expert_group=self.num_expert_group, - custom_routing_function=self.custom_routing_function, - scoring_func=self.scoring_func, - e_score_correction_bias=self.e_score_correction_bias, - activation=self.activation, - apply_router_weight_on_input=self.apply_router_weight_on_input, - ) + return torch.ops.aiter.moe_forward( + hidden_states, router_logits, self.layer_name + ) - if not skip_result_store: - full_fused_final_hidden_states[ - chunk_start:chunk_end, :].copy_(final_hidden_states, - non_blocking=True) - - ctx = get_forward_context() - max_tokens_across_dispatchers = ctx.dp_metadata.max_tokens_across_dp_cpu - moe_dp_chunk_size_per_rank = self.vllm_moe_dp_chunk_size - - num_tokens = full_hidden_states.size(0) - for chunk_idx, chunk_start_ in enumerate( - range(0, get_max_tokens_across_dispatchers(max_tokens_across_dispatchers), - moe_dp_chunk_size_per_rank)): - chunk_start = chunk_start_ - chunk_end = min(chunk_start + moe_dp_chunk_size_per_rank, - max_tokens_across_dispatchers) - # clamp start and end - chunk_start = min(chunk_start, num_tokens - 1) - chunk_end = min(chunk_end, num_tokens) - with ctx.dp_metadata.chunked_sizes(moe_dp_chunk_size_per_rank, - chunk_idx): - process_chunk(chunk_start, - chunk_end, - skip_result_store=chunk_start_ >= num_tokens) - - - return full_fused_final_hidden_states - - def forward_impl_graph(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): + def forward_impl_graph( + self, hidden_states: torch.Tensor, router_logits: torch.Tensor + ): dp_group = get_dp_group() if dp_group.world_size > 1: hidden_states, original_hidden_size = all_gather_with_padding(hidden_states) - router_logits, _ = all_gather_with_padding(router_logits) + router_logits, _ = all_gather_with_padding(router_logits) # Matrix multiply. final_hidden_states = self.quant_method.apply( @@ -1408,11 +1338,12 @@ def forward_impl_graph(self, hidden_states: torch.Tensor, router_logits: torch.T activation=self.activation, apply_router_weight_on_input=self.apply_router_weight_on_input, ) - + dp_group = get_dp_group() if dp_group.world_size > 1: - final_hidden_states = reduce_scatter_with_unpadding(final_hidden_states, original_hidden_size) - + final_hidden_states = reduce_scatter_with_unpadding( + final_hidden_states, original_hidden_size + ) if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): # Default set to False. (May have to add shared expert outputs.) @@ -1422,23 +1353,21 @@ def forward_impl_graph(self, hidden_states: torch.Tensor, router_logits: torch.T return final_hidden_states - - - def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor): assert self.quant_method is not None # cuda graph not supported forward with combine and dispatch - is_prefill = get_forward_context().context.is_prefill if self.use_chunked: return self.forward_impl_graph(hidden_states, router_logits) # return self.forward_impl_chunked(hidden_states, router_logits) dp_group = get_dp_group() if dp_group.world_size > 1: - cu_tokens_across_dp_cpu = get_forward_context().dp_metadata.cu_tokens_across_dp_cpu + cu_tokens_across_dp_cpu = ( + get_forward_context().dp_metadata.cu_tokens_across_dp_cpu + ) hidden_states = naive_multicast(hidden_states, cu_tokens_across_dp_cpu) - router_logits = naive_multicast(router_logits, cu_tokens_across_dp_cpu) + router_logits = naive_multicast(router_logits, cu_tokens_across_dp_cpu) # Matrix multiply. final_hidden_states = self.quant_method.apply( @@ -1458,18 +1387,16 @@ def forward_impl(self, hidden_states: torch.Tensor, router_logits: torch.Tensor) activation=self.activation, apply_router_weight_on_input=self.apply_router_weight_on_input, ) - + dp_group = get_dp_group() if dp_group.world_size > 1: dp_rank = dp_group.rank_in_group - start = 0 if dp_rank == 0 else cu_tokens_across_dp_cpu[ - dp_rank - 1] + start = 0 if dp_rank == 0 else cu_tokens_across_dp_cpu[dp_rank - 1] end = cu_tokens_across_dp_cpu[dp_rank] all_hidden_states = get_dp_group().all_reduce(final_hidden_states) final_hidden_states = all_hidden_states[start:end, :] - if self.reduce_results and (self.tp_size > 1 or self.ep_size > 1): # Default set to False. (May have to add shared expert outputs.) final_hidden_states = get_tp_group().all_reduce( From b27db6c406c97e7efed6bdd863a03504deac4c9c Mon Sep 17 00:00:00 2001 From: valarLip <340077269@qq.com> Date: Fri, 21 Nov 2025 17:38:32 +0800 Subject: [PATCH 119/119] merge README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b26c5361fc..8f9607b9a6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ATOM -**ATOM** (AI Tensor Offline Model) is a lightweight vLLM-like implementation, focusing on integration and optimization based on [aiter](https://github.com/ROCm/aiter). +**ATOM** (AiTer Optimized Model) is a lightweight vLLM-like implementation, focusing on integration and optimization based on [aiter](https://github.com/ROCm/aiter). ## πŸš€ Features @@ -146,4 +146,4 @@ lm_eval --model local-completions \ --tasks gsm8k \ --num_fewshot 3 \ --batch_size 16 -``` \ No newline at end of file +```