From 55dcb4a75f640089a960db9b33b83116817eab2f Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sat, 27 Jun 2026 04:05:32 +0000 Subject: [PATCH 1/2] perf(server): cut event-loop work in streaming hot path - Reuse engine-computed num_prompt_tokens in the stream response generators instead of re-encoding the prompt on the event loop at stream start (drops a redundant per-request tokenize). - Run multimodal input prep (image download + HF processor) in a worker thread instead of synchronously on the event loop. - Batch-decode a whole step's buffered stream chunks with one tokenizer.batch_decode in flush_stream_batch instead of one decode per seq on the output thread (one GIL-released call instead of N). - Coalesce each request's finalization SSE messages (content/finish + usage + [DONE]) into a single send to cut socket-write syscalls when many requests finish simultaneously. --- atom/entrypoints/openai/api_server.py | 112 ++++++++++++------ atom/entrypoints/openai/serving_chat.py | 57 +++++---- atom/entrypoints/openai/serving_completion.py | 80 ++++++++----- 3 files changed, 157 insertions(+), 92 deletions(-) diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index dfc9b5cec8..48a334d36c 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -289,8 +289,10 @@ def _prepare_multimodal_inputs( # ── Batched stream dispatch ────────────────────────────────────────────── # Per-seq `call_soon_threadsafe` floods the API event loop at high batch size -# (one call per token). Instead the callback only decodes + buffers; the mgr -# flushes a whole step with one scheduled call (see `flush_stream_batch`). +# (one call per token). Instead the callback only buffers the raw chunk; the +# mgr flushes a whole step with a single `tokenizer.batch_decode` (one +# GIL-released call instead of one decode per seq) plus one scheduled call per +# loop (see `flush_stream_batch`). import threading as _threading # noqa: E402 _stream_batch_tls = _threading.local() @@ -302,13 +304,14 @@ def _send_stream_chunk_direct( stream_queue: asyncio.Queue, loop: AbstractEventLoop, ) -> None: - """Decode the chunk and buffer it; dispatch happens in flush_stream_batch.""" - global tokenizer + """Buffer the chunk; decode + dispatch happen batched in flush_stream_batch. - new_text = tokenizer.decode(request_output.output_tokens, skip_special_tokens=True) + ``text`` is intentionally left unset here — it is filled by a single + ``tokenizer.batch_decode`` over the whole step in :func:`flush_stream_batch`, + avoiding one decode call per seq on the output thread. + """ started_at = _request_start_times.get(request_id) chunk_data = { - "text": new_text, "token_ids": request_output.output_tokens, "finished": request_output.finished, "finish_reason": request_output.finish_reason, @@ -333,12 +336,25 @@ def _drain_batch_into_queues(items: list) -> None: def flush_stream_batch() -> None: - """Flush a step's buffered chunks: one call_soon_threadsafe per loop - (normally one — all requests on a rank share the API loop).""" + """Flush a step's buffered chunks: one ``batch_decode`` for the whole step, + then one call_soon_threadsafe per loop (normally one — all requests on a + rank share the API loop).""" + global tokenizer + buf = getattr(_stream_batch_tls, "buf", None) if not buf: return _stream_batch_tls.buf = [] + # Decode the whole step in a single call. batch_decode is element-wise + # identical to per-seq decode but acquires/releases the GIL once instead of + # once per seq, cutting GIL ping-pong against the other rank output threads + # and the API event loop at high batch size. + texts = tokenizer.batch_decode( + [chunk["token_ids"] for _loop, _q, chunk in buf], + skip_special_tokens=True, + ) + for (_loop, _q, chunk), text in zip(buf, texts): + chunk["text"] = text # Group by loop (normally a single loop). dict preserves insertion order # so per-request chunk ordering within the step is maintained. by_loop: Dict[AbstractEventLoop, list] = {} @@ -359,6 +375,12 @@ def _send_stream_chunk_tagged( Pushes ``(sibling_index, chunk_data)`` tuples onto a single shared queue so the merge-stream consumer in :mod:`serving_chat` / :mod:`serving_completion` can demultiplex by index. + + Unlike :func:`_send_stream_chunk_direct`, this fan-out path decodes and + dispatches immediately rather than going through the batched flush: it + serves ``SamplingParams.n > 1`` (a handful of siblings of one request), + not the many-concurrent-requests regime that motivates batch decoding, so + a separate per-step buffer here would add complexity for little gain. """ global tokenizer @@ -677,8 +699,13 @@ async def setup_streaming_request( request_id: str, kv_transfer_params: Optional[Dict[str, Any]] = None, multimodal_data: Optional[Dict[str, Any]] = None, -) -> Tuple[int, asyncio.Queue]: - """Set up a streaming request with the engine.""" +) -> Tuple[int, asyncio.Queue, int]: + """Set up a streaming request with the engine. + + Returns ``(seq_id, stream_queue, num_prompt_tokens)``. ``num_prompt_tokens`` + is the engine-computed prompt length so the stream response generator does + not have to re-tokenize the prompt on the event loop. + """ global engine, _stream_queues, _seq_id_to_request_id global _stream_loops, _request_start_times @@ -710,7 +737,7 @@ def do_preprocess(): logger.info(f"API: Created request_id={request_id}, seq_id={seq_id}") engine.core_mgr.add_request([seq]) - return seq_id, stream_queue + return seq_id, stream_queue, seq.num_prompt_tokens def cleanup_streaming_request(request_id: str, seq_id: int) -> None: @@ -736,12 +763,16 @@ async def setup_streaming_request_fanout( request_id: str, kv_transfer_params: Optional[Dict[str, Any]] = None, multimodal_data: Optional[Dict[str, Any]] = None, -) -> Tuple[List[int], asyncio.Queue]: +) -> Tuple[List[int], asyncio.Queue, int]: """Fan-out variant of :func:`setup_streaming_request`. Creates ``sampling_params.n`` sibling sequences sharing one output queue. Every callback pushes ``(sibling_index, chunk_data)`` tuples so the merge-stream consumer can rewrite ``choices[0].index`` correctly. + + Returns ``(seq_ids, shared_queue, num_prompt_tokens)``. All siblings + tokenize the same prompt once, so ``num_prompt_tokens`` is shared and lets + the stream response generator skip re-tokenizing on the event loop. """ global engine, _stream_queues, _seq_id_to_request_id global _stream_loops, _request_start_times @@ -784,7 +815,7 @@ def do_preprocess(): f"API: Created fan-out request_id={request_id}, n={n}, seq_ids={seq_ids}" ) engine.core_mgr.add_request(seqs) - return seq_ids, shared_queue + return seq_ids, shared_queue, seqs[0].num_prompt_tokens # ============================================================================ @@ -871,9 +902,14 @@ async def chat_completions(request: ChatCompletionRequest): is_multimodal = _has_multimodal_content(messages) if is_multimodal: - token_ids, multimodal_data = _prepare_multimodal_inputs( - messages, - merged_kwargs, + # Image loading (blocking network I/O, up to a 30s urlopen) plus + # processor preprocessing are heavy and would stall the event loop; + # run them in a worker thread. Warm the processor on the loop first + # so concurrent cold-start requests don't race on its lazy init. + _get_multimodal_processor() + loop = asyncio.get_running_loop() + token_ids, multimodal_data = await loop.run_in_executor( + None, _prepare_multimodal_inputs, messages, merged_kwargs ) else: prompt = apply_chat_template( @@ -889,24 +925,25 @@ async def chat_completions(request: ChatCompletionRequest): stream_input = token_ids if is_multimodal else prompt stream_multimodal_data = multimodal_data if is_multimodal else None if effective_n > 1: - seq_ids, stream_queue = await setup_streaming_request_fanout( - stream_input, - sampling_params, - request_id, - multimodal_data=stream_multimodal_data, - kv_transfer_params=request.kv_transfer_params, + seq_ids, stream_queue, num_prompt_tokens = ( + await setup_streaming_request_fanout( + stream_input, + sampling_params, + request_id, + multimodal_data=stream_multimodal_data, + kv_transfer_params=request.kv_transfer_params, + ) ) gen = stream_chat_response_fanout( request_id, model_name, - stream_input, stream_queue, seq_ids, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) else: - seq_id, stream_queue = await setup_streaming_request( + seq_id, stream_queue, num_prompt_tokens = await setup_streaming_request( stream_input, sampling_params, request_id, @@ -916,10 +953,9 @@ async def chat_completions(request: ChatCompletionRequest): gen = stream_chat_response( request_id, model_name, - stream_input, stream_queue, seq_id, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) return StreamingResponse( @@ -1014,23 +1050,24 @@ async def completions(request: CompletionRequest): # Streaming if request.stream: if effective_n > 1: - seq_ids, stream_queue = await setup_streaming_request_fanout( - request.prompt, - sampling_params, - request_id, - kv_transfer_params=request.kv_transfer_params, + seq_ids, stream_queue, num_prompt_tokens = ( + await setup_streaming_request_fanout( + request.prompt, + sampling_params, + request_id, + kv_transfer_params=request.kv_transfer_params, + ) ) gen = stream_completion_response_fanout( request_id, model_name, - request.prompt, stream_queue, seq_ids, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) else: - seq_id, stream_queue = await setup_streaming_request( + seq_id, stream_queue, num_prompt_tokens = await setup_streaming_request( request.prompt, sampling_params, request_id, @@ -1039,10 +1076,9 @@ async def completions(request: CompletionRequest): gen = stream_completion_response( request_id, model_name, - request.prompt, stream_queue, seq_id, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) return StreamingResponse( @@ -1155,7 +1191,7 @@ async def anthropic_messages(request: AnthropicMessagesRequest, raw_request: Req if request.stream: # Streaming response - seq_id, stream_queue = await setup_streaming_request( + seq_id, stream_queue, _num_prompt_tokens = await setup_streaming_request( prompt, sampling_params, request_id ) diff --git a/atom/entrypoints/openai/serving_chat.py b/atom/entrypoints/openai/serving_chat.py index d12692bae8..41fc6abb40 100644 --- a/atom/entrypoints/openai/serving_chat.py +++ b/atom/entrypoints/openai/serving_chat.py @@ -7,7 +7,7 @@ import json import logging import time -from typing import Any, AsyncGenerator, Dict, List, Optional, Union +from typing import Any, AsyncGenerator, Dict, List, Optional from .protocol import ( CHAT_COMPLETION_CHUNK_OBJECT, @@ -20,12 +20,6 @@ logger = logging.getLogger("atom") -def _prompt_token_count(prompt_or_tokens: Union[str, List[int]], tokenizer) -> int: - if isinstance(prompt_or_tokens, str): - return len(tokenizer.encode(prompt_or_tokens)) - return len(prompt_or_tokens) - - def create_chat_chunk( request_id: str, model: str, @@ -61,10 +55,9 @@ def create_chat_chunk( async def stream_chat_response( request_id: str, model: str, - prompt: Union[str, List[int]], stream_queue: asyncio.Queue, seq_id: int, - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: """Generate streaming chat completion response with reasoning and tool calls. @@ -73,8 +66,12 @@ async def stream_chat_response( - reasoning_content deltas during thinking phase - content deltas for the answer - tool_calls deltas when model invokes tools + + ``num_prompt_tokens`` is the engine-computed prompt length (``Sequence. + num_prompt_tokens``); reusing it avoids re-tokenizing the prompt on the + event loop at stream start. """ - num_tokens_input = _prompt_token_count(prompt, tokenizer) + num_tokens_input = num_prompt_tokens num_tokens_output = 0 reasoning_filter = ReasoningFilter() tool_parser = ToolCallStreamParser() @@ -152,7 +149,6 @@ async def stream_chat_response( "completion_tokens": num_tokens_output, "total_tokens": num_tokens_input + num_tokens_output, } - yield create_chat_chunk(request_id, model, finish_reason=finish_reason) usage_chunk = { "id": request_id, "object": CHAT_COMPLETION_CHUNK_OBJECT, @@ -162,8 +158,14 @@ async def stream_chat_response( } if kv_transfer_params_value is not None: usage_chunk["kv_transfer_params"] = kv_transfer_params_value - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + # Coalesce finish + usage + [DONE] into one send: at a wave boundary many + # requests finalize at once, so collapsing 3 socket writes/req to 1 cuts + # the syscalls that saturate the API event loop. + yield ( + create_chat_chunk(request_id, model, finish_reason=finish_reason) + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) def _build_chat_choice( @@ -271,10 +273,9 @@ def build_chat_response_multi( async def stream_chat_response_fanout( request_id: str, model: str, - prompt: Union[str, List[int]], shared_queue: asyncio.Queue, seq_ids: List[int], - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: """Streaming variant that multiplexes ``len(seq_ids)`` fan-out siblings @@ -283,9 +284,13 @@ async def stream_chat_response_fanout( The shared queue receives ``(sibling_index, chunk_data)`` tuples from the engine callbacks registered in :func:`setup_streaming_request_fanout`. Reasoning + tool-call state is kept independently per sibling. + + ``num_prompt_tokens`` is the engine-computed prompt length shared by all + siblings (they tokenize the same prompt once); reusing it avoids + re-tokenizing on the event loop at stream start. """ n = len(seq_ids) - num_tokens_input = _prompt_token_count(prompt, tokenizer) + num_tokens_input = num_prompt_tokens num_tokens_output = [0] * n reasoning_filters = [ReasoningFilter() for _ in range(n)] tool_parsers = [ToolCallStreamParser() for _ in range(n)] @@ -369,10 +374,6 @@ async def stream_chat_response_fanout( for sid in seq_ids: cleanup_fn(request_id, sid) - for i in range(n): - finish_reason = "tool_calls" if has_tool_calls[i] else "stop" - yield create_chat_chunk(request_id, model, finish_reason=finish_reason, index=i) - usage = { "prompt_tokens": num_tokens_input, "completion_tokens": sum(num_tokens_output), @@ -388,5 +389,17 @@ async def stream_chat_response_fanout( } if kv_transfer_params_value is not None: usage_chunk["kv_transfer_params"] = kv_transfer_params_value - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + # Coalesce the per-sibling finish chunks + usage + [DONE] into one send. + yield ( + "".join( + create_chat_chunk( + request_id, + model, + finish_reason="tool_calls" if has_tool_calls[i] else "stop", + index=i, + ) + for i in range(n) + ) + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) diff --git a/atom/entrypoints/openai/serving_completion.py b/atom/entrypoints/openai/serving_completion.py index fc06c627c8..47eb2e29a5 100644 --- a/atom/entrypoints/openai/serving_completion.py +++ b/atom/entrypoints/openai/serving_completion.py @@ -55,14 +55,18 @@ def create_completion_chunk( async def stream_completion_response( request_id: str, model: str, - prompt: str, stream_queue: asyncio.Queue, seq_id: int, - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: - """Generate streaming text completion response.""" - num_tokens_input = len(tokenizer.encode(prompt)) + """Generate streaming text completion response. + + ``num_prompt_tokens`` is the engine-computed prompt length (``Sequence. + num_prompt_tokens``); reusing it avoids re-tokenizing the prompt on the + event loop at stream start. + """ + num_tokens_input = num_prompt_tokens num_tokens_output = 0 while True: @@ -74,7 +78,7 @@ async def stream_completion_response( if "kv_transfer_params" in chunk_data: extra_fields["kv_transfer_params"] = chunk_data["kv_transfer_params"] - yield create_completion_chunk( + content_chunk = create_completion_chunk( request_id, model, new_text, @@ -83,26 +87,31 @@ async def stream_completion_response( ) if chunk_data.get("finished", False): - break + # Coalesce the finalization SSE messages (content + stop + usage + + # [DONE]) into a single send. At a wave boundary many requests + # finish simultaneously; collapsing 4 sends/req to 1 cuts the + # per-request socket-write syscalls that saturate the API loop. + cleanup_fn(request_id, seq_id) + usage_chunk = { + "id": request_id, + "object": TEXT_COMPLETION_OBJECT, + "created": int(time.time()), + "model": model, + "usage": { + "prompt_tokens": num_tokens_input, + "completion_tokens": num_tokens_output, + "total_tokens": num_tokens_input + num_tokens_output, + }, + } + yield ( + content_chunk + + create_completion_chunk(request_id, model, "", "stop") + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) + return - cleanup_fn(request_id, seq_id) - - usage = { - "prompt_tokens": num_tokens_input, - "completion_tokens": num_tokens_output, - "total_tokens": num_tokens_input + num_tokens_output, - } - yield create_completion_chunk(request_id, model, "", "stop") - # Usage-only chunk - usage_chunk = { - "id": request_id, - "object": TEXT_COMPLETION_OBJECT, - "created": int(time.time()), - "model": model, - "usage": usage, - } - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + yield content_chunk def build_completion_response( @@ -184,10 +193,9 @@ def build_completion_response_multi( async def stream_completion_response_fanout( request_id: str, model: str, - prompt: str, shared_queue: asyncio.Queue, seq_ids: List[int], - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: """Streaming variant multiplexing ``len(seq_ids)`` siblings into one SSE. @@ -195,9 +203,13 @@ async def stream_completion_response_fanout( Each chunk pulled from ``shared_queue`` is a ``(sibling_index, chunk_data)`` tuple; we re-emit with ``choices[0].index = sibling_index``. Finishes only when every sibling has reported ``finished=True``. + + ``num_prompt_tokens`` is the engine-computed prompt length shared by all + siblings; reusing it avoids re-tokenizing on the event loop at stream + start. """ n = len(seq_ids) - num_tokens_input = len(tokenizer.encode(prompt)) + num_tokens_input = num_prompt_tokens num_tokens_output = [0] * n finished = [False] * n @@ -227,9 +239,6 @@ async def stream_completion_response_fanout( for sid in seq_ids: cleanup_fn(request_id, sid) - for i in range(n): - yield create_completion_chunk(request_id, model, "", "stop", index=i) - usage = { "prompt_tokens": num_tokens_input, "completion_tokens": sum(num_tokens_output), @@ -243,5 +252,12 @@ async def stream_completion_response_fanout( "model": model, "usage": usage, } - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + # Coalesce the per-sibling stop chunks + usage + [DONE] into one send. + yield ( + "".join( + create_completion_chunk(request_id, model, "", "stop", index=i) + for i in range(n) + ) + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) From 9b8f451da4799fd85bc5f8605383e93af29a8fcd Mon Sep 17 00:00:00 2001 From: Lingpeng Jin <103567126+valarLip@users.noreply.github.com> Date: Sat, 27 Jun 2026 06:21:26 +0000 Subject: [PATCH 2/2] perf(server): enable uvloop event loop; fix gpt-oss embed param naming uvloop: - Run uvicorn on uvloop (libuv) instead of the stdlib asyncio selector loop, with graceful fallback to the default loop if uvloop is absent. Under high streaming concurrency this cuts the event-loop cost of SSE socket I/O (sock.send / selector register-unregister): steady-state TPOT P99 8.50ms -> 8.18ms and frontend loop-scheduling delay roughly halved. Adds uvloop to dependencies. gpt-oss: - Register `embed_tokens` first (with `embedding` as the shared-storage alias) so it stays the primary, non-deduped name in named_parameters(). The checkpoint stores `model.embed_tokens.weight`; with `embedding` as the primary name the load-completeness check falsely flagged `model.embedding.weight` as unloaded even though the weight is loaded via the alias. Byte-identical weights (GSM8K 0.8832, unchanged); the spurious "parameters were NOT loaded" warning is gone. --- atom/entrypoints/openai/api_server.py | 20 ++++++++++++++++++-- atom/models/gpt_oss.py | 11 +++++++++-- pyproject.toml | 2 +- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index 48a334d36c..ed4c6167ae 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -1556,8 +1556,24 @@ def _sigint_handler(signum, frame): signal.signal(signal.SIGINT, _sigint_handler) - logger.info(f"Starting server on {args.host}:{args.server_port}...") - uvicorn.run(app, host=args.host, port=args.server_port) + # uvloop replaces the stdlib asyncio selector loop with a libuv-backed one, + # which is markedly faster at the SSE socket I/O (sock.send / selector + # register-unregister) that saturates the event loop under high streaming + # concurrency. Fall back to the default loop if uvloop is unavailable. + try: + import uvloop # noqa: F401 + + loop_impl = "uvloop" + except ImportError: + loop_impl = "auto" + logger.warning( + "uvloop not installed; falling back to the default asyncio loop." + ) + + logger.info( + f"Starting server on {args.host}:{args.server_port} (loop={loop_impl})..." + ) + uvicorn.run(app, host=args.host, port=args.server_port, loop=loop_impl) if __name__ == "__main__": diff --git a/atom/models/gpt_oss.py b/atom/models/gpt_oss.py index bf49decd13..ed53777381 100644 --- a/atom/models/gpt_oss.py +++ b/atom/models/gpt_oss.py @@ -336,11 +336,18 @@ def __init__( self.config = atom_config.hf_config self.quant_config = atom_config.quant_config self.config.hidden_size = self.config.hidden_size - self.embedding = VocabParallelEmbedding( + # Register `embed_tokens` first so it stays the primary (non-deduped) + # name reported by `named_parameters()`. The checkpoint stores this + # tensor as `model.embed_tokens.weight`; if `embedding` were the primary + # name instead, the load-completeness check would falsely flag + # `model.embedding.weight` as unloaded (the weight is in fact loaded via + # the shared-storage alias). `embedding` remains as an alias for the + # internal call sites below. + self.embed_tokens = VocabParallelEmbedding( self.config.vocab_size, self.config.hidden_size, ) - self.embed_tokens = self.embedding + self.embedding = self.embed_tokens self.start_layer, self.end_layer, self.layers = make_layers( self.config.num_hidden_layers, lambda prefix, layer_num=None: TransformerBlock( diff --git a/pyproject.toml b/pyproject.toml index e71ae4425a..528375be31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" description = "a lightweight vLLM implementation built from scratch" requires-python = ">=3.10" dynamic = ["version"] -dependencies = ["pybind11", "transformers==5.2.0", "zmq", "msgspec", "xxhash", "fastapi>=0.115,<0.137", "psutil", "protobuf", "uvicorn", "aiohttp", "datasets", "openpyxl", "tqdm"] +dependencies = ["pybind11", "transformers==5.2.0", "zmq", "msgspec", "xxhash", "fastapi>=0.115,<0.137", "psutil", "protobuf", "uvicorn", "uvloop", "aiohttp", "datasets", "openpyxl", "tqdm"] [project.urls] Homepage = "https://github.com/ROCm/ATOM"