diff --git a/atom/entrypoints/openai/api_server.py b/atom/entrypoints/openai/api_server.py index dfc9b5cec8..ed4c6167ae 100644 --- a/atom/entrypoints/openai/api_server.py +++ b/atom/entrypoints/openai/api_server.py @@ -289,8 +289,10 @@ def _prepare_multimodal_inputs( # ── Batched stream dispatch ────────────────────────────────────────────── # Per-seq `call_soon_threadsafe` floods the API event loop at high batch size -# (one call per token). Instead the callback only decodes + buffers; the mgr -# flushes a whole step with one scheduled call (see `flush_stream_batch`). +# (one call per token). Instead the callback only buffers the raw chunk; the +# mgr flushes a whole step with a single `tokenizer.batch_decode` (one +# GIL-released call instead of one decode per seq) plus one scheduled call per +# loop (see `flush_stream_batch`). import threading as _threading # noqa: E402 _stream_batch_tls = _threading.local() @@ -302,13 +304,14 @@ def _send_stream_chunk_direct( stream_queue: asyncio.Queue, loop: AbstractEventLoop, ) -> None: - """Decode the chunk and buffer it; dispatch happens in flush_stream_batch.""" - global tokenizer + """Buffer the chunk; decode + dispatch happen batched in flush_stream_batch. - new_text = tokenizer.decode(request_output.output_tokens, skip_special_tokens=True) + ``text`` is intentionally left unset here — it is filled by a single + ``tokenizer.batch_decode`` over the whole step in :func:`flush_stream_batch`, + avoiding one decode call per seq on the output thread. + """ started_at = _request_start_times.get(request_id) chunk_data = { - "text": new_text, "token_ids": request_output.output_tokens, "finished": request_output.finished, "finish_reason": request_output.finish_reason, @@ -333,12 +336,25 @@ def _drain_batch_into_queues(items: list) -> None: def flush_stream_batch() -> None: - """Flush a step's buffered chunks: one call_soon_threadsafe per loop - (normally one — all requests on a rank share the API loop).""" + """Flush a step's buffered chunks: one ``batch_decode`` for the whole step, + then one call_soon_threadsafe per loop (normally one — all requests on a + rank share the API loop).""" + global tokenizer + buf = getattr(_stream_batch_tls, "buf", None) if not buf: return _stream_batch_tls.buf = [] + # Decode the whole step in a single call. batch_decode is element-wise + # identical to per-seq decode but acquires/releases the GIL once instead of + # once per seq, cutting GIL ping-pong against the other rank output threads + # and the API event loop at high batch size. + texts = tokenizer.batch_decode( + [chunk["token_ids"] for _loop, _q, chunk in buf], + skip_special_tokens=True, + ) + for (_loop, _q, chunk), text in zip(buf, texts): + chunk["text"] = text # Group by loop (normally a single loop). dict preserves insertion order # so per-request chunk ordering within the step is maintained. by_loop: Dict[AbstractEventLoop, list] = {} @@ -359,6 +375,12 @@ def _send_stream_chunk_tagged( Pushes ``(sibling_index, chunk_data)`` tuples onto a single shared queue so the merge-stream consumer in :mod:`serving_chat` / :mod:`serving_completion` can demultiplex by index. + + Unlike :func:`_send_stream_chunk_direct`, this fan-out path decodes and + dispatches immediately rather than going through the batched flush: it + serves ``SamplingParams.n > 1`` (a handful of siblings of one request), + not the many-concurrent-requests regime that motivates batch decoding, so + a separate per-step buffer here would add complexity for little gain. """ global tokenizer @@ -677,8 +699,13 @@ async def setup_streaming_request( request_id: str, kv_transfer_params: Optional[Dict[str, Any]] = None, multimodal_data: Optional[Dict[str, Any]] = None, -) -> Tuple[int, asyncio.Queue]: - """Set up a streaming request with the engine.""" +) -> Tuple[int, asyncio.Queue, int]: + """Set up a streaming request with the engine. + + Returns ``(seq_id, stream_queue, num_prompt_tokens)``. ``num_prompt_tokens`` + is the engine-computed prompt length so the stream response generator does + not have to re-tokenize the prompt on the event loop. + """ global engine, _stream_queues, _seq_id_to_request_id global _stream_loops, _request_start_times @@ -710,7 +737,7 @@ def do_preprocess(): logger.info(f"API: Created request_id={request_id}, seq_id={seq_id}") engine.core_mgr.add_request([seq]) - return seq_id, stream_queue + return seq_id, stream_queue, seq.num_prompt_tokens def cleanup_streaming_request(request_id: str, seq_id: int) -> None: @@ -736,12 +763,16 @@ async def setup_streaming_request_fanout( request_id: str, kv_transfer_params: Optional[Dict[str, Any]] = None, multimodal_data: Optional[Dict[str, Any]] = None, -) -> Tuple[List[int], asyncio.Queue]: +) -> Tuple[List[int], asyncio.Queue, int]: """Fan-out variant of :func:`setup_streaming_request`. Creates ``sampling_params.n`` sibling sequences sharing one output queue. Every callback pushes ``(sibling_index, chunk_data)`` tuples so the merge-stream consumer can rewrite ``choices[0].index`` correctly. + + Returns ``(seq_ids, shared_queue, num_prompt_tokens)``. All siblings + tokenize the same prompt once, so ``num_prompt_tokens`` is shared and lets + the stream response generator skip re-tokenizing on the event loop. """ global engine, _stream_queues, _seq_id_to_request_id global _stream_loops, _request_start_times @@ -784,7 +815,7 @@ def do_preprocess(): f"API: Created fan-out request_id={request_id}, n={n}, seq_ids={seq_ids}" ) engine.core_mgr.add_request(seqs) - return seq_ids, shared_queue + return seq_ids, shared_queue, seqs[0].num_prompt_tokens # ============================================================================ @@ -871,9 +902,14 @@ async def chat_completions(request: ChatCompletionRequest): is_multimodal = _has_multimodal_content(messages) if is_multimodal: - token_ids, multimodal_data = _prepare_multimodal_inputs( - messages, - merged_kwargs, + # Image loading (blocking network I/O, up to a 30s urlopen) plus + # processor preprocessing are heavy and would stall the event loop; + # run them in a worker thread. Warm the processor on the loop first + # so concurrent cold-start requests don't race on its lazy init. + _get_multimodal_processor() + loop = asyncio.get_running_loop() + token_ids, multimodal_data = await loop.run_in_executor( + None, _prepare_multimodal_inputs, messages, merged_kwargs ) else: prompt = apply_chat_template( @@ -889,24 +925,25 @@ async def chat_completions(request: ChatCompletionRequest): stream_input = token_ids if is_multimodal else prompt stream_multimodal_data = multimodal_data if is_multimodal else None if effective_n > 1: - seq_ids, stream_queue = await setup_streaming_request_fanout( - stream_input, - sampling_params, - request_id, - multimodal_data=stream_multimodal_data, - kv_transfer_params=request.kv_transfer_params, + seq_ids, stream_queue, num_prompt_tokens = ( + await setup_streaming_request_fanout( + stream_input, + sampling_params, + request_id, + multimodal_data=stream_multimodal_data, + kv_transfer_params=request.kv_transfer_params, + ) ) gen = stream_chat_response_fanout( request_id, model_name, - stream_input, stream_queue, seq_ids, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) else: - seq_id, stream_queue = await setup_streaming_request( + seq_id, stream_queue, num_prompt_tokens = await setup_streaming_request( stream_input, sampling_params, request_id, @@ -916,10 +953,9 @@ async def chat_completions(request: ChatCompletionRequest): gen = stream_chat_response( request_id, model_name, - stream_input, stream_queue, seq_id, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) return StreamingResponse( @@ -1014,23 +1050,24 @@ async def completions(request: CompletionRequest): # Streaming if request.stream: if effective_n > 1: - seq_ids, stream_queue = await setup_streaming_request_fanout( - request.prompt, - sampling_params, - request_id, - kv_transfer_params=request.kv_transfer_params, + seq_ids, stream_queue, num_prompt_tokens = ( + await setup_streaming_request_fanout( + request.prompt, + sampling_params, + request_id, + kv_transfer_params=request.kv_transfer_params, + ) ) gen = stream_completion_response_fanout( request_id, model_name, - request.prompt, stream_queue, seq_ids, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) else: - seq_id, stream_queue = await setup_streaming_request( + seq_id, stream_queue, num_prompt_tokens = await setup_streaming_request( request.prompt, sampling_params, request_id, @@ -1039,10 +1076,9 @@ async def completions(request: CompletionRequest): gen = stream_completion_response( request_id, model_name, - request.prompt, stream_queue, seq_id, - tokenizer, + num_prompt_tokens, cleanup_streaming_request, ) return StreamingResponse( @@ -1155,7 +1191,7 @@ async def anthropic_messages(request: AnthropicMessagesRequest, raw_request: Req if request.stream: # Streaming response - seq_id, stream_queue = await setup_streaming_request( + seq_id, stream_queue, _num_prompt_tokens = await setup_streaming_request( prompt, sampling_params, request_id ) @@ -1520,8 +1556,24 @@ def _sigint_handler(signum, frame): signal.signal(signal.SIGINT, _sigint_handler) - logger.info(f"Starting server on {args.host}:{args.server_port}...") - uvicorn.run(app, host=args.host, port=args.server_port) + # uvloop replaces the stdlib asyncio selector loop with a libuv-backed one, + # which is markedly faster at the SSE socket I/O (sock.send / selector + # register-unregister) that saturates the event loop under high streaming + # concurrency. Fall back to the default loop if uvloop is unavailable. + try: + import uvloop # noqa: F401 + + loop_impl = "uvloop" + except ImportError: + loop_impl = "auto" + logger.warning( + "uvloop not installed; falling back to the default asyncio loop." + ) + + logger.info( + f"Starting server on {args.host}:{args.server_port} (loop={loop_impl})..." + ) + uvicorn.run(app, host=args.host, port=args.server_port, loop=loop_impl) if __name__ == "__main__": diff --git a/atom/entrypoints/openai/serving_chat.py b/atom/entrypoints/openai/serving_chat.py index d12692bae8..41fc6abb40 100644 --- a/atom/entrypoints/openai/serving_chat.py +++ b/atom/entrypoints/openai/serving_chat.py @@ -7,7 +7,7 @@ import json import logging import time -from typing import Any, AsyncGenerator, Dict, List, Optional, Union +from typing import Any, AsyncGenerator, Dict, List, Optional from .protocol import ( CHAT_COMPLETION_CHUNK_OBJECT, @@ -20,12 +20,6 @@ logger = logging.getLogger("atom") -def _prompt_token_count(prompt_or_tokens: Union[str, List[int]], tokenizer) -> int: - if isinstance(prompt_or_tokens, str): - return len(tokenizer.encode(prompt_or_tokens)) - return len(prompt_or_tokens) - - def create_chat_chunk( request_id: str, model: str, @@ -61,10 +55,9 @@ def create_chat_chunk( async def stream_chat_response( request_id: str, model: str, - prompt: Union[str, List[int]], stream_queue: asyncio.Queue, seq_id: int, - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: """Generate streaming chat completion response with reasoning and tool calls. @@ -73,8 +66,12 @@ async def stream_chat_response( - reasoning_content deltas during thinking phase - content deltas for the answer - tool_calls deltas when model invokes tools + + ``num_prompt_tokens`` is the engine-computed prompt length (``Sequence. + num_prompt_tokens``); reusing it avoids re-tokenizing the prompt on the + event loop at stream start. """ - num_tokens_input = _prompt_token_count(prompt, tokenizer) + num_tokens_input = num_prompt_tokens num_tokens_output = 0 reasoning_filter = ReasoningFilter() tool_parser = ToolCallStreamParser() @@ -152,7 +149,6 @@ async def stream_chat_response( "completion_tokens": num_tokens_output, "total_tokens": num_tokens_input + num_tokens_output, } - yield create_chat_chunk(request_id, model, finish_reason=finish_reason) usage_chunk = { "id": request_id, "object": CHAT_COMPLETION_CHUNK_OBJECT, @@ -162,8 +158,14 @@ async def stream_chat_response( } if kv_transfer_params_value is not None: usage_chunk["kv_transfer_params"] = kv_transfer_params_value - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + # Coalesce finish + usage + [DONE] into one send: at a wave boundary many + # requests finalize at once, so collapsing 3 socket writes/req to 1 cuts + # the syscalls that saturate the API event loop. + yield ( + create_chat_chunk(request_id, model, finish_reason=finish_reason) + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) def _build_chat_choice( @@ -271,10 +273,9 @@ def build_chat_response_multi( async def stream_chat_response_fanout( request_id: str, model: str, - prompt: Union[str, List[int]], shared_queue: asyncio.Queue, seq_ids: List[int], - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: """Streaming variant that multiplexes ``len(seq_ids)`` fan-out siblings @@ -283,9 +284,13 @@ async def stream_chat_response_fanout( The shared queue receives ``(sibling_index, chunk_data)`` tuples from the engine callbacks registered in :func:`setup_streaming_request_fanout`. Reasoning + tool-call state is kept independently per sibling. + + ``num_prompt_tokens`` is the engine-computed prompt length shared by all + siblings (they tokenize the same prompt once); reusing it avoids + re-tokenizing on the event loop at stream start. """ n = len(seq_ids) - num_tokens_input = _prompt_token_count(prompt, tokenizer) + num_tokens_input = num_prompt_tokens num_tokens_output = [0] * n reasoning_filters = [ReasoningFilter() for _ in range(n)] tool_parsers = [ToolCallStreamParser() for _ in range(n)] @@ -369,10 +374,6 @@ async def stream_chat_response_fanout( for sid in seq_ids: cleanup_fn(request_id, sid) - for i in range(n): - finish_reason = "tool_calls" if has_tool_calls[i] else "stop" - yield create_chat_chunk(request_id, model, finish_reason=finish_reason, index=i) - usage = { "prompt_tokens": num_tokens_input, "completion_tokens": sum(num_tokens_output), @@ -388,5 +389,17 @@ async def stream_chat_response_fanout( } if kv_transfer_params_value is not None: usage_chunk["kv_transfer_params"] = kv_transfer_params_value - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + # Coalesce the per-sibling finish chunks + usage + [DONE] into one send. + yield ( + "".join( + create_chat_chunk( + request_id, + model, + finish_reason="tool_calls" if has_tool_calls[i] else "stop", + index=i, + ) + for i in range(n) + ) + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) diff --git a/atom/entrypoints/openai/serving_completion.py b/atom/entrypoints/openai/serving_completion.py index fc06c627c8..47eb2e29a5 100644 --- a/atom/entrypoints/openai/serving_completion.py +++ b/atom/entrypoints/openai/serving_completion.py @@ -55,14 +55,18 @@ def create_completion_chunk( async def stream_completion_response( request_id: str, model: str, - prompt: str, stream_queue: asyncio.Queue, seq_id: int, - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: - """Generate streaming text completion response.""" - num_tokens_input = len(tokenizer.encode(prompt)) + """Generate streaming text completion response. + + ``num_prompt_tokens`` is the engine-computed prompt length (``Sequence. + num_prompt_tokens``); reusing it avoids re-tokenizing the prompt on the + event loop at stream start. + """ + num_tokens_input = num_prompt_tokens num_tokens_output = 0 while True: @@ -74,7 +78,7 @@ async def stream_completion_response( if "kv_transfer_params" in chunk_data: extra_fields["kv_transfer_params"] = chunk_data["kv_transfer_params"] - yield create_completion_chunk( + content_chunk = create_completion_chunk( request_id, model, new_text, @@ -83,26 +87,31 @@ async def stream_completion_response( ) if chunk_data.get("finished", False): - break + # Coalesce the finalization SSE messages (content + stop + usage + + # [DONE]) into a single send. At a wave boundary many requests + # finish simultaneously; collapsing 4 sends/req to 1 cuts the + # per-request socket-write syscalls that saturate the API loop. + cleanup_fn(request_id, seq_id) + usage_chunk = { + "id": request_id, + "object": TEXT_COMPLETION_OBJECT, + "created": int(time.time()), + "model": model, + "usage": { + "prompt_tokens": num_tokens_input, + "completion_tokens": num_tokens_output, + "total_tokens": num_tokens_input + num_tokens_output, + }, + } + yield ( + content_chunk + + create_completion_chunk(request_id, model, "", "stop") + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) + return - cleanup_fn(request_id, seq_id) - - usage = { - "prompt_tokens": num_tokens_input, - "completion_tokens": num_tokens_output, - "total_tokens": num_tokens_input + num_tokens_output, - } - yield create_completion_chunk(request_id, model, "", "stop") - # Usage-only chunk - usage_chunk = { - "id": request_id, - "object": TEXT_COMPLETION_OBJECT, - "created": int(time.time()), - "model": model, - "usage": usage, - } - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + yield content_chunk def build_completion_response( @@ -184,10 +193,9 @@ def build_completion_response_multi( async def stream_completion_response_fanout( request_id: str, model: str, - prompt: str, shared_queue: asyncio.Queue, seq_ids: List[int], - tokenizer, + num_prompt_tokens: int, cleanup_fn, ) -> AsyncGenerator[str, None]: """Streaming variant multiplexing ``len(seq_ids)`` siblings into one SSE. @@ -195,9 +203,13 @@ async def stream_completion_response_fanout( Each chunk pulled from ``shared_queue`` is a ``(sibling_index, chunk_data)`` tuple; we re-emit with ``choices[0].index = sibling_index``. Finishes only when every sibling has reported ``finished=True``. + + ``num_prompt_tokens`` is the engine-computed prompt length shared by all + siblings; reusing it avoids re-tokenizing on the event loop at stream + start. """ n = len(seq_ids) - num_tokens_input = len(tokenizer.encode(prompt)) + num_tokens_input = num_prompt_tokens num_tokens_output = [0] * n finished = [False] * n @@ -227,9 +239,6 @@ async def stream_completion_response_fanout( for sid in seq_ids: cleanup_fn(request_id, sid) - for i in range(n): - yield create_completion_chunk(request_id, model, "", "stop", index=i) - usage = { "prompt_tokens": num_tokens_input, "completion_tokens": sum(num_tokens_output), @@ -243,5 +252,12 @@ async def stream_completion_response_fanout( "model": model, "usage": usage, } - yield f"data: {json.dumps(usage_chunk)}\n\n" - yield STREAM_DONE_MESSAGE + # Coalesce the per-sibling stop chunks + usage + [DONE] into one send. + yield ( + "".join( + create_completion_chunk(request_id, model, "", "stop", index=i) + for i in range(n) + ) + + f"data: {json.dumps(usage_chunk)}\n\n" + + STREAM_DONE_MESSAGE + ) diff --git a/atom/models/gpt_oss.py b/atom/models/gpt_oss.py index bf49decd13..ed53777381 100644 --- a/atom/models/gpt_oss.py +++ b/atom/models/gpt_oss.py @@ -336,11 +336,18 @@ def __init__( self.config = atom_config.hf_config self.quant_config = atom_config.quant_config self.config.hidden_size = self.config.hidden_size - self.embedding = VocabParallelEmbedding( + # Register `embed_tokens` first so it stays the primary (non-deduped) + # name reported by `named_parameters()`. The checkpoint stores this + # tensor as `model.embed_tokens.weight`; if `embedding` were the primary + # name instead, the load-completeness check would falsely flag + # `model.embedding.weight` as unloaded (the weight is in fact loaded via + # the shared-storage alias). `embedding` remains as an alias for the + # internal call sites below. + self.embed_tokens = VocabParallelEmbedding( self.config.vocab_size, self.config.hidden_size, ) - self.embed_tokens = self.embedding + self.embedding = self.embed_tokens self.start_layer, self.end_layer, self.layers = make_layers( self.config.num_hidden_layers, lambda prefix, layer_num=None: TransformerBlock( diff --git a/pyproject.toml b/pyproject.toml index e71ae4425a..528375be31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" description = "a lightweight vLLM implementation built from scratch" requires-python = ">=3.10" dynamic = ["version"] -dependencies = ["pybind11", "transformers==5.2.0", "zmq", "msgspec", "xxhash", "fastapi>=0.115,<0.137", "psutil", "protobuf", "uvicorn", "aiohttp", "datasets", "openpyxl", "tqdm"] +dependencies = ["pybind11", "transformers==5.2.0", "zmq", "msgspec", "xxhash", "fastapi>=0.115,<0.137", "psutil", "protobuf", "uvicorn", "uvloop", "aiohttp", "datasets", "openpyxl", "tqdm"] [project.urls] Homepage = "https://github.com/ROCm/ATOM"