From e60d782656c4214bdbc739556a702aabd6521d36 Mon Sep 17 00:00:00 2001 From: Jie Hao Date: Wed, 4 Nov 2026 17:52:40 -0500 Subject: [PATCH] [None][fix] Count prompt tokens without materializing the prompt list GenerationRequest keeps a flat array prompt as an int32 buffer and builds the Python list lazily in the prompt_token_ids property. The per-request metrics paths read prompt_token_ids only to take its length: GenerationResultBase.record_stats, the postprocess-worker PROMPT_TOKENS fill-in in _handle_response, and the tracing span. With per-request perf metrics on, every finished request therefore built (and later freed) its whole prompt as a list of Python ints, O(ISL), in the serving process. At long agentic prompts this saturates a single serving process. Dynamo's TensorRT-LLM worker turns per-request perf metrics on with its metrics switch and handles every request of an engine in one process; on GB300 DeepSeek-V4.1-Flash disaggregated serving (~140K-token prompts) it cut decode throughput by up to 37%. Counting from the buffer removed the loss. Add GenerationRequest.prompt_len, which counts the list or the int32 buffer without building the list, and use it on those three paths. Prefill perplexity, which needs the ids themselves, still reads prompt_token_ids, now only when prompt logprobs are present. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jie Hao --- tensorrt_llm/executor/request.py | 13 +++++ tensorrt_llm/executor/result.py | 31 ++++++++---- .../test_generation_request_array_prompt.py | 21 ++++++++ tests/unittest/llmapi/test_executor.py | 48 ++++++++++++++++++- 4 files changed, 102 insertions(+), 11 deletions(-) diff --git a/tensorrt_llm/executor/request.py b/tensorrt_llm/executor/request.py index b18d350bd59f..e57f7c779bbd 100644 --- a/tensorrt_llm/executor/request.py +++ b/tensorrt_llm/executor/request.py @@ -250,6 +250,19 @@ def prompt_token_ids(self): # `_prompt_token_ids_i32`; base_worker assumes both forms stay synced. return ptids + @property + def prompt_len(self) -> int: + """Number of prompt tokens, read without materializing the lazy list. + + Equals ``len(self.prompt_token_ids)``, but an int32-buffered prompt is + counted from its buffer, so callers that only need the count do not + pay the O(ISL) ``.tolist()``. + """ + ptids = self.__dict__.get("_prompt_token_ids") + if ptids is None: + ptids = self.__dict__.get("_prompt_token_ids_i32") + return len(ptids) if ptids is not None else 0 + @prompt_token_ids.setter def prompt_token_ids(self, value): self._prompt_token_ids = value diff --git a/tensorrt_llm/executor/result.py b/tensorrt_llm/executor/result.py index 43bcee12ff40..9706522309ab 100644 --- a/tensorrt_llm/executor/result.py +++ b/tensorrt_llm/executor/result.py @@ -617,12 +617,10 @@ def _handle_response(self, # count prompt tokens. It reports cached tokens only for a # request's finished first candidate, which is where # `record_stats` counts the prompt too. - prompt_token_ids = getattr(self, "prompt_token_ids", None) if (MetricNames.PROMPT_CACHE_CACHED_TOKENS in response.metrics and MetricNames.PROMPT_TOKENS not in self.metrics_dict - and prompt_token_ids): - self.metrics_dict[MetricNames.PROMPT_TOKENS] = len( - prompt_token_ids) + and (prompt_len := self._prompt_len())): + self.metrics_dict[MetricNames.PROMPT_TOKENS] = prompt_len if self._done: req_perf_metrics_dict = _build_perf_metrics_dict( @@ -789,6 +787,19 @@ def _handle_response(self, else: raise ValueError(f"Unknown response type: {response}") + def _prompt_len(self) -> int: + """Number of prompt tokens, for metrics and tracing. + + Reads ``GenerationRequest.prompt_len`` when this result has a request: + ``len(self.prompt_token_ids)`` would materialize an int32-buffered + prompt as a Python list (O(ISL)) on every finished request. + """ + request = getattr(self, "_generation_request", None) + if request is not None: + return request.prompt_len + prompt_token_ids = getattr(self, "prompt_token_ids", None) + return len(prompt_token_ids) if prompt_token_ids is not None else 0 + def record_stats(self, output: CompletionOutput, stats: Optional[dict[str, float]] = None, @@ -819,10 +830,9 @@ def record_stats(self, metrics_stats.update(processed_metrics_stat) # Record prompt tokens only for the first candidate to avoid # double-counting the shared prompt across n candidates. - prompt_token_ids = getattr(self, "prompt_token_ids", None) if output.finish_reason and sequence_index == 0: - if prompt_token_ids is not None and len(prompt_token_ids) > 0: - metrics_stats[MetricNames.PROMPT_TOKENS] = len(prompt_token_ids) + if (prompt_len := self._prompt_len()) > 0: + metrics_stats[MetricNames.PROMPT_TOKENS] = prompt_len # Request-scoped metrics: only record for the first candidate to avoid # double-counting across n candidates. @@ -846,6 +856,9 @@ def record_stats(self, metrics_stats[MetricNames.SPEC_DEC_DRAFTED_PER_POS] = \ [spec_dec.total_draft_tokens] + # Only prefill perplexity needs the prompt ids themselves. + prompt_token_ids = getattr(self, "prompt_token_ids", + None) if output.prompt_logprobs else None if output.prompt_logprobs and prompt_token_ids: try: prompt_lps = [] @@ -959,10 +972,10 @@ def safe_set_attr(span, attr, value): sampling_params.n) safe_set_attr(span, tracing.SpanAttributes.GEN_AI_REQUEST_ID, self.id) - if prompt_token_ids := getattr(self, "prompt_token_ids", None): + if prompt_len := self._prompt_len(): safe_set_attr(span, tracing.SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS, - len(prompt_token_ids)) + prompt_len) safe_set_attr(span, tracing.SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS, output.length) diff --git a/tests/unittest/executor/test_generation_request_array_prompt.py b/tests/unittest/executor/test_generation_request_array_prompt.py index b678e435a316..85dd0ffd45cd 100644 --- a/tests/unittest/executor/test_generation_request_array_prompt.py +++ b/tests/unittest/executor/test_generation_request_array_prompt.py @@ -66,3 +66,24 @@ def test_replacing_the_prompt_drops_the_int32_buffer(): np.testing.assert_array_equal( clone.__dict__["_prompt_token_ids_i32"], np.array([6, 7, 8], dtype=np.int32) ) + + +@pytest.mark.parametrize("as_tensor", [False, True]) +def test_prompt_len_counts_an_int32_prompt_without_a_list(as_tensor): + ids = _ids() + prompt = torch.from_numpy(ids.copy()) if as_tensor else ids + req = GenerationRequest(prompt_token_ids=prompt, sampling_params=SamplingParams(max_tokens=1)) + assert req.prompt_len == len(ids) + assert req.__dict__["_prompt_token_ids"] is None + clone = pickle.loads(pickle.dumps(req)) + assert clone.prompt_len == len(ids) + assert clone.__dict__["_prompt_token_ids"] is None + + +@pytest.mark.parametrize("prompt", [[3, 4, 5], _ids(), np.array([[1, 2], [3, 4]], dtype=np.int32)]) +def test_prompt_len_matches_the_prompt_list(prompt): + req = GenerationRequest(prompt_token_ids=prompt, sampling_params=SamplingParams(max_tokens=1)) + length = req.prompt_len + assert length == len(req.prompt_token_ids) + # Once the list is materialized, prompt_len reads it. + assert req.prompt_len == length diff --git a/tests/unittest/llmapi/test_executor.py b/tests/unittest/llmapi/test_executor.py index 6dbe016150c0..e9f8aaa13f0c 100644 --- a/tests/unittest/llmapi/test_executor.py +++ b/tests/unittest/llmapi/test_executor.py @@ -14,6 +14,7 @@ from types import SimpleNamespace from unittest.mock import Mock +import numpy as np import pytest import torch import zmq @@ -21,7 +22,8 @@ from tensorrt_llm._utils import mpi_world_size, nvtx_range_debug from tensorrt_llm.bindings import executor as tllm from tensorrt_llm.disaggregated_params import DisaggregatedParams -from tensorrt_llm.executor import (DetokenizedGenerationResultBase, +from tensorrt_llm.executor import (CompletionOutput, + DetokenizedGenerationResultBase, GenerationRequest, GenerationResult, GenerationResultBase, PostprocWorker) from tensorrt_llm.executor.ipc import FusedIpcQueue, ZeroMqQueue @@ -30,7 +32,7 @@ from tensorrt_llm.inputs.multimodal import MultimodalInput, MultimodalParams from tensorrt_llm.llmapi.tokenizer import TransformersTokenizer from tensorrt_llm.llmapi.utils import AsyncQueue -from tensorrt_llm.metrics import MetricNames +from tensorrt_llm.metrics import MetricNames, RequestEventTiming from tensorrt_llm.sampling_params import SamplingParams # isort: off @@ -497,6 +499,48 @@ def test_handle_response_postproc_adds_prompt_tokens(): assert result.metrics_dict[MetricNames.PROMPT_CACHE_CACHED_TOKENS] == 256 +def _int32_prompt_result(prompt_len: int): + request = GenerationRequest( + prompt_token_ids=np.arange(prompt_len, dtype=np.int32), + sampling_params=SamplingParams(max_tokens=4)).set_id(1) + return request, GenerationResult(request) + + +def test_record_stats_counts_an_int32_prompt_without_a_list(): + """PROMPT_TOKENS comes from the request's int32 prompt buffer. + + A finished request's prompt-token count must not materialize the prompt + as a Python list (O(ISL) per request) when only its length is needed. + """ + request, result = _int32_prompt_result(4096) + output = CompletionOutput(index=0, + token_ids=[1, 2, 3, 4], + finish_reason="length") + result.record_stats( + output, { + RequestEventTiming.ARRIVAL_TIME: 1.0, + RequestEventTiming.FIRST_SCHEDULED_TIME: 1.1, + RequestEventTiming.FIRST_TOKEN_TIME: 1.5, + RequestEventTiming.LAST_TOKEN_TIME: 2.0, + }) + assert result.metrics_dict[MetricNames.PROMPT_TOKENS] == 4096 + assert request.__dict__["_prompt_token_ids"] is None + + +def test_handle_response_postproc_counts_an_int32_prompt_without_a_list(): + request, result = _int32_prompt_result(300) + result._handle_response( + PostprocWorker.Output( + client_id=1, + res="mock_sse_data", + is_final=True, + metrics={MetricNames.PROMPT_CACHE_CACHED_TOKENS: 256}, + finish_reason="length", + num_generated_tokens=1)) + assert result.metrics_dict[MetricNames.PROMPT_TOKENS] == 300 + assert request.__dict__["_prompt_token_ids"] is None + + def _ZeroMqQueue_sync_sync_task(addr: str): print(f"Setup receiver: {addr}") pull_pipe = ZeroMqQueue(address=addr, is_server=False, is_async=True)