Skip to content
Open
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,7 @@ jobs:
uses: $/.github/workflows/shared-test.yml
with:
test_suite_name: triton
framework: triton
test_type: CPU Test
amd_runner: prod-tester-amd-v2
target_tag_plain: ${{ needs.triton-build.outputs.target_tag_plain }}
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/router/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
import logging
import os

from prefix_data_generator.synthesizer import Synthesizer

# Default values
DEFAULT_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
DEFAULT_URL = "http://localhost:8000"
Expand Down Expand Up @@ -304,6 +302,8 @@ def prepare_trace_dataset(args, output_dir, logger):
return requests, trace_dataset_path

# Generate synthetic data based on input dataset
from prefix_data_generator.synthesizer import Synthesizer
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
Comment thread
glamr-agent marked this conversation as resolved.
Resolved

logger.info("Generating synthetic trace data...")
logger.info(f" Base dataset: {args.input_dataset}")
logger.info(f" Num requests: {args.num_requests if args.num_requests else 'all'}")
Expand Down
4 changes: 2 additions & 2 deletions lib/llm/src/http/service/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1545,7 +1545,7 @@ impl InflightGuard {

tracing::Span::current().record("model", model.as_str());

tracing::info!(
tracing::debug!(
request_id = %request_id,
model = %model,
endpoint = %endpoint,
Expand Down Expand Up @@ -1668,7 +1668,7 @@ impl Drop for InflightGuard {
);
}
RequestCompletion::Success => {
tracing::info!(
tracing::debug!(
request_id = %self.request_id,
model = %self.model,
endpoint = %self.endpoint,
Expand Down
4 changes: 2 additions & 2 deletions lib/runtime/src/pipeline/network/ingress/push_handler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ impl Drop for RequestMetricsGuard {
self.request_duration
.observe(self.start_time.elapsed().as_secs_f64());
if let Some(request_id) = &self.request_id {
tracing::info!(request_id = %request_id, "request completed");
tracing::debug!(request_id = %request_id, "request completed");
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
}
}
}
Expand Down Expand Up @@ -780,7 +780,7 @@ where
m.inflight_requests.inc();
m.request_bytes.inc_by(payload.len() as u64);
if let Some(rid) = &request_id {
tracing::info!(request_id = %rid, "request received");
tracing::debug!(request_id = %rid, "request received");
}
RequestMetricsGuard {
inflight_requests: m.inflight_requests.clone(),
Expand Down
160 changes: 118 additions & 42 deletions tests/frontend/test_request_tracing_logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,21 @@
Verifies that JSONL logs contain consistent structured fields for all request
lifecycle events: "request received", "http response sent", "request completed".

The routine per-request lifecycle events ("request received", and "request
completed" on the success path) are emitted at DEBUG so they do not dominate a
default-level production log; the fixtures below therefore run with
`DYN_LOG=debug` to observe them. Abnormal completions stay at ERROR and the
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
Axum access log ("http response sent") stays at INFO, both of which remain
visible at the default level — see `test_agg_lifecycle_absent_at_info_level`.

Tests cover: unary success, streaming success, 404 error, 400 invalid UUID,
cancellation, frontend-worker trace_id correlation, aggregated deployment,
and disaggregated (prefill+decode) deployment.
"""

from __future__ import annotations

import contextlib
import json
import logging
import os
Expand Down Expand Up @@ -116,18 +124,37 @@ def get_request_logs(process, request_id: str) -> List[Dict[str, Any]]:
return find_logs_by_request_id(parse_jsonl_logs(read_log_file(process)), request_id)


def wait_for_request_logs(
process, request_id: str, expected_messages: set[str], timeout: float = 10.0
) -> List[Dict[str, Any]]:
"""Wait for request-scoped structured events to reach the process log."""
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
deadline = time.monotonic() + timeout
while True:
req_logs = get_request_logs(process, request_id)
messages = {e.get("message") for e in req_logs}
if expected_messages <= messages:
return req_logs
if time.monotonic() >= deadline:
pytest.fail(
f"Timed out waiting for request {request_id} log messages "
f"{expected_messages}; observed {messages}"
)
time.sleep(0.1)


def assert_lifecycle_logs(req_logs, expected_status="success"):
"""Assert received/completed/http_sent exist and return them."""
received = [e for e in req_logs if e.get("message") == "request received"]
completed = [e for e in req_logs if e.get("message") == "request completed"]
http_sent = [e for e in req_logs if e.get("message") == "http response sent"]
msgs = [e.get("message") for e in req_logs]
lifecycle_hint = " (DEBUG-level event: does the fixture env set DYN_LOG=debug?)"
assert (
len(received) == 1
), f"Expected 1 'request received', got {len(received)}: {msgs}"
), f"Expected 1 'request received', got {len(received)}: {msgs}{lifecycle_hint}"
assert (
len(completed) == 1
), f"Expected 1 'request completed', got {len(completed)}: {msgs}"
), f"Expected 1 'request completed', got {len(completed)}: {msgs}{lifecycle_hint}"
assert (
len(http_sent) == 1
), f"Expected 1 'http response sent', got {len(http_sent)}: {msgs}"
Expand Down Expand Up @@ -170,35 +197,33 @@ def assert_error_completion(req_logs):
return completed


JSONL_ENV = {"DYN_LOGGING_JSONL": "1", "DYN_LOG": "info"}
# Lifecycle assertions require DEBUG; the INFO fixture checks filtering.
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
JSONL_ENV = {"DYN_LOGGING_JSONL": "1", "DYN_LOG": "debug"}
Comment thread
MatejKosec marked this conversation as resolved.
Resolved
JSONL_ENV_INFO = {"DYN_LOGGING_JSONL": "1", "DYN_LOG": "info"}


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(scope="function")
def tracing_services(
request,
runtime_services_dynamic_ports,
dynamo_dynamic_ports,
predownload_tokenizers,
):
"""Aggregated frontend + mocker with JSONL logging."""
ports = dynamo_dynamic_ports
@contextlib.contextmanager
def _agg_services(request, ports, env, speedup_ratio=None):
"""Start an aggregated frontend and mocker."""
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
worker_kwargs = {} if speedup_ratio is None else {"speedup_ratio": speedup_ratio}
with DynamoFrontendProcess(
request,
frontend_port=ports.frontend_port,
terminate_all_matching_process_names=False,
extra_env=JSONL_ENV,
extra_env=env,
) as frontend:
with MockerWorkerProcess(
request,
model=TEST_MODEL,
frontend_port=ports.frontend_port,
system_port=ports.system_ports[0],
extra_env=JSONL_ENV,
extra_env=env,
**worker_kwargs,
) as worker:
wait_for_http_completions_ready(
frontend_port=ports.frontend_port, model=TEST_MODEL
Expand All @@ -210,6 +235,30 @@ def tracing_services(
}


@pytest.fixture(scope="function")
def tracing_services(
request,
runtime_services_dynamic_ports,
dynamo_dynamic_ports,
predownload_tokenizers,
):
"""Aggregated frontend + mocker with JSONL logging at DYN_LOG=debug."""
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
with _agg_services(request, dynamo_dynamic_ports, JSONL_ENV) as services:
yield services


@pytest.fixture(scope="function")
def tracing_services_info_level(
request,
runtime_services_dynamic_ports,
dynamo_dynamic_ports,
predownload_tokenizers,
):
"""Aggregated frontend + mocker at the default DYN_LOG=info level."""
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
with _agg_services(request, dynamo_dynamic_ports, JSONL_ENV_INFO) as services:
yield services


@pytest.fixture(scope="function")
def tracing_services_slow(
request,
Expand All @@ -218,29 +267,10 @@ def tracing_services_slow(
predownload_tokenizers,
):
"""Aggregated frontend + slow mocker for cancellation/crash testing."""
ports = dynamo_dynamic_ports
with DynamoFrontendProcess(
request,
frontend_port=ports.frontend_port,
terminate_all_matching_process_names=False,
extra_env=JSONL_ENV,
) as frontend:
with MockerWorkerProcess(
request,
model=TEST_MODEL,
frontend_port=ports.frontend_port,
system_port=ports.system_ports[0],
speedup_ratio=0.1,
extra_env=JSONL_ENV,
) as worker:
wait_for_http_completions_ready(
frontend_port=ports.frontend_port, model=TEST_MODEL
)
yield {
"frontend_port": ports.frontend_port,
"frontend": frontend,
"worker": worker,
}
with _agg_services(
request, dynamo_dynamic_ports, JSONL_ENV, speedup_ratio=0.1
) as services:
yield services


@pytest.fixture(scope="function")
Expand Down Expand Up @@ -296,24 +326,32 @@ def tracing_services_disagg(
# ---------------------------------------------------------------------------


# Keep the DEBUG contract covered by the generic runtime CI selector.
@pytest.mark.pre_merge
def test_agg_unary_success(tracing_services) -> None:
"""Aggregated unary: full lifecycle logs + token counts + worker logs."""
port = tracing_services["frontend_port"]
rid = str(uuid.uuid4())

resp = _send_chat_completions(port, request_id=rid)
assert resp.status_code == 200
time.sleep(1)

req_logs = get_request_logs(tracing_services["frontend"], rid)
req_logs = wait_for_request_logs(
tracing_services["frontend"],
rid,
{"request received", "request completed", "http response sent"},
)
received, completed, http_sent = assert_lifecycle_logs(req_logs)

assert received[0]["level"] == "INFO"
assert received[0]["level"] == "DEBUG"
assert completed[0]["level"] == "DEBUG"
assert received[0].get("x_request_id") == rid
assert "request_id" in received[0]
assert received[0].get("request_id"), "request_id must survive the DEBUG demotion"
assert completed[0].get("request_id") == received[0]["request_id"]
assert "model" in received[0]
assert "endpoint" in received[0]
assert "elapsed_ms" in completed[0]
assert http_sent[0]["level"] == "INFO"
assert http_sent[0].get("status") == "200"

# Token counts on inference span
Expand All @@ -324,15 +362,20 @@ def test_agg_unary_success(tracing_services) -> None:

# Worker lifecycle — verify both x_request_id and request_id propagated
server_rid = received[0].get("request_id")
wk_logs = get_request_logs(tracing_services["worker"], rid)
wk_logs = wait_for_request_logs(
tracing_services["worker"], rid, {"request received", "request completed"}
)
wk_received = [e for e in wk_logs if e.get("message") == "request received"]
wk_completed = [e for e in wk_logs if e.get("message") == "request completed"]
assert len(wk_received) == 1, "Worker should log 1 'request received'"
assert len(wk_completed) == 1, "Worker should log 1 'request completed'"
assert wk_received[0]["level"] == "DEBUG"
assert wk_completed[0]["level"] == "DEBUG"
assert wk_received[0].get("x_request_id") == rid, "Worker should have x_request_id"
assert (
wk_received[0].get("request_id") == server_rid
), "Worker request_id should match frontend"
assert wk_completed[0].get("request_id") == server_rid


def test_agg_streaming_success(tracing_services) -> None:
Expand Down Expand Up @@ -380,6 +423,39 @@ def test_agg_404_error(tracing_services) -> None:
assert http_sent[0].get("status") == "404"


# The generic runtime jobs select pre_merge in both PR and post-merge CI.
@pytest.mark.pre_merge
def test_agg_lifecycle_absent_at_info_level(tracing_services_info_level) -> None:
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
"""Routine lifecycle events are filtered at DYN_LOG=info."""
port = tracing_services_info_level["frontend_port"]
rid = str(uuid.uuid4())

resp = _send_chat_completions(port, request_id=rid)
assert resp.status_code == 200
fe_logs = wait_for_request_logs(
tracing_services_info_level["frontend"], rid, {"http response sent"}
)

for name in ("frontend", "worker"):
req_logs = get_request_logs(tracing_services_info_level[name], rid)
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
lifecycle = [
e
for e in req_logs
if e.get("message") in ("request received", "request completed")
]
assert not lifecycle, (
f"{name} emitted routine lifecycle events at DYN_LOG=info: "
f"{[(e.get('message'), e.get('level')) for e in lifecycle]}"
)

http_sent = [e for e in fe_logs if e.get("message") == "http response sent"]
assert len(http_sent) == 1, (
"Expected the INFO-level access log to still be present, got: "
f"{[e.get('message') for e in fe_logs]}"
)
assert http_sent[0].get("status") == "200"


def test_agg_invalid_uuid_warn(tracing_services) -> None:
"""Invalid x-dynamo-request-id: WARN logged, request proceeds with generated ID."""
port = tracing_services["frontend_port"]
Expand Down
Loading