Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lib/llm/src/http/service/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1382,7 +1382,7 @@ impl InflightGuard {

tracing::Span::current().record("model", model.as_str());

tracing::info!(
tracing::debug!(
request_id = %request_id,
model = %model,
endpoint = %endpoint,
Expand Down Expand Up @@ -1478,7 +1478,7 @@ impl Drop for InflightGuard {
);
}
Status::Success => {
tracing::info!(
tracing::debug!(
request_id = %self.request_id,
model = %self.model,
endpoint = %self.endpoint,
Expand Down
4 changes: 2 additions & 2 deletions lib/runtime/src/pipeline/network/ingress/push_handler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ impl Drop for RequestMetricsGuard {
self.request_duration
.observe(self.start_time.elapsed().as_secs_f64());
if let Some(request_id) = &self.request_id {
tracing::info!(request_id = %request_id, "request completed");
tracing::debug!(request_id = %request_id, "request completed");
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
}
}
}
Expand Down Expand Up @@ -596,7 +596,7 @@ where
m.inflight_requests.inc();
m.request_bytes.inc_by(payload.len() as u64);
if let Some(rid) = &request_id {
tracing::info!(request_id = %rid, "request received");
tracing::debug!(request_id = %rid, "request received");
}
RequestMetricsGuard {
inflight_requests: m.inflight_requests.clone(),
Expand Down
167 changes: 127 additions & 40 deletions tests/frontend/test_request_tracing_logs.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,21 @@
Verifies that JSONL logs contain consistent structured fields for all request
lifecycle events: "request received", "http response sent", "request completed".

The routine per-request lifecycle events ("request received", and "request
completed" on the success path) are emitted at DEBUG so they do not dominate a
default-level production log; the fixtures below therefore run with
`DYN_LOG=debug` to observe them. Abnormal completions stay at ERROR and the
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
Axum access log ("http response sent") stays at INFO, both of which remain
visible at the default level — see `test_agg_lifecycle_absent_at_info_level`.

Tests cover: unary success, streaming success, 404 error, 400 invalid UUID,
cancellation, frontend-worker trace_id correlation, aggregated deployment,
and disaggregated (prefill+decode) deployment.
"""

from __future__ import annotations

import contextlib
import json
import logging
import os
Expand Down Expand Up @@ -117,17 +125,25 @@ def get_request_logs(process, request_id: str) -> List[Dict[str, Any]]:


def assert_lifecycle_logs(req_logs, expected_status="success"):
"""Assert received/completed/http_sent exist and return them."""
"""Assert received/completed/http_sent exist and return them.

Requires a DEBUG-level log stream: 'request received' and 'request
completed' are emitted at DEBUG, so callers must use a fixture built on
JSONL_ENV (which sets DYN_LOG=debug), not JSONL_ENV_INFO.
"""
received = [e for e in req_logs if e.get("message") == "request received"]
completed = [e for e in req_logs if e.get("message") == "request completed"]
http_sent = [e for e in req_logs if e.get("message") == "http response sent"]
msgs = [e.get("message") for e in req_logs]
# A zero count here is far more often a log-level problem than a missing
# event, so say so rather than leaving a bare count mismatch.
lifecycle_hint = " (DEBUG-level event: does the fixture env set DYN_LOG=debug?)"
assert (
len(received) == 1
), f"Expected 1 'request received', got {len(received)}: {msgs}"
), f"Expected 1 'request received', got {len(received)}: {msgs}{lifecycle_hint}"
assert (
len(completed) == 1
), f"Expected 1 'request completed', got {len(completed)}: {msgs}"
), f"Expected 1 'request completed', got {len(completed)}: {msgs}{lifecycle_hint}"
assert (
len(http_sent) == 1
), f"Expected 1 'http response sent', got {len(http_sent)}: {msgs}"
Expand Down Expand Up @@ -165,35 +181,53 @@ def assert_error_completion(req_logs):
return completed


JSONL_ENV = {"DYN_LOGGING_JSONL": "1", "DYN_LOG": "info"}
# DYN_LOG=debug here is load-bearing, not incidental. Routine per-request
# lifecycle events ('request received' / 'request completed') are emitted at
# DEBUG, so every fixture using this env feeds tests whose assertions in
# assert_lifecycle_logs() above depend on the debug stream. Resetting this to
# "info" makes those assertions fail with a count mismatch, not a level error.
# Use JSONL_ENV_INFO below for tests that need the default operational level.
#
# The debug stream costs a fixed startup/registration delta here, not a
# per-token one: no debug site sits on the token loop, so a 50-token stream
# logs one line more than a 5-token unary response (measured: frontend 163 vs.
# 164 lines). read_log_file()/parse_jsonl_logs() therefore stay cheap even for
# the 2000-token cancellation and crash tests below.
JSONL_ENV = {"DYN_LOGGING_JSONL": "1", "DYN_LOG": "debug"}
Comment thread
MatejKosec marked this conversation as resolved.
Resolved

# The default operational level, used to pin that the routine lifecycle events
# stay out of a production log while the error path and access log remain.
JSONL_ENV_INFO = {"DYN_LOGGING_JSONL": "1", "DYN_LOG": "info"}


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(scope="function")
def tracing_services(
request,
runtime_services_dynamic_ports,
dynamo_dynamic_ports,
predownload_tokenizers,
):
"""Aggregated frontend + mocker with JSONL logging."""
ports = dynamo_dynamic_ports
@contextlib.contextmanager
def _agg_services(request, ports, env, speedup_ratio=None):
"""Start one aggregated frontend + mocker pair and yield the test handle.

The single implementation of the aggregated startup. The fixtures below
differ only in the JSONL env dict they pass (debug vs. the default info
level) and, for the cancellation/crash fixture, the mocker speedup ratio,
so they configure this helper rather than restating the startup.
"""
worker_kwargs = {} if speedup_ratio is None else {"speedup_ratio": speedup_ratio}
with DynamoFrontendProcess(
request,
frontend_port=ports.frontend_port,
terminate_all_matching_process_names=False,
extra_env=JSONL_ENV,
extra_env=env,
) as frontend:
with MockerWorkerProcess(
request,
model=TEST_MODEL,
frontend_port=ports.frontend_port,
system_port=ports.system_ports[0],
extra_env=JSONL_ENV,
extra_env=env,
**worker_kwargs,
) as worker:
wait_for_http_completions_ready(
frontend_port=ports.frontend_port, model=TEST_MODEL
Expand All @@ -205,6 +239,30 @@ def tracing_services(
}


@pytest.fixture(scope="function")
def tracing_services(
request,
runtime_services_dynamic_ports,
dynamo_dynamic_ports,
predownload_tokenizers,
):
"""Aggregated frontend + mocker with JSONL logging at DYN_LOG=debug."""
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
with _agg_services(request, dynamo_dynamic_ports, JSONL_ENV) as services:
yield services


@pytest.fixture(scope="function")
def tracing_services_info_level(
request,
runtime_services_dynamic_ports,
dynamo_dynamic_ports,
predownload_tokenizers,
):
"""Aggregated frontend + mocker at the default DYN_LOG=info level."""
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
with _agg_services(request, dynamo_dynamic_ports, JSONL_ENV_INFO) as services:
yield services


@pytest.fixture(scope="function")
def tracing_services_slow(
request,
Expand All @@ -213,29 +271,10 @@ def tracing_services_slow(
predownload_tokenizers,
):
"""Aggregated frontend + slow mocker for cancellation/crash testing."""
ports = dynamo_dynamic_ports
with DynamoFrontendProcess(
request,
frontend_port=ports.frontend_port,
terminate_all_matching_process_names=False,
extra_env=JSONL_ENV,
) as frontend:
with MockerWorkerProcess(
request,
model=TEST_MODEL,
frontend_port=ports.frontend_port,
system_port=ports.system_ports[0],
speedup_ratio=0.1,
extra_env=JSONL_ENV,
) as worker:
wait_for_http_completions_ready(
frontend_port=ports.frontend_port, model=TEST_MODEL
)
yield {
"frontend_port": ports.frontend_port,
"frontend": frontend,
"worker": worker,
}
with _agg_services(
request, dynamo_dynamic_ports, JSONL_ENV, speedup_ratio=0.1
) as services:
yield services


@pytest.fixture(scope="function")
Expand Down Expand Up @@ -303,12 +342,17 @@ def test_agg_unary_success(tracing_services) -> None:
req_logs = get_request_logs(tracing_services["frontend"], rid)
received, completed, http_sent = assert_lifecycle_logs(req_logs)

assert received[0]["level"] == "INFO"
# Routine lifecycle events are DEBUG; correlation fields survive the demotion.
assert received[0]["level"] == "DEBUG"
assert completed[0]["level"] == "DEBUG"
assert received[0].get("x_request_id") == rid
assert "request_id" in received[0]
assert received[0].get("request_id"), "request_id must survive the DEBUG demotion"
assert completed[0].get("request_id"), "request_id must survive the DEBUG demotion"
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
assert "model" in received[0]
assert "endpoint" in received[0]
assert "elapsed_ms" in completed[0]
# The Axum access log is not part of the demotion and stays at INFO.
assert http_sent[0]["level"] == "INFO"
assert http_sent[0].get("status") == "200"

# Token counts on inference span
Expand All @@ -324,10 +368,16 @@ def test_agg_unary_success(tracing_services) -> None:
wk_completed = [e for e in wk_logs if e.get("message") == "request completed"]
assert len(wk_received) == 1, "Worker should log 1 'request received'"
assert len(wk_completed) == 1, "Worker should log 1 'request completed'"
# Worker-side (request-plane ingress) lifecycle events are DEBUG too.
assert wk_received[0]["level"] == "DEBUG"
assert wk_completed[0]["level"] == "DEBUG"
assert wk_received[0].get("x_request_id") == rid, "Worker should have x_request_id"
assert (
wk_received[0].get("request_id") == server_rid
), "Worker request_id should match frontend"
assert wk_completed[0].get(
"request_id"
), "Worker request_id must survive the DEBUG demotion"


def test_agg_streaming_success(tracing_services) -> None:
Expand Down Expand Up @@ -375,6 +425,43 @@ def test_agg_404_error(tracing_services) -> None:
assert http_sent[0].get("status") == "404"


def test_agg_lifecycle_absent_at_info_level(tracing_services_info_level) -> None:
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
"""At the default DYN_LOG=info, routine lifecycle events do not reach the log.

This is the user-visible outcome: an operator tailing a busy worker at the
default level no longer sees two lines per request. The access log is still
asserted present so a broken logging pipeline cannot make this pass vacuously.
"""
port = tracing_services_info_level["frontend_port"]
rid = str(uuid.uuid4())

resp = _send_chat_completions(port, request_id=rid)
assert resp.status_code == 200
time.sleep(1)
Comment thread
glamr-agent marked this conversation as resolved.
Resolved

for name in ("frontend", "worker"):
req_logs = get_request_logs(tracing_services_info_level[name], rid)
Comment thread
glamr-agent marked this conversation as resolved.
Resolved
lifecycle = [
e
for e in req_logs
if e.get("message") in ("request received", "request completed")
]
assert not lifecycle, (
f"{name} emitted routine lifecycle events at DYN_LOG=info: "
f"{[(e.get('message'), e.get('level')) for e in lifecycle]}"
)

# Non-vacuity: the request really did happen and INFO logging really is on,
# so the absence above is the level filter and not a dead log pipeline.
fe_logs = get_request_logs(tracing_services_info_level["frontend"], rid)
http_sent = [e for e in fe_logs if e.get("message") == "http response sent"]
assert len(http_sent) == 1, (
"Expected the INFO-level access log to still be present, got: "
f"{[e.get('message') for e in fe_logs]}"
)
assert http_sent[0].get("status") == "200"


def test_agg_invalid_uuid_warn(tracing_services) -> None:
"""Invalid x-dynamo-request-id: WARN logged, request proceeds with generated ID."""
port = tracing_services["frontend_port"]
Expand Down
Loading