Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions fern/versions/latest/pages/observability/configuration.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,22 @@ Every span and metric carries these process-lifetime attributes:
| `nemo.gym.server.type` | `responses_api_models` |
| `nemo.gym.version` | `0.7.0` |
| `nemo.run.id` | `4f2a91c0be31` |
| `service.instance.id` | `4f2a91c0be31-policy_model-31844-9c1e02af` |

`nemo.run.id` and `service.instance.id` work as a pair, and they are deliberately
opposites. The run id is shared by every process of a run, so you can group the whole
fleet. The instance id identifies one process within it, so you can separate them again —
it is built from the server's config name, the process ID, and a random suffix, prefixed
by the run id when the run was configured with one. The suffix is what makes it unique: a
process ID repeats across hosts and across container PID namespaces, so the readable part
alone can collide. Use `nemo.run.id` to scope a query to a run and `service.instance.id`
to tell two Uvicorn workers of the same server apart.

A `service.instance.id` set in `OTEL_RESOURCE_ATTRIBUTES` is replaced, not kept. NeMo Gym
spawns its servers into one shared environment, so an inherited value would name every one
of them identically — the collision the instance id exists to break. Code that embeds NeMo
Gym can still pass its own id programmatically as a resource attribute, which takes
precedence.

## Verify That It Works

Expand Down
31 changes: 31 additions & 0 deletions nemo_gym/telemetry/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,27 @@ def _build_resource_attributes(server_name: Optional[str], server_type: Optional
return attrs


def _service_instance_id(run_id: str, server_name: Optional[str]) -> str:
"""A globally unique name for this process, behind a readable prefix.

The server name makes it legible and the PID separates one server's Uvicorn workers,
but neither is unique on its own: a PID repeats across hosts and across container PID
namespaces. The random suffix is what actually satisfies the semantic convention.
Absent parts are omitted rather than left as empty segments.

The run id is included only when this process was configured with one. A standalone
server gets ``<name>-<pid>-<nonce>``; nemo-lens still mints a ``nemo.run.id`` for it
afterwards, inside ``setup_telemetry``, which is too late to appear here.

Called once per process: ``init_telemetry`` sets ``_INITIALISED`` under ``_INIT_LOCK``
and returns early afterwards, so the suffix is stable for the process's lifetime.
"""
# Eight hex digits, like the truncated run id above: this only has to separate the
# processes of one run, not be unique across every run ever submitted.
nonce = uuid4().hex[:8]
return "-".join(part for part in (run_id, server_name, str(os.getpid()), nonce) if part)


def _sdk_meter_provider_installed() -> bool:
"""Whether a real OpenTelemetry SDK meter provider, not a no-op or proxy, is registered."""
try:
Expand Down Expand Up @@ -557,6 +578,16 @@ def init_telemetry(
# reports no rank, since the collector could not filter it by rank.
attrs[NV_DL_RANK] = 0
attrs[NV_DL_WORLD_SIZE] = 1
# Named here because nemo-lens derives it as f"{run_id}-rank{rank}" otherwise.
# With the rank above pinned to 0 and run_id shared fleet-wide by design, that
# derivation hands every server in the job the same instance id.
#
# nemo-lens ranks caller attributes above environment ones, so this replaces a
# ``service.instance.id`` set in ``OTEL_RESOURCE_ATTRIBUTES``. That is what a
# fleet needs: Gym's servers are spawned into one shared environment, so a single
# inherited value would name every one of them identically. A caller that wants
# its own id passes ``resource_attributes`` below, which still wins.
attrs["service.instance.id"] = _service_instance_id(config.run_id, server_name)
if resource_attributes:
attrs.update(resource_attributes)

Expand Down
23 changes: 23 additions & 0 deletions tests/unit_tests/orchestration/test_slurm_script.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import ast
import base64
import contextlib
import os
Expand Down Expand Up @@ -494,6 +495,28 @@ def test_build_vllm_ray_serve_command_embeds_actual_gateway_source(vllm_service)
assert decoded == _RAY_SERVE_GATEWAY_SOURCE_PATH.read_text()


def test_embedded_gateway_imports_nothing_from_nemo_gym():
"""The embedded gateway has to run where `nemo_gym` was never installed.

Only this one file is written into the vLLM container, with aiohttp and Ray installed
beside it and no `driver.gym_install`. A `nemo_gym` import therefore fails at import
time, before argv is parsed, and the model service never becomes available. Parsed
rather than grepped so an import inside a function counts too: deferring one only
moves the failure to the first call.
"""
tree = ast.parse(_RAY_SERVE_GATEWAY_SOURCE_PATH.read_text())

imported = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
imported.update(alias.name for alias in node.names)
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
imported.add(node.module)

offenders = sorted(name for name in imported if name == "nemo_gym" or name.startswith("nemo_gym."))
assert not offenders, f"ray_serve_gateway.py must stay self-contained, but imports {offenders}"


def test_build_vllm_ray_serve_command_single_node_ensures_ray_installed(vllm_service):
# Regression test: the single-node path invokes python3 directly, so ray isn't guaranteed
# importable there unlike the multi-node path (gated via `ray symmetric-run`/`ray start`).
Expand Down
93 changes: 93 additions & 0 deletions tests/unit_tests/telemetry/test_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,99 @@ def capture(config, **kwargs):
assert captured[NV_DL_WORLD_SIZE] == 1


def test_each_server_names_its_own_instance(enabled_console_env, monkeypatch):
"""The companion to the rank above: nemo-lens derives `service.instance.id` as
f"{run_id}-rank{rank}" when a rank is present and no identity was supplied. Rank is
pinned to 0 for every server and run_id is deliberately shared fleet-wide, so
leaving the derivation to lens gives the whole job one instance id.

The readable prefix is asserted but the suffix only has to be hex: it is a random
nonce, so pinning the whole string would pin a value the next run will not produce.
"""
monkeypatch.setenv("NEMO_GYM_OTEL_RUN_ID", "job-1416252")
captured = {}

def capture(config, **kwargs):
captured.update(kwargs.get("resource_attributes") or {})
raise RuntimeError("stop here")

monkeypatch.setattr("nemo.lens.setup_telemetry", capture)
init_telemetry(server_name="weather")

instance_id = captured["service.instance.id"]
prefix = f"job-1416252-weather-{os.getpid()}-"
assert instance_id.startswith(prefix), instance_id
nonce = instance_id.removeprefix(prefix)
assert nonce and all(c in "0123456789abcdef" for c in nonce), instance_id


def test_two_processes_sharing_a_run_a_name_and_a_pid_still_differ(monkeypatch):
"""A PID is only unique within its host and its container PID namespace.

Two servers of one run can therefore agree on every readable part of the identity,
which is the collision the nonce exists to break. Driven through the helper rather
than two `init_telemetry` lifecycles because the helper is where the uniqueness
comes from, and the one-shot init guard makes a second lifecycle awkward to stage.
"""
monkeypatch.setattr(os, "getpid", lambda: 42)

first = telemetry_setup._service_instance_id("job-1", "weather")
second = telemetry_setup._service_instance_id("job-1", "weather")

assert first != second, "two processes would report the same instance id"
assert first.startswith("job-1-weather-42-") and second.startswith("job-1-weather-42-")


def test_an_instance_id_stays_nonempty_without_a_run_or_a_name(monkeypatch):
"""Absent parts are dropped, so a standalone server must not get a leading `--`."""
monkeypatch.setattr(os, "getpid", lambda: 42)

assert telemetry_setup._service_instance_id("", None).startswith("42-")


def test_an_instance_id_from_the_caller_wins(enabled_console_env, monkeypatch):
"""Only the `resource_attributes` argument keeps a caller's own identity.

nemo-lens ranks caller attributes above environment ones and Gym now always passes
an id, so a `service.instance.id` in `OTEL_RESOURCE_ATTRIBUTES` is replaced rather
than kept -- see `test_an_environment_instance_id_is_replaced` below.
"""
captured = {}

def capture(config, **kwargs):
captured.update(kwargs.get("resource_attributes") or {})
raise RuntimeError("stop here")

monkeypatch.setattr("nemo.lens.setup_telemetry", capture)
init_telemetry(
server_name="weather",
resource_attributes={"service.instance.id": "chosen-by-the-caller"},
)

assert captured["service.instance.id"] == "chosen-by-the-caller"


def test_an_environment_instance_id_is_replaced(enabled_console_env, monkeypatch):
"""Gym's servers are spawned into one shared environment.

An inherited `OTEL_RESOURCE_ATTRIBUTES` id would therefore name every one of them
identically, which is the collision this whole helper exists to break, so Gym passes
its own and lets lens rank it above the environment.
"""
monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", "service.instance.id=chosen-by-launcher")
monkeypatch.setenv("NEMO_GYM_OTEL_RUN_ID", "job-1")
captured = {}

def capture(config, **kwargs):
captured.update(kwargs.get("resource_attributes") or {})
raise RuntimeError("stop here")

monkeypatch.setattr("nemo.lens.setup_telemetry", capture)
init_telemetry(server_name="weather")

assert captured["service.instance.id"].startswith(f"job-1-weather-{os.getpid()}-")
Comment thread
rrs45 marked this conversation as resolved.
Resolved


# --------------------------------------------------------------------------- #
# Another library already initialised nemo-lens in this process (NeMo-RL)
# --------------------------------------------------------------------------- #
Expand Down
Loading