From a806105619d546f85af71ebbbdde35310dd2472b Mon Sep 17 00:00:00 2001 From: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> Date: Mon, 2 Nov 2026 10:55:12 +0000 Subject: [PATCH 1/5] fix(engine): honor ATOM_TORCH_PROFILER_DIR Omit an unspecified profiler directory from EngineArgs kwargs so Config can resolve its existing environment default. Explicit CLI values retain precedence, and direct Config callers keep their current semantics. Read an empty ATOM_TORCH_PROFILER_DIR as unset in envs, so `VAR=` clears it instead of aborting startup on os.makedirs(""). Give the online-quant dump its own ATOM_ONLINE_QUANT_INFO_DIR, falling back to the profiler directory, and stop the ATOMesh launcher exporting a profiler directory on every launch now that the engine honors it. Cover the hand-off through EngineArgs.create_engine() -> LLMEngine -> Config with the environment unset, empty and set, and with and without an explicit --torch-profiler-dir. Signed-off-by: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> --- .github/scripts/atomesh/pd_server_atom.sh | 19 +++-- atom/model_engine/arg_utils.py | 3 + atom/model_loader/loader.py | 4 +- atom/utils/envs.py | 15 +++- docs/configuration_guide.md | 4 +- docs/environment_variables.md | 3 +- docs/online_quantization_guide.md | 5 +- tests/test_arg_utils_profiler.py | 97 +++++++++++++++++++++++ tests/test_envs.py | 32 ++++++++ 9 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 tests/test_arg_utils_profiler.py diff --git a/.github/scripts/atomesh/pd_server_atom.sh b/.github/scripts/atomesh/pd_server_atom.sh index 5e26cae3e0..871c31255b 100644 --- a/.github/scripts/atomesh/pd_server_atom.sh +++ b/.github/scripts/atomesh/pd_server_atom.sh @@ -347,16 +347,19 @@ ATOMESH_TORCH_PROFILE_ROLES="${ATOMESH_TORCH_PROFILE_ROLES:-prefill,decode}" PREFILL_KV_TRANSFER_CONFIG="${PREFILL_KV_TRANSFER_CONFIG:-}" DECODE_KV_TRANSFER_CONFIG="${DECODE_KV_TRANSFER_CONFIG:-}" -default_profiler_dir="${RUN_DIR}/online_quant/rank-${NODE_RANK}" +# Rank 0's online_quant_info_*.json dump for this run. It has its own variable +# so that collecting it does not also hand the engine a profiler directory on +# every launch; the profiler window below supplies one per worker instead. +online_quant_dir="${RUN_DIR}/online_quant/rank-${NODE_RANK}" if [[ "${ATOMESH_EXECUTION_PHASE}" != "combined" ]]; then - default_profiler_dir="${RUN_DIR}/online_quant/${ATOMESH_EXECUTION_PHASE}/rank-${NODE_RANK}" + online_quant_dir="${RUN_DIR}/online_quant/${ATOMESH_EXECUTION_PHASE}/rank-${NODE_RANK}" fi -export ATOM_TORCH_PROFILER_DIR="${ATOM_TORCH_PROFILER_DIR:-${default_profiler_dir}}" +export ATOM_ONLINE_QUANT_INFO_DIR="${ATOM_ONLINE_QUANT_INFO_DIR:-${online_quant_dir}}" RUNTIME_LOG_DIR="${RUN_DIR}/logs" if [[ "${ATOMESH_EXECUTION_PHASE}" != "combined" ]]; then RUNTIME_LOG_DIR="${RUNTIME_LOG_DIR}/${ATOMESH_EXECUTION_PHASE}" fi -mkdir -p "${RUNTIME_LOG_DIR}" "${RUN_DIR}"/{benchmark_results,eval_results} "${ATOM_TORCH_PROFILER_DIR}" +mkdir -p "${RUNTIME_LOG_DIR}" "${RUN_DIR}"/{benchmark_results,eval_results} "${ATOM_ONLINE_QUANT_INFO_DIR}" role_tp="${PREFILL_TP_SIZE}" if [[ "${AGGREGATED}" == "1" ]]; then @@ -1942,10 +1945,10 @@ write_metadata() { EOF } -# ATOM_TORCH_PROFILER_DIR above does not reach the engine: EngineArgs always -# passes its own torch_profiler_dir (None by default), which overrides the env -# default in Config, so the profiler is off unless the flag is on the command. -# One directory per worker keeps P/D ranks that share a node apart. +# Put a profiler directory on the command line only while the window is on: +# this script exports no ATOM_TORCH_PROFILER_DIR, so a launch that does not +# profile carries none. One directory per worker keeps P/D ranks that share a +# node apart; an explicit flag also outranks the environment default in Config. build_torch_profiler_args() { local log_name="$1" local -n out_args="$2" diff --git a/atom/model_engine/arg_utils.py b/atom/model_engine/arg_utils.py index a6fb3343fd..5d112d63fe 100644 --- a/atom/model_engine/arg_utils.py +++ b/atom/model_engine/arg_utils.py @@ -902,6 +902,9 @@ def _get_engine_kwargs(self) -> dict: kwargs = { f.name: getattr(self, f.name) for f in fields(self) if f.name != "model" } + # Let Config resolve ATOM_TORCH_PROFILER_DIR when no path was supplied. + if self.torch_profiler_dir is None: + kwargs.pop("torch_profiler_dir") # Handle special transformations kwargs["kv_cache_block_size"] = kwargs.pop("block_size") diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index 72f88af166..0afd3aa4c3 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -177,7 +177,9 @@ def _save_online_quant_info( """Save online quantization info to a JSON file (rank 0 only).""" if get_tp_group().rank_in_group != 0: return - output_dir = envs.ATOM_TORCH_PROFILER_DIR or os.getcwd() + # Its own knob (falling back to ATOM_TORCH_PROFILER_DIR), then the + # historical default. + output_dir = envs.ATOM_ONLINE_QUANT_INFO_DIR or os.getcwd() os.makedirs(output_dir, exist_ok=True) timestamp = time.strftime("%Y%m%d_%H%M%S") timestamp_ns = time.time_ns() % 1_000_000_000 diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 2984e728a7..e33e8ef950 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -822,7 +822,10 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: "ATOM_ENABLE_METRICS_DEVICE_TIMER": lambda: ( os.getenv("ATOM_ENABLE_METRICS_DEVICE_TIMER", "0") == "1" ), - "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR", None), + # Unset reads as None, and so does empty: `ATOM_TORCH_PROFILER_DIR=` is + # how a launcher clears the variable inline, and Config would otherwise + # abort startup on os.makedirs("") for the empty path. + "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR") or None, # Move the startup heap (model, compiled graph, tokenizer, KV block pool) # into CPython's permanent generation once warmup is done, so collections # stop scanning it. On by default; set 0 to keep the old behaviour. @@ -939,6 +942,16 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: "ATOM_LOADER_STRICT_COVERAGE": lambda: ( os.getenv("ATOM_LOADER_STRICT_COVERAGE", "true").lower() == "true" ), + # Directory rank 0 writes the online_quant_info_*.json dump to. Its own + # knob so that a launcher collecting the dump no longer has to hand the + # engine a profiler directory to get it: unset (or empty) falls back to + # ATOM_TORCH_PROFILER_DIR, the dump's only home before this one existed. + # The caller falls back to the working directory from there. + "ATOM_ONLINE_QUANT_INFO_DIR": lambda: ( + os.getenv("ATOM_ONLINE_QUANT_INFO_DIR") + or os.getenv("ATOM_TORCH_PROFILER_DIR") + or None + ), # Quantize eligible modules as they load to reduce peak memory. Streaming # quantizes local TP shards, so results may differ slightly from offline. "ATOM_ONLINE_QUANT_STREAMING": lambda: ( diff --git a/docs/configuration_guide.md b/docs/configuration_guide.md index e5d0d08edd..dd9c23fda7 100644 --- a/docs/configuration_guide.md +++ b/docs/configuration_guide.md @@ -58,7 +58,7 @@ Defined in `atom/config.py`. The root dataclass that the engine consumes. | `replay_history_tokens` | `int` | `0` | Set by the model, not a flag: the tokens before a forward that starts below where its prefill resumed which the backend reads (`Replay.history_ids`). DeepSeek-V4.1: its Engram n-gram lookback, `engram_max_ngram_size - 1` | | `state_replay_tokens` | `int` | `0` | Set by the model, not a flag: how far below a prefix hit with no state checkpoint behind it a prefill starts to rebuild that state exactly (`BlockManager.replay_start`); 0 takes no such hit. DeepSeek-V4 under `ATOM_DSV4_STATE_REPLAY`: the window receptive field `W + (L-1)(W-1)`, 7,748 on V4-Pro and 7,875 with MTP or DSpark | | `port` | `int` | `8006` | Engine internal communication port | -| `torch_profiler_dir` | `str \| None` | `os.getenv("ATOM_TORCH_PROFILER_DIR", None)` | Directory for saving PyTorch profiler traces; creates the directory if it does not exist | +| `torch_profiler_dir` | `str \| None` | `envs.ATOM_TORCH_PROFILER_DIR` (unset or empty → `None`) | Directory for saving PyTorch profiler traces; creates the directory if it does not exist. `--torch-profiler-dir` takes precedence over the environment default | | `compilation_config` | `CompilationConfig` | `CompilationConfig()` | Compilation and CUDA graph settings (see Section 2) | | `quant_config` | `QuantizationConfig` | *(auto-detected)* | Quantization settings; auto-detected from HuggingFace config during `__post_init__` via `QuantizationConfig(hf_config)` (see Section 3) | | `asyncio_mode` | `bool` | `False` | Enable asyncio-based engine loop | @@ -259,7 +259,7 @@ Notes: - An empty JSON object (`--online_quant_config '{}'`) is treated the same as not passing the flag and does not enable online quantization. - Online quantization currently applies when the source model is unquantized or uses supported FP8 block quantization (`QuantType.per_1x128`). The loader dequantizes FP8 block weights before applying the requested target format. - Tensor-parallel weights are gathered before quantization only when local quantization would produce different scales than quantizing the full unpartitioned weight. -- Rank 0 writes an `online_quant_info_*.json` summary with elapsed time and per-layer target formats. The file is written under `ATOM_TORCH_PROFILER_DIR` when set, otherwise the current working directory. +- Rank 0 writes an `online_quant_info_*.json` summary with elapsed time and per-layer target formats. The file is written under `ATOM_ONLINE_QUANT_INFO_DIR` when set, otherwise under `ATOM_TORCH_PROFILER_DIR`, otherwise in the current working directory. ## Parallel configuration (`ParallelConfig`) diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 7be57bbe41..494534a5b4 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -527,7 +527,8 @@ server's address), `topic`, and `block_size`, the tokens per hashed block. It is |----------|------|---------|-------------| | **ATOM_METRICS_UPDATE_INTERVAL_S** | float | 1.0 | Shared interval in seconds for ordinary/DP/PP engine metrics pushes and API snapshot refresh. Must be finite and positive; read when each loop starts, so set it before starting every service process. Prometheus scraping is configured independently. Does not cache rendered `/metrics` responses or change when histogram observations are recorded. | | **ATOM_ENABLE_METRICS_DEVICE_TIMER** | bool | 0 (false) | Set to `1` before starting the service to collect GPU forward duration and cumulative request prefill GPU time. Uses CUDA/HIP events, a reusable pool capped at 256 pending pairs, and FIFO polling that stops at the first incomplete event. Adds event recording and query overhead; disabled services emit no GPU timing samples. Agentic dashboard CI explicitly enables it. | -| **ATOM_TORCH_PROFILER_DIR** | str | — | When set, enables PyTorch profiler and writes traces to this directory. Create subdirectories per rank (e.g., `rank_0`, `dp0_tp0`). | +| **ATOM_TORCH_PROFILER_DIR** | str | — | When set, enables PyTorch profiler and writes traces to this directory. Create subdirectories per rank (e.g., `rank_0`, `dp0_tp0`). An empty value (`ATOM_TORCH_PROFILER_DIR=`) reads as unset, so a launcher can clear it inline. `--torch-profiler-dir` takes precedence over it. | +| **ATOM_ONLINE_QUANT_INFO_DIR** | str | — | Directory where rank 0 writes the `online_quant_info_*.json` dump after online quantization. Unset or empty falls back to `ATOM_TORCH_PROFILER_DIR`, then to the working directory. Unlike `ATOM_TORCH_PROFILER_DIR`, setting it does not also give the engine a profiler directory. | | **ATOM_PROFILER_MORE** | bool | 0 (false) | When `ATOM_TORCH_PROFILER_DIR` is set and this is `1`, enables detailed profiling: `record_shapes`, `with_stack`, and `profile_memory`. Applies to both the run-phase profiler and the CUDA-graph capture profiler. Each option can be overridden individually by the per-option flags below. | | **ATOM_PROFILER_RECORD_SHAPES** | bool | `ATOM_PROFILER_MORE` | Set to `1`/`0` to enable/disable `record_shapes`. Takes precedence over `ATOM_PROFILER_MORE`; unset or empty falls back to it. | | **ATOM_PROFILER_WITH_STACK** | bool | `ATOM_PROFILER_MORE` | Set to `1`/`0` to enable/disable `with_stack` (Python stack capture). Takes precedence over `ATOM_PROFILER_MORE`; unset or empty falls back to it. | diff --git a/docs/online_quantization_guide.md b/docs/online_quantization_guide.md index 81ea4964f9..f5aa09711c 100644 --- a/docs/online_quantization_guide.md +++ b/docs/online_quantization_guide.md @@ -244,8 +244,9 @@ quantization activates only for a supported source format (see § 1). When online quantization runs, rank 0 writes `online_quant_info__.json` to: -1. `$ATOM_TORCH_PROFILER_DIR` if the env var is set, otherwise -2. the current working directory. +1. `$ATOM_ONLINE_QUANT_INFO_DIR` if the env var is set, otherwise +2. `$ATOM_TORCH_PROFILER_DIR` if that one is set, otherwise +3. the current working directory. A representative payload: diff --git a/tests/test_arg_utils_profiler.py b/tests/test_arg_utils_profiler.py new file mode 100644 index 0000000000..bac9de58a6 --- /dev/null +++ b/tests/test_arg_utils_profiler.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: MIT + +import argparse +import os +from types import SimpleNamespace + +import pytest +from transformers import PretrainedConfig + +import atom.config as config_module +import atom.model_engine.llm_engine as engine_module +from atom.model_engine.arg_utils import EngineArgs + +UNSET = "" + + +@pytest.fixture(autouse=True) +def _cpu_handoff(monkeypatch): + """Drive the real CLI -> EngineArgs -> LLMEngine -> Config hand-off on CPU. + + Only the parts that need a checkpoint, a HuggingFace reach or GPU workers + are replaced: Config's checkpoint and quantization lookups, and the + engine's tokenizer and CoreManager. Argument parsing, + ``_get_engine_kwargs``, ``LLMEngine.__init__`` and Config construction run + for real. + """ + monkeypatch.delenv("ATOM_TORCH_PROFILER_DIR", raising=False) + monkeypatch.delenv("ATOM_KV_OFFLOAD", raising=False) + monkeypatch.delenv("ATOM_KV_OFFLOAD_EXTRA_CONFIG", raising=False) + hf_config = PretrainedConfig( + architectures=["LlamaForCausalLM"], max_position_embeddings=8192 + ) + monkeypatch.setattr( + config_module, + "get_hf_config", + lambda _model, trust_remote_code=False: hf_config, + ) + monkeypatch.setattr(config_module, "get_generation_config", lambda _model: None) + monkeypatch.setattr( + config_module, + "QuantizationConfig", + lambda *_args, **_kwargs: SimpleNamespace(exclude_layers=[]), + ) + monkeypatch.setattr( + engine_module, + "load_tokenizer", + lambda *_args, **_kwargs: SimpleNamespace(bos_token_id=0, eos_token_id=1), + ) + monkeypatch.setattr( + engine_module, + "CoreManager", + lambda _config: SimpleNamespace(close=lambda: None), + ) + + +def _create_engine(argv): + """The hand-off every server and example uses: CLI -> EngineArgs -> LLMEngine.""" + parser = EngineArgs.add_cli_args(argparse.ArgumentParser()) + return EngineArgs.from_cli_args(parser.parse_args(argv)).create_engine() + + +@pytest.mark.parametrize( + "env_dir, cli_dir, expected", + [ + (UNSET, None, None), + (UNSET, "cli", "cli"), + ("env", None, "env"), + ("env", "cli", "cli"), + ("", None, None), + ("", "cli", "cli"), + ], +) +def test_torch_profiler_dir_handoff(monkeypatch, tmp_path, env_dir, cli_dir, expected): + if env_dir != UNSET: + monkeypatch.setenv( + "ATOM_TORCH_PROFILER_DIR", "" if env_dir == "" else str(tmp_path / env_dir) + ) + argv = ["--model", "test-model"] + if cli_dir is not None: + argv += ["--torch-profiler-dir", str(tmp_path / cli_dir)] + + engine = _create_engine(argv) + + expected_dir = None if expected is None else str(tmp_path / expected) + assert engine.config.torch_profiler_dir == expected_dir + # Config creates and validates the directory it was handed. + assert expected_dir is None or os.path.isdir(expected_dir) + engine.close() + + +def test_empty_cli_directory_is_not_treated_as_none(monkeypatch, tmp_path): + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", str(tmp_path / "env")) + # An explicitly empty CLI value still reaches Config's validation; it does + # not silently select the environment default. + parser = EngineArgs.add_cli_args(argparse.ArgumentParser()) + args = EngineArgs.from_cli_args(parser.parse_args(["--torch-profiler-dir", ""])) + assert args._get_engine_kwargs()["torch_profiler_dir"] == "" diff --git a/tests/test_envs.py b/tests/test_envs.py index aacd6db8ea..2eadb2cddb 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -29,6 +29,7 @@ "ATOM_LLAMA_ENABLE_AITER_TRITON_FUSED_SILU_MUL_QUANT", "ATOM_USE_MODEL_SENSITIVE_RMSNORM", "ATOM_TORCH_PROFILER_DIR", + "ATOM_ONLINE_QUANT_INFO_DIR", "ATOM_ENABLE_METRICS_DEVICE_TIMER", "ATOM_METRICS_UPDATE_INTERVAL_S", "ATOM_SHUTDOWN_TIMEOUT_S", @@ -110,6 +111,9 @@ def test_model_sensitive_rmsnorm_default_disabled(self): def test_torch_profiler_dir_default(self): assert _get_envs().ATOM_TORCH_PROFILER_DIR is None + def test_online_quant_info_dir_default(self): + assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR is None + def test_metrics_device_timer_default_disabled(self): assert _get_envs().ATOM_ENABLE_METRICS_DEVICE_TIMER is False @@ -190,6 +194,34 @@ def test_torch_profiler_dir_override(self, monkeypatch): monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof") assert _get_envs().ATOM_TORCH_PROFILER_DIR == "/tmp/prof" + def test_empty_torch_profiler_dir_reads_as_unset(self, monkeypatch): + # `VAR=` is how a launcher clears it inline. Handing "" to Config would + # abort startup on os.makedirs("") instead. + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "") + assert _get_envs().ATOM_TORCH_PROFILER_DIR is None + + def test_online_quant_info_dir_override(self, monkeypatch): + monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", "/tmp/oq") + assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR == "/tmp/oq" + + @pytest.mark.parametrize("value", [None, ""]) + def test_online_quant_info_dir_falls_back_to_profiler_dir(self, monkeypatch, value): + # The dump lived under ATOM_TORCH_PROFILER_DIR before it had a knob of + # its own; unset or empty keeps landing there. + if value is not None: + monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", value) + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof") + assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR == "/tmp/prof" + + def test_online_quant_info_dir_beats_profiler_dir(self, monkeypatch): + monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", "/tmp/oq") + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof") + assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR == "/tmp/oq" + + def test_online_quant_info_dir_ignores_empty_profiler_dir(self, monkeypatch): + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "") + assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR is None + def test_profiler_more_enabled(self, monkeypatch): monkeypatch.setenv("ATOM_PROFILER_MORE", "1") assert _get_envs().ATOM_PROFILER_MORE is True From 167fe0d73aa520bf3ad29238572cb7545f9e6fb9 Mon Sep 17 00:00:00 2001 From: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:55:41 +0000 Subject: [PATCH 2/5] fix: address review: profiling example and empty env dirs Honor ATOM_TORCH_PROFILER_DIR in the offline profiling example, which replaced an omitted flag with its own default before EngineArgs saw it. Resolve the way the engine does -- flag, then environment, then the example's ./profiler_traces -- and name the source that supplied the directory in the notice. Read an empty ATOM_ENGRAM_CACHE_DIR as its default cache path rather than "", which Path resolves to the working directory, and an empty ATOM_RTL_TRACE_DIR as None, like ATOM_TORCH_PROFILER_DIR. State the rule once in _optional_path_env / _path_env, which the profiler, online-quant and ATOM_V4_CG_INSTR_DIR knobs now use as well. Cover the example's resolution over seven cases, including the notice, and the two sibling variables with default, override and empty-value tests. Signed-off-by: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> --- atom/examples/profile_offline.py | 37 ++++++++++++++--- atom/utils/envs.py | 34 ++++++++++++---- docs/environment_variables.md | 2 +- docs/serving_benchmarking_guide.md | 5 ++- tests/test_envs.py | 31 ++++++++++++++ tests/test_profile_offline.py | 65 ++++++++++++++++++++++++++++++ 6 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 tests/test_profile_offline.py diff --git a/atom/examples/profile_offline.py b/atom/examples/profile_offline.py index 9ea6706946..3eb5fe8f7f 100644 --- a/atom/examples/profile_offline.py +++ b/atom/examples/profile_offline.py @@ -3,12 +3,16 @@ import argparse import random + from transformers import AutoTokenizer from atom import SamplingParams from atom.model_engine.arg_utils import EngineArgs +from atom.utils import envs from atom.utils.arg_parser import FlexibleArgumentParser +DEFAULT_PROFILER_DIR = "./profiler_traces" + parser = FlexibleArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="Offline profiling example for Atom LLM Engine", @@ -41,15 +45,36 @@ ) +def resolve_profiler_dir(cli_value: str | None) -> tuple[str, str | None]: + """Resolve the trace directory, and the notice to print when it was not given. + + The precedence is the one every other entry point uses: an explicit value, + then ``ATOM_TORCH_PROFILER_DIR`` (where empty reads as unset), then this + example's own ``./profiler_traces``. It ends with a directory rather than + ``None`` because the example always profiles; an empty + ``--torch-profiler-dir`` counts as not given, like an empty environment + value. + """ + if cli_value: + return cli_value, None + env_dir = envs.ATOM_TORCH_PROFILER_DIR + if env_dir is not None: + return env_dir, ( + "Warning: --torch-profiler-dir not specified, using " + f"ATOM_TORCH_PROFILER_DIR: {env_dir}" + ) + return DEFAULT_PROFILER_DIR, ( + "Warning: --torch-profiler-dir not specified and " + f"ATOM_TORCH_PROFILER_DIR unset, using default: {DEFAULT_PROFILER_DIR}" + ) + + def main(): args = parser.parse_args() - # Set default torch_profiler_dir to current directory if not provided - if args.torch_profiler_dir is None: - args.torch_profiler_dir = "./profiler_traces" - print( - "Warning: --torch-profiler-dir not specified, using default: ./profiler_traces" - ) + args.torch_profiler_dir, notice = resolve_profiler_dir(args.torch_profiler_dir) + if notice is not None: + print(notice) print("\nInitializing LLM engine...") engine_args = EngineArgs.from_cli_args(args) diff --git a/atom/utils/envs.py b/atom/utils/envs.py index e33e8ef950..1de99d3151 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -109,6 +109,20 @@ def _nonnegative_int_env(name: str, default: int) -> int: return value +def _optional_path_env(name: str) -> str | None: + """Unset or empty reads as None: ``VAR=`` clears a path inline.""" + return os.getenv(name) or None + + +def _path_env(name: str, default: str) -> str: + """Unset or empty reads as ``default``, never as the empty path. + + ``Path("")`` is the working directory, so falling through to it would + silently write wherever the process happens to run. + """ + return os.getenv(name) or default + + def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: """Unset or empty reads as ``default``; a set value must be a valid float.""" raw = os.getenv(name) @@ -825,7 +839,7 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # Unset reads as None, and so does empty: `ATOM_TORCH_PROFILER_DIR=` is # how a launcher clears the variable inline, and Config would otherwise # abort startup on os.makedirs("") for the empty path. - "ATOM_TORCH_PROFILER_DIR": lambda: os.getenv("ATOM_TORCH_PROFILER_DIR") or None, + "ATOM_TORCH_PROFILER_DIR": lambda: _optional_path_env("ATOM_TORCH_PROFILER_DIR"), # Move the startup heap (model, compiled graph, tokenizer, KV block pool) # into CPython's permanent generation once warmup is done, so collections # stop scanning it. On by default; set 0 to keep the old behaviour. @@ -880,8 +894,10 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: "ATOM_LOG_LEVEL": lambda: os.getenv("ATOM_LOG_LEVEL", "INFO"), # RTL (rocm-trace-lite) GPU kernel tracing — set to output directory to enable. # When set, the server launch is wrapped with `rtl trace` to collect per-kernel - # GPU timestamps for both prefill and decode phases. - "ATOM_RTL_TRACE_DIR": lambda: os.getenv("ATOM_RTL_TRACE_DIR", None), + # GPU timestamps for both prefill and decode phases. Empty reads as unset, + # like the other directory knobs, so `VAR=` disables it rather than naming + # "" as the output directory. + "ATOM_RTL_TRACE_DIR": lambda: _optional_path_env("ATOM_RTL_TRACE_DIR"), # --- Model Loading --- "ATOM_DISABLE_MMAP": lambda: ( os.getenv("ATOM_DISABLE_MMAP", "false").lower() == "true" @@ -948,9 +964,8 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # ATOM_TORCH_PROFILER_DIR, the dump's only home before this one existed. # The caller falls back to the working directory from there. "ATOM_ONLINE_QUANT_INFO_DIR": lambda: ( - os.getenv("ATOM_ONLINE_QUANT_INFO_DIR") - or os.getenv("ATOM_TORCH_PROFILER_DIR") - or None + _optional_path_env("ATOM_ONLINE_QUANT_INFO_DIR") + or _optional_path_env("ATOM_TORCH_PROFILER_DIR") ), # Quantize eligible modules as they load to reduce peak memory. Streaming # quantizes local TP shards, so results may differ slightly from offline. @@ -1028,7 +1043,7 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # vLLM plugin, DeepSeek-V4: directory for per-process cudagraph dispatch # counters (modes, overridden steps, guard trips), one JSON file per # process, rewritten every step. Unset: off. - "ATOM_V4_CG_INSTR_DIR": lambda: os.getenv("ATOM_V4_CG_INSTR_DIR") or None, + "ATOM_V4_CG_INSTR_DIR": lambda: _optional_path_env("ATOM_V4_CG_INSTR_DIR"), "ATOM_USE_CUSTOM_ALL_GATHER": lambda: ( os.getenv("ATOM_USE_CUSTOM_ALL_GATHER", "1").lower() == "1" ), @@ -1362,7 +1377,10 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # Where the compressed-vocab table is cached between runs. The table is # reproducible from the tokenizer, so this only trades startup time for # disk; point it at shared storage to let several servers build it once. - "ATOM_ENGRAM_CACHE_DIR": lambda: os.getenv( + # Empty reads as the default, like the other directory knobs: Path("") is + # the working directory, so `ATOM_ENGRAM_CACHE_DIR=` would otherwise drop + # the cache beside whatever directory the server was launched from. + "ATOM_ENGRAM_CACHE_DIR": lambda: _path_env( "ATOM_ENGRAM_CACHE_DIR", str(Path.home() / ".cache" / "atom" / "engram") ), # Overlap hash/UVA lookup and TP reassembly with early layers, using private diff --git a/docs/environment_variables.md b/docs/environment_variables.md index 494534a5b4..7a494a323c 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -346,7 +346,7 @@ some layers and the device path for others, which is the confusing state. | **ATOM_ENGRAM_UVA** | bool | 1 (true) | Page-lock this rank's shard of the hash tables (copied into page-locked memory by default, see `ATOM_ENGRAM_UVA_COPY`) and let a device kernel read the rows it needs across the bus, dequantizing there. No HBM for the table. `0` falls back to gathering the rows on the host, which returns the same rows but costs ~50 ms of CPU per decode step with the GPU idle behind it. Anything that would make the device path unsafe — no CUDA, more TP ranks than hash heads, a registration that will not fit — falls back on its own, so the switch is for taking the host path deliberately. The fallback is the whole TP group's: the lookup ends in an all-gather, so one rank that cannot register turns every rank around rather than leaving the others in a collective it never enters. Also governs the Qwen3.8 PLE n-gram table (~95 GiB in BF16): page-locked on the host and read over UVA on gfx950 and gfx942, in HBM elsewhere, for a dummy load, under the SGLang / RTP-LLM plugins and with `0`, except where Qwen Mono serves the deployment (TP1 Quark MXFP4 with MTP3, `atom/models/qwen38/mono/ple.py`), which keeps it on the host whatever this variable says. Each data-parallel rank pins its own copy (sharing one is tracked in [#2861](https://github.com/ROCm/ATOM/issues/2861)), and a rank whose shard cannot be page-locked keeps its table in HBM with a warning. | | **ATOM_ENGRAM_UVA_COPY** | bool | unset: 1, except under DP attention | How the UVA path page-locks this rank's shard. Unset, a TP or single-GPU rank copies (`1`), and a DP-attention rank -- whose TP group is one GPU, so it holds the whole table, the same one as every other DP rank of the node -- registers in place (`0`), copying only if that is refused: a copy per rank is ~183 GiB each for V4.1-Flash (~730 GiB on DEP4), the in-place registration pins the shared checkpoint page cache once per node. Set explicitly, it decides for every rank. `1` copies the rows into page-locked memory of exactly their size (`hipHostMalloc`, on base pages: with transparent huge pages it compacts memory once the page cache has fragmented it, ~197 s per TP2 rank instead of ~12 s): one `copy_` below 512 MiB, otherwise one thread per 128 MiB but at least 8, at most 16 and the CPUs this process may run on (two or three threads lose to one `copy_`); each copied chunk of the checkpoint is dropped from the process (`madvise(DONTNEED)`; it stays in the page cache, reclaimable), so the shard is resident once and the peak stays near one shard. `0` registers the checkpoint mapping in place (`hipHostRegister`). The Engram tables are mapped shared and read-only, so this pins the page cache itself -- the same pages for every process that maps the file, no copy -- faulted in on several threads first and registered read-only in 1 GiB pieces (one `cudaHostRegister` covering a whole ~92 GiB table, issued by several DP ranks at once, does not return), and `mlock`ed while registered (faulted in and locked in one parallel pass, unlocked when the last tensor over it goes). The lock covers the shared checkpoint page cache node-wide, so host memory pressure ends in the OOM killer rather than in reclaim. A refused `mlock` (`RLIMIT_MEMLOCK` without `CAP_IPC_LOCK`) warns once and the registration stands. **Required host settings for in-place registration:** `vm.compact_unevictable_allowed=0` and `vm.compaction_proactiveness=0`; with the defaults kcompactd migrates the registered pages, locked or not, and each migration evicts the process's GPU queues (decode stalls of seconds, or a hang; watch `/sys/class/kfd/kfd/proc//stats_*/evicted_ms`). Ranges that share a page (a layer's scales right before its values) share one reference-counted registration; a refused registration (or any failed HIP call) clears HIP's last error so the host-path fallback still runs. The page-locked memory is owned by the tensors the kernel reads and released (and unlocked) with the last of them. Needs the HIP runtime -- the one torch loaded, found in `/proc/self/maps` -- and without it the shard is registered in place. `tools/engram/bench_uva_shard.py` reproduces per-rank time, memory and gather latency for either mode. The Qwen3.8 PLE table is always copied: its shards share safetensors files with other weights, whose host-to-device copies fail once part of the mapping is registered in place. | | **ATOM_ENGRAM_REG_LOCK** | path | per checkpoint, under `/dev/shm` | Lock file for in-place registration of a shared Engram table (DP attention: every rank page-locks the same checkpoint). Unset, the path is `/dev/shm/engram-uva-.lock`, hashed from that checkpoint, so two servers loading different checkpoints do not queue on each other; set it to force one path. Taken only around `cudaHostRegister` / `cudaHostUnregister` (fault-in runs before it), and only for that shared registration — a TP rank registering its own shard does not take it. A wait is logged. The holder touches the file after every 1 GiB chunk, so a slow but progressing holder (a cold page cache, minutes per rank) is waited on for as long as it takes; only a holder with no progress for 600 s makes the waiter give up and fall back to copying. After that the path is never waited on again in that process: its later tables copy at once, and its releases unregister without the lock. A lock file that cannot be opened or locked is logged and skipped, and the rank registers without it. `/dev/shm` is private to a container unless shared (`docker run --ipc=host`), so ranks that do not share it must set this to a path they all see. | -| **ATOM_ENGRAM_CACHE_DIR** | path | `~/.cache/atom/engram` | Where the compressed-vocab table is cached between runs. The table is reproducible from the tokenizer, so this only trades startup time for disk; point it at shared storage to let several servers build it once. A truncated or stale cache is rebuilt rather than raised. | +| **ATOM_ENGRAM_CACHE_DIR** | path | `~/.cache/atom/engram` | Where the compressed-vocab table is cached between runs. The table is reproducible from the tokenizer, so this only trades startup time for disk; point it at shared storage to let several servers build it once. A truncated or stale cache is rebuilt rather than raised. An empty value reads as the default, not as the working directory. | ## Attention side streams (DeepSeek-V4.1) diff --git a/docs/serving_benchmarking_guide.md b/docs/serving_benchmarking_guide.md index 230f78b3cb..ba9d3b1ef1 100644 --- a/docs/serving_benchmarking_guide.md +++ b/docs/serving_benchmarking_guide.md @@ -839,8 +839,9 @@ Script-specific arguments: | `--bs` | `1` | Batch size (number of parallel requests) | | `--random-input` | `False` | Use random token input instead of predefined text | -If `--torch-profiler-dir` is not specified, the script defaults to -`./profiler_traces`. +If `--torch-profiler-dir` is not specified, the script writes to +`ATOM_TORCH_PROFILER_DIR` when that is set (and not empty), and otherwise +defaults to `./profiler_traces`. ### Profiling during benchmarks diff --git a/tests/test_envs.py b/tests/test_envs.py index 2eadb2cddb..cd48e7744f 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -2,6 +2,7 @@ # Tests for atom/utils/envs.py — lazy env var evaluation import logging +from pathlib import Path import pytest @@ -30,6 +31,8 @@ "ATOM_USE_MODEL_SENSITIVE_RMSNORM", "ATOM_TORCH_PROFILER_DIR", "ATOM_ONLINE_QUANT_INFO_DIR", + "ATOM_ENGRAM_CACHE_DIR", + "ATOM_RTL_TRACE_DIR", "ATOM_ENABLE_METRICS_DEVICE_TIMER", "ATOM_METRICS_UPDATE_INTERVAL_S", "ATOM_SHUTDOWN_TIMEOUT_S", @@ -114,6 +117,14 @@ def test_torch_profiler_dir_default(self): def test_online_quant_info_dir_default(self): assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR is None + def test_engram_cache_dir_default(self): + assert _get_envs().ATOM_ENGRAM_CACHE_DIR == str( + Path.home() / ".cache" / "atom" / "engram" + ) + + def test_rtl_trace_dir_default(self): + assert _get_envs().ATOM_RTL_TRACE_DIR is None + def test_metrics_device_timer_default_disabled(self): assert _get_envs().ATOM_ENABLE_METRICS_DEVICE_TIMER is False @@ -222,6 +233,26 @@ def test_online_quant_info_dir_ignores_empty_profiler_dir(self, monkeypatch): monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "") assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR is None + def test_engram_cache_dir_override(self, monkeypatch): + monkeypatch.setenv("ATOM_ENGRAM_CACHE_DIR", "/tmp/engram") + assert _get_envs().ATOM_ENGRAM_CACHE_DIR == "/tmp/engram" + + def test_empty_engram_cache_dir_reads_as_its_default(self, monkeypatch): + # Path("") is the working directory, so an empty value would otherwise + # drop the compressed-vocab cache beside wherever the server started. + monkeypatch.setenv("ATOM_ENGRAM_CACHE_DIR", "") + assert _get_envs().ATOM_ENGRAM_CACHE_DIR == str( + Path.home() / ".cache" / "atom" / "engram" + ) + + def test_rtl_trace_dir_override(self, monkeypatch): + monkeypatch.setenv("ATOM_RTL_TRACE_DIR", "/tmp/rtl") + assert _get_envs().ATOM_RTL_TRACE_DIR == "/tmp/rtl" + + def test_empty_rtl_trace_dir_reads_as_unset(self, monkeypatch): + monkeypatch.setenv("ATOM_RTL_TRACE_DIR", "") + assert _get_envs().ATOM_RTL_TRACE_DIR is None + def test_profiler_more_enabled(self, monkeypatch): monkeypatch.setenv("ATOM_PROFILER_MORE", "1") assert _get_envs().ATOM_PROFILER_MORE is True diff --git a/tests/test_profile_offline.py b/tests/test_profile_offline.py new file mode 100644 index 0000000000..ea31ee7f3d --- /dev/null +++ b/tests/test_profile_offline.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: MIT +# Tests for the offline profiling example's trace-directory resolution. + +import pytest + +from atom.examples.profile_offline import ( + DEFAULT_PROFILER_DIR, + resolve_profiler_dir, +) + +_DEFAULT_NOTICE = ( + "Warning: --torch-profiler-dir not specified and " + f"ATOM_TORCH_PROFILER_DIR unset, using default: {DEFAULT_PROFILER_DIR}" +) + + +@pytest.fixture(autouse=True) +def _clear_profiler_env(monkeypatch): + monkeypatch.delenv("ATOM_TORCH_PROFILER_DIR", raising=False) + + +@pytest.mark.parametrize( + "cli_value, env_value, expected_dir, expected_notice", + [ + # The example always profiles: with neither source set it keeps its + # own default instead of leaving the directory at None. + (None, None, DEFAULT_PROFILER_DIR, _DEFAULT_NOTICE), + # The environment now reaches the example, as it does the server. + ( + None, + "/tmp/env-traces", + "/tmp/env-traces", + ( + "Warning: --torch-profiler-dir not specified, using " + "ATOM_TORCH_PROFILER_DIR: /tmp/env-traces" + ), + ), + # An explicit flag still outranks the environment. + ("/tmp/cli-traces", None, "/tmp/cli-traces", None), + ("/tmp/cli-traces", "/tmp/env-traces", "/tmp/cli-traces", None), + # Empty reads as unset on both sides: the variable (the rule shared + # with the engine) and, in this example, the flag. + (None, "", DEFAULT_PROFILER_DIR, _DEFAULT_NOTICE), + ( + "", + "/tmp/env-traces", + "/tmp/env-traces", + ( + "Warning: --torch-profiler-dir not specified, using " + "ATOM_TORCH_PROFILER_DIR: /tmp/env-traces" + ), + ), + ("", None, DEFAULT_PROFILER_DIR, _DEFAULT_NOTICE), + ], +) +def test_resolve_profiler_dir( + monkeypatch, cli_value, env_value, expected_dir, expected_notice +): + if env_value is not None: + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", env_value) + + directory, notice = resolve_profiler_dir(cli_value) + + assert directory == expected_dir + assert notice == expected_notice From d9dd55bc07fc98c573990ec0d3dee5146d165043 Mon Sep 17 00:00:00 2001 From: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:12:47 +0000 Subject: [PATCH 3/5] refactor(profiler): keep online-quant fallback in the loader; trim example ATOM_ONLINE_QUANT_INFO_DIR reported ATOM_TORCH_PROFILER_DIR's value when its own variable was unset, so the attribute did not name what it returned. Make it a plain _optional_path_env of its own variable and write the fallback once at its only call site, in the loader: the dump's directory, else the profiler directory, else the working directory. Pin that effective directory in a new loader test, and have the envs test assert the knob reads only its own variable. Reduce the offline profiling example to the resolution it needs: the flag, or ATOM_TORCH_PROFILER_DIR, or the example's own directory, then a print of the directory. The empty-flag rule is now a comment where it applies and states that it is specific to this example, since EngineArgs forwards "" to Config. Revert the ATOM_V4_CG_INSTR_DIR edit, which changed nothing: the knob already read an empty value as None. Rebased onto 004f342ab; docs/environment_variables.md resolves to main's Engram rows plus this branch's single added sentence. Signed-off-by: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> --- atom/examples/profile_offline.py | 34 ++------- atom/model_loader/loader.py | 6 +- atom/utils/envs.py | 15 ++-- tests/test_envs.py | 18 ++--- tests/test_online_quant_info_dir.py | 104 ++++++++++++++++++++++++++++ tests/test_profile_offline.py | 71 +++++++++---------- 6 files changed, 159 insertions(+), 89 deletions(-) create mode 100644 tests/test_online_quant_info_dir.py diff --git a/atom/examples/profile_offline.py b/atom/examples/profile_offline.py index 3eb5fe8f7f..0688607621 100644 --- a/atom/examples/profile_offline.py +++ b/atom/examples/profile_offline.py @@ -45,36 +45,16 @@ ) -def resolve_profiler_dir(cli_value: str | None) -> tuple[str, str | None]: - """Resolve the trace directory, and the notice to print when it was not given. - - The precedence is the one every other entry point uses: an explicit value, - then ``ATOM_TORCH_PROFILER_DIR`` (where empty reads as unset), then this - example's own ``./profiler_traces``. It ends with a directory rather than - ``None`` because the example always profiles; an empty - ``--torch-profiler-dir`` counts as not given, like an empty environment - value. - """ - if cli_value: - return cli_value, None - env_dir = envs.ATOM_TORCH_PROFILER_DIR - if env_dir is not None: - return env_dir, ( - "Warning: --torch-profiler-dir not specified, using " - f"ATOM_TORCH_PROFILER_DIR: {env_dir}" - ) - return DEFAULT_PROFILER_DIR, ( - "Warning: --torch-profiler-dir not specified and " - f"ATOM_TORCH_PROFILER_DIR unset, using default: {DEFAULT_PROFILER_DIR}" - ) - - def main(): args = parser.parse_args() - args.torch_profiler_dir, notice = resolve_profiler_dir(args.torch_profiler_dir) - if notice is not None: - print(notice) + # Flag, then the environment, then this example's own directory, the order + # the engine uses. An empty flag counts as unset only here; EngineArgs + # forwards "" to Config, which aborts on os.makedirs(""). + args.torch_profiler_dir = ( + args.torch_profiler_dir or envs.ATOM_TORCH_PROFILER_DIR or DEFAULT_PROFILER_DIR + ) + print(f"Writing profiler traces to {args.torch_profiler_dir}") print("\nInitializing LLM engine...") engine_args = EngineArgs.from_cli_args(args) diff --git a/atom/model_loader/loader.py b/atom/model_loader/loader.py index 0afd3aa4c3..5d5cce5a58 100644 --- a/atom/model_loader/loader.py +++ b/atom/model_loader/loader.py @@ -177,9 +177,9 @@ def _save_online_quant_info( """Save online quantization info to a JSON file (rank 0 only).""" if get_tp_group().rank_in_group != 0: return - # Its own knob (falling back to ATOM_TORCH_PROFILER_DIR), then the - # historical default. - output_dir = envs.ATOM_ONLINE_QUANT_INFO_DIR or os.getcwd() + output_dir = ( + envs.ATOM_ONLINE_QUANT_INFO_DIR or envs.ATOM_TORCH_PROFILER_DIR or os.getcwd() + ) os.makedirs(output_dir, exist_ok=True) timestamp = time.strftime("%Y%m%d_%H%M%S") timestamp_ns = time.time_ns() % 1_000_000_000 diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 1de99d3151..97d4edb1fa 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -958,14 +958,11 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: "ATOM_LOADER_STRICT_COVERAGE": lambda: ( os.getenv("ATOM_LOADER_STRICT_COVERAGE", "true").lower() == "true" ), - # Directory rank 0 writes the online_quant_info_*.json dump to. Its own - # knob so that a launcher collecting the dump no longer has to hand the - # engine a profiler directory to get it: unset (or empty) falls back to - # ATOM_TORCH_PROFILER_DIR, the dump's only home before this one existed. - # The caller falls back to the working directory from there. - "ATOM_ONLINE_QUANT_INFO_DIR": lambda: ( - _optional_path_env("ATOM_ONLINE_QUANT_INFO_DIR") - or _optional_path_env("ATOM_TORCH_PROFILER_DIR") + # Directory rank 0 writes the online_quant_info_*.json dump to, when set. + # Its own knob so that a launcher collecting the dump no longer has to + # hand the engine a profiler directory to get it. + "ATOM_ONLINE_QUANT_INFO_DIR": lambda: _optional_path_env( + "ATOM_ONLINE_QUANT_INFO_DIR" ), # Quantize eligible modules as they load to reduce peak memory. Streaming # quantizes local TP shards, so results may differ slightly from offline. @@ -1043,7 +1040,7 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # vLLM plugin, DeepSeek-V4: directory for per-process cudagraph dispatch # counters (modes, overridden steps, guard trips), one JSON file per # process, rewritten every step. Unset: off. - "ATOM_V4_CG_INSTR_DIR": lambda: _optional_path_env("ATOM_V4_CG_INSTR_DIR"), + "ATOM_V4_CG_INSTR_DIR": lambda: os.getenv("ATOM_V4_CG_INSTR_DIR") or None, "ATOM_USE_CUSTOM_ALL_GATHER": lambda: ( os.getenv("ATOM_USE_CUSTOM_ALL_GATHER", "1").lower() == "1" ), diff --git a/tests/test_envs.py b/tests/test_envs.py index cd48e7744f..b24a3a9af3 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -216,21 +216,15 @@ def test_online_quant_info_dir_override(self, monkeypatch): assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR == "/tmp/oq" @pytest.mark.parametrize("value", [None, ""]) - def test_online_quant_info_dir_falls_back_to_profiler_dir(self, monkeypatch, value): - # The dump lived under ATOM_TORCH_PROFILER_DIR before it had a knob of - # its own; unset or empty keeps landing there. + def test_online_quant_info_dir_reads_only_its_own_variable( + self, monkeypatch, value + ): + # The knob reports its own variable and nothing else; the fallback to + # ATOM_TORCH_PROFILER_DIR lives at the loader's one call site + # (tests/test_online_quant_info_dir.py pins the effective directory). if value is not None: monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", value) monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof") - assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR == "/tmp/prof" - - def test_online_quant_info_dir_beats_profiler_dir(self, monkeypatch): - monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", "/tmp/oq") - monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "/tmp/prof") - assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR == "/tmp/oq" - - def test_online_quant_info_dir_ignores_empty_profiler_dir(self, monkeypatch): - monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", "") assert _get_envs().ATOM_ONLINE_QUANT_INFO_DIR is None def test_engram_cache_dir_override(self, monkeypatch): diff --git a/tests/test_online_quant_info_dir.py b/tests/test_online_quant_info_dir.py new file mode 100644 index 0000000000..1e26f7edf1 --- /dev/null +++ b/tests/test_online_quant_info_dir.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: MIT +# Tests for the directory rank 0's online-quantization dump lands in. + +import importlib +from types import SimpleNamespace + +import pytest + + +def _import_loader(): + """The loader module, against AITER's stand-ins when it cannot import. + + Its import chain binds AITER kernels, which a CPU-only runner does not + have. ``atom_under_stubs`` drops the modules it imported on the way out; + the module object returned here keeps its own globals, which is all these + tests reach for. + """ + try: + return importlib.import_module("atom.model_loader.loader") + except ImportError as exc: + # A third-party module missing is this CPU image. An import error + # naming one of ours is a bug in the loader and has to surface + # (tests/import_guard.py's rule), not be retried behind stand-ins. + if (exc.name or "").startswith("atom"): + raise + from aiter_stub import atom_under_stubs + + with atom_under_stubs() as load: + return load("atom.model_loader.loader") + + +@pytest.fixture(scope="module") +def loader(): + return _import_loader() + + +@pytest.fixture(autouse=True) +def _clear_dir_envs(monkeypatch): + for name in ("ATOM_ONLINE_QUANT_INFO_DIR", "ATOM_TORCH_PROFILER_DIR"): + monkeypatch.delenv(name, raising=False) + + +def _save(loader, monkeypatch, tmp_path, rank): + monkeypatch.setattr( + loader, "get_tp_group", lambda: SimpleNamespace(rank_in_group=rank) + ) + monkeypatch.chdir(tmp_path) + loader._save_online_quant_info( + oq_layers=[], + model_name_or_path="dummy", + elapsed_seconds=1.0, + online_quant_config={}, + timing_scope="load", + peak_gpu_memory_gb=None, + ) + + +def _saved(directory): + return list(directory.glob("online_quant_info_*.json")) + + +def test_own_knob_names_the_directory(loader, monkeypatch, tmp_path): + own = tmp_path / "oq" + monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", str(own)) + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", str(tmp_path / "prof")) + + _save(loader, monkeypatch, tmp_path, rank=0) + + assert len(_saved(own)) == 1 + assert not (tmp_path / "prof").exists() + + +@pytest.mark.parametrize("own_value", [None, ""]) +def test_unset_knob_falls_back_to_the_profiler_dir( + loader, monkeypatch, tmp_path, own_value +): + # The dump lived under ATOM_TORCH_PROFILER_DIR before it had a knob of its + # own; unset or empty keeps landing there. + if own_value is not None: + monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", own_value) + prof = tmp_path / "prof" + monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", str(prof)) + + _save(loader, monkeypatch, tmp_path, rank=0) + + assert len(_saved(prof)) == 1 + assert not _saved(tmp_path) + + +def test_no_directory_set_falls_back_to_the_working_directory( + loader, monkeypatch, tmp_path +): + _save(loader, monkeypatch, tmp_path, rank=0) + + assert len(_saved(tmp_path)) == 1 + + +def test_other_ranks_write_nothing(loader, monkeypatch, tmp_path): + own = tmp_path / "oq" + monkeypatch.setenv("ATOM_ONLINE_QUANT_INFO_DIR", str(own)) + + _save(loader, monkeypatch, tmp_path, rank=1) + + assert not own.exists() diff --git a/tests/test_profile_offline.py b/tests/test_profile_offline.py index ea31ee7f3d..b5dbad56bc 100644 --- a/tests/test_profile_offline.py +++ b/tests/test_profile_offline.py @@ -3,15 +3,11 @@ import pytest -from atom.examples.profile_offline import ( - DEFAULT_PROFILER_DIR, - resolve_profiler_dir, -) +from atom.examples import profile_offline -_DEFAULT_NOTICE = ( - "Warning: --torch-profiler-dir not specified and " - f"ATOM_TORCH_PROFILER_DIR unset, using default: {DEFAULT_PROFILER_DIR}" -) + +class _EngineArgsReached(Exception): + """Stops main() once the resolved directory has been handed over.""" @pytest.fixture(autouse=True) @@ -20,46 +16,45 @@ def _clear_profiler_env(monkeypatch): @pytest.mark.parametrize( - "cli_value, env_value, expected_dir, expected_notice", + "cli_value, env_value, expected", [ # The example always profiles: with neither source set it keeps its # own default instead of leaving the directory at None. - (None, None, DEFAULT_PROFILER_DIR, _DEFAULT_NOTICE), - # The environment now reaches the example, as it does the server. - ( - None, - "/tmp/env-traces", - "/tmp/env-traces", - ( - "Warning: --torch-profiler-dir not specified, using " - "ATOM_TORCH_PROFILER_DIR: /tmp/env-traces" - ), - ), + (None, None, profile_offline.DEFAULT_PROFILER_DIR), + # The environment reaches the example, as it does the server. + (None, "/tmp/env-traces", "/tmp/env-traces"), # An explicit flag still outranks the environment. - ("/tmp/cli-traces", None, "/tmp/cli-traces", None), - ("/tmp/cli-traces", "/tmp/env-traces", "/tmp/cli-traces", None), + ("/tmp/cli-traces", None, "/tmp/cli-traces"), + ("/tmp/cli-traces", "/tmp/env-traces", "/tmp/cli-traces"), # Empty reads as unset on both sides: the variable (the rule shared # with the engine) and, in this example, the flag. - (None, "", DEFAULT_PROFILER_DIR, _DEFAULT_NOTICE), - ( - "", - "/tmp/env-traces", - "/tmp/env-traces", - ( - "Warning: --torch-profiler-dir not specified, using " - "ATOM_TORCH_PROFILER_DIR: /tmp/env-traces" - ), - ), - ("", None, DEFAULT_PROFILER_DIR, _DEFAULT_NOTICE), + (None, "", profile_offline.DEFAULT_PROFILER_DIR), + ("", "/tmp/env-traces", "/tmp/env-traces"), + ("", None, profile_offline.DEFAULT_PROFILER_DIR), ], ) -def test_resolve_profiler_dir( - monkeypatch, cli_value, env_value, expected_dir, expected_notice +def test_main_resolves_profiler_dir( + monkeypatch, capsys, cli_value, env_value, expected ): if env_value is not None: monkeypatch.setenv("ATOM_TORCH_PROFILER_DIR", env_value) - directory, notice = resolve_profiler_dir(cli_value) + seen = {} + + class _RecordingEngineArgs: + @staticmethod + def from_cli_args(args): + seen["dir"] = args.torch_profiler_dir + raise _EngineArgsReached + + monkeypatch.setattr(profile_offline, "EngineArgs", _RecordingEngineArgs) + argv = ["profile_offline.py"] + if cli_value is not None: + argv += ["--torch-profiler-dir", cli_value] + monkeypatch.setattr("sys.argv", argv) + + with pytest.raises(_EngineArgsReached): + profile_offline.main() - assert directory == expected_dir - assert notice == expected_notice + assert seen["dir"] == expected + assert f"Writing profiler traces to {expected}" in capsys.readouterr().out From ab725c85735eea9795b9f4ef20d7a73e5fd1c64d Mon Sep 17 00:00:00 2001 From: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:14:33 +0000 Subject: [PATCH 4/5] test(loader): reuse atom_without_aiter in the online-quant dir test Replace the test's hand-written try/except ImportError fallback with aiter_stub.atom_without_aiter, the helper the other stand-in tests use, so the loader is loaded exactly as theirs is. That helper gated on `import aiter`, which succeeds on a host with an AITER build and no GPU: AITER defers its arch probe, and the submodule that binds a kernel -- reached from the loader's import chain -- then fails on the missing device. Take the stand-ins unless a GPU is available, the line tests/conftest.gpu_stack_import already draws. A GPU host and the CI unit runner (no AITER at all) keep the paths they had. Signed-off-by: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> --- tests/aiter_stub.py | 19 +++++++++++++++---- tests/test_online_quant_info_dir.py | 28 +++++++--------------------- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/tests/aiter_stub.py b/tests/aiter_stub.py index a1ca85cb14..c0c5fb56fc 100644 --- a/tests/aiter_stub.py +++ b/tests/aiter_stub.py @@ -293,12 +293,23 @@ def atom_under_stubs(): @contextmanager def atom_without_aiter(): - """`importlib.import_module`, against the stand-ins only where AITER is - not installed (the CPU job); a GPU host keeps the real kernels.""" + """`importlib.import_module`, against the stand-ins only where AITER + cannot run here (the CPU job); a GPU host keeps the real kernels. + + `import aiter` alone does not answer that: an AITER build on a host with + no GPU still imports, because its arch probe is deferred, and the + submodule that binds a kernel then fails on the missing device. An ATOM + module whose import chain reaches that submodule needs the stand-ins + there too. `gpu_stack_import` in tests/conftest.py draws the same line. + """ try: import aiter # noqa: F401 except ImportError: - with atom_under_stubs() as load: - yield load + usable = False else: + usable = torch.cuda.is_available() + if usable: yield importlib.import_module + else: + with atom_under_stubs() as load: + yield load diff --git a/tests/test_online_quant_info_dir.py b/tests/test_online_quant_info_dir.py index 1e26f7edf1..b360c82a75 100644 --- a/tests/test_online_quant_info_dir.py +++ b/tests/test_online_quant_info_dir.py @@ -1,37 +1,23 @@ # SPDX-License-Identifier: MIT # Tests for the directory rank 0's online-quantization dump lands in. -import importlib from types import SimpleNamespace import pytest +from aiter_stub import atom_without_aiter -def _import_loader(): +@pytest.fixture(scope="module") +def loader(): """The loader module, against AITER's stand-ins when it cannot import. Its import chain binds AITER kernels, which a CPU-only runner does not - have. ``atom_under_stubs`` drops the modules it imported on the way out; - the module object returned here keeps its own globals, which is all these + have. The stand-ins drop the modules they imported on the way out; the + module object returned here keeps its own globals, which is all these tests reach for. """ - try: - return importlib.import_module("atom.model_loader.loader") - except ImportError as exc: - # A third-party module missing is this CPU image. An import error - # naming one of ours is a bug in the loader and has to surface - # (tests/import_guard.py's rule), not be retried behind stand-ins. - if (exc.name or "").startswith("atom"): - raise - from aiter_stub import atom_under_stubs - - with atom_under_stubs() as load: - return load("atom.model_loader.loader") - - -@pytest.fixture(scope="module") -def loader(): - return _import_loader() + with atom_without_aiter() as load: + return load("atom.model_loader.loader") @pytest.fixture(autouse=True) From 4954cac80917b0eb4063c31ea2ec31841dad0630 Mon Sep 17 00:00:00 2001 From: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:08:44 +0000 Subject: [PATCH 5/5] docs(envs): describe ATOM_RTL_TRACE_DIR's actual consumer The comment said an empty value disables tracing, but no Python code reads envs.ATOM_RTL_TRACE_DIR: the variable's only consumer is .github/scripts/atom_test.sh, which uses it as the output directory for `rtl trace -o` under ENABLE_RTL_PROFILER=1 and, there, an empty value falls back to /app/rtl_traces. Say that instead, and note the empty-reads-as-unset rule only for the Python reader, consistent with the other directory knobs. Signed-off-by: Yaakoub El Khamra <176948988+yelkhamra@users.noreply.github.com> --- atom/utils/envs.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/atom/utils/envs.py b/atom/utils/envs.py index 97d4edb1fa..3d4ddbabf9 100644 --- a/atom/utils/envs.py +++ b/atom/utils/envs.py @@ -892,11 +892,12 @@ def _finite_float_env(name: str, default: float, *, allow_zero: bool) -> float: # WARNING, ...); applied by atom.utils.set_atom_log_level(), which turns # NOTSET and unknown names into INFO. "ATOM_LOG_LEVEL": lambda: os.getenv("ATOM_LOG_LEVEL", "INFO"), - # RTL (rocm-trace-lite) GPU kernel tracing — set to output directory to enable. - # When set, the server launch is wrapped with `rtl trace` to collect per-kernel - # GPU timestamps for both prefill and decode phases. Empty reads as unset, - # like the other directory knobs, so `VAR=` disables it rather than naming - # "" as the output directory. + # RTL (rocm-trace-lite) GPU kernel tracing. No Python code outside tests + # consumes this value: it names the output directory that + # .github/scripts/atom_test.sh passes to `rtl trace -o` when + # ENABLE_RTL_PROFILER=1, and reads back when the server stops; an empty + # value falls back to /app/rtl_traces there. Empty reads as unset here, + # like the other directory knobs. "ATOM_RTL_TRACE_DIR": lambda: _optional_path_env("ATOM_RTL_TRACE_DIR"), # --- Model Loading --- "ATOM_DISABLE_MMAP": lambda: (