From a8d69c9daba9b2a6b4e2404b6dcf9eb30d108a0d Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Thu, 22 Oct 2026 14:32:16 -0500 Subject: [PATCH 1/8] Prune uv cache to what uv.lock still pins The release image's shipped uv cache (UV_CACHE_DIR) is append-only: uv.lock's own cache-seed key is BASE_IMAGE+UV_VERSION+HYBRID_EP_MULTINODE, not the lock content, so every routine dependency-version bump (torch, vLLM, SGLang, ...) leaves its predecessor's wheel behind forever as long as that key stays the same -- which is most of the time. Add docker/prune_uv_cache.py, run once after all uv-writing RUNs finish and before the ownership handoff. It parses uv.lock for every (name, version) pin across all extras (installed in this build or not, so it cannot break the offline `uv run --extra ` promise UV_CACHE_DIR exists for) and removes only cache entries nothing in the current lock still references. Signed-off-by: Kajal Jain --- docker/Dockerfile | 15 ++ docker/prune_uv_cache.py | 187 ++++++++++++++++++++++++ docs/docker.md | 6 + tests/unit/tools/test_prune_uv_cache.py | 122 ++++++++++++++++ 4 files changed, 330 insertions(+) create mode 100644 docker/prune_uv_cache.py create mode 100644 tests/unit/tools/test_prune_uv_cache.py diff --git a/docker/Dockerfile b/docker/Dockerfile index 7f3c00a994c..4a8a739cc97 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -690,6 +690,21 @@ if [[ -n "${NEMO_GYM_PREFETCH_CONFIGS:-}" ]]; then rm -f "${GYM_VLLM_OVERRIDE}" fi EOF + +# Drop uv cache entries no longer pinned by uv.lock -- see docker/prune_uv_cache.py's +# module docstring for the full accumulation mechanism this addresses (every +# dependency-version bump leaves its predecessor's wheel behind forever, because the +# uv cache seed's own staleness key is BASE_IMAGE+UV_VERSION+HYBRID_EP_MULTINODE, not +# the lock content -- see the dependency-install RUN above). Must run after every +# uv-writing RUN above (dependency-install, TRT-LLM, Dynamo, worker-venv prefetch, +# Gym prefetch) and before the ownership handoff below, so it can only ever remove +# entries nothing in THIS build still needs -- it keeps every (name, version) pair +# uv.lock pins across every extra, installed in this build or not, so it cannot break +# the offline `uv run --extra ` promise that UV_CACHE_DIR exists for. +RUN echo "uv cache size before pruning:" && du -sh "${UV_CACHE_DIR}" && \ + python docker/prune_uv_cache.py --lock uv.lock --cache-dir "${UV_CACHE_DIR}" && \ + echo "uv cache size after pruning:" && du -sh "${UV_CACHE_DIR}" + # ray's bundled Java worker jar ships an outdated httpcore5 with a known CVE # (GHSA-hf6x-8p5f-cgmf). NeMo-RL never uses Ray's Java cross-language worker, # and removing the jar doesn't affect ray.init(), remote tasks, actors, or diff --git a/docker/prune_uv_cache.py b/docker/prune_uv_cache.py new file mode 100644 index 00000000000..a98d6d1ac33 --- /dev/null +++ b/docker/prune_uv_cache.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prune uv's PyPI wheel/sdist cache down to what uv.lock still pins. + +uv.lock pins exactly one version per package across the WHOLE project (every +extra, installed in this build or not -- see the offline-extras comment on +UV_CACHE_DIR in docker/Dockerfile). But uv's cache is append-only: `uv sync` +only ever downloads what a build needs, it never removes a version a prior +lockfile revision pinned and this one no longer does. Every routine +dependency bump (torch, vllm, sglang, ...) therefore leaves its predecessor's +wheel sitting in the cache forever, as long as BASE_IMAGE/UV_VERSION/ +HYBRID_EP_MULTINODE haven't changed (see docker/Dockerfile's dependency-install +RUN for why those three are the only things that ever reset the cache). + +This script removes exactly those superseded entries -- never anything +`uv.lock` still references, installed or not -- so it is safe to run before +shipping UV_CACHE_DIR in the release image: it cannot remove a wheel the +offline `uv run --extra ` promise depends on, because that promise +is itself "every wheel uv.lock could need," which is exactly the keep set +computed here. + +Scope: only the `pypi/` subtree of `wheels-v*`/`sdists-v*` (the dominant +case -- every normal PyPI-sourced dependency). Git/URL/local-path/custom-index +sources (`wheels-v*/url/`, `wheels-v*/index/`, `git-v*/`) are deliberately +left untouched: their directory naming does not reliably encode a PEP 503 +package name + exact version the way the pypi/ tree does, so matching them +against uv.lock risks a false match/miss. They are a small fraction of the +cache's total size (see the before/after `du -sh` this script prints). +""" + +from __future__ import annotations + +import argparse +import re +import shutil +import sys +import tomllib +from pathlib import Path + +# PEP 503 normalization: case-insensitive, runs of -_. collapse to one '-'. +# uv.lock package names and the cache's on-disk directory names are not +# guaranteed to use the same casing/separator, so both sides must go through +# this before comparing. +_NORMALIZE_RE = re.compile(r"[-_.]+") + + +def normalize(name: str) -> str: + return _NORMALIZE_RE.sub("-", name).lower() + + +def load_keep_set(lock_path: Path) -> set[tuple[str, str]]: + with lock_path.open("rb") as f: + data = tomllib.load(f) + packages = data.get("package", []) + keep = { + (normalize(p["name"]), p["version"]) + for p in packages + if "name" in p and "version" in p + } + return keep + + +def _entry_name_version(entry: Path) -> tuple[str, str] | None: + # entry is e.g. wheels-v6/pypi//- + # or sdists-v*/pypi// + name = normalize(entry.parent.name) + version = entry.name.split("-", 1)[0] + if not version: + return None + return name, version + + +def prune( + cache_dir: Path, keep: set[tuple[str, str]], *, dry_run: bool +) -> tuple[int, int]: + """Returns (entries_removed, archives_removed).""" + reachable: set[str] = set() + entries_removed = 0 + + pypi_roots = [ + p + for pattern in ("wheels-v*/pypi", "sdists-v*/pypi") + for p in cache_dir.glob(pattern) + ] + + for pypi_root in pypi_roots: + for name_dir in pypi_root.iterdir(): + if not name_dir.is_dir(): + continue + for entry in name_dir.iterdir(): + parsed = _entry_name_version(entry) + if parsed is None: + continue + if parsed in keep: + if entry.is_symlink(): + reachable.add(Path(entry.resolve()).name) + continue + entries_removed += 1 + if dry_run: + continue + # Remove the pointer directory/symlink and its sibling + # metadata files (.msgpack, .http). + for sibling in name_dir.glob(f"{entry.name}*"): + if sibling.is_symlink() or sibling.is_file(): + sibling.unlink() + elif sibling.is_dir(): + shutil.rmtree(sibling) + if not dry_run and not any(name_dir.iterdir()): + name_dir.rmdir() + + # Second pass: also mark reachable archives for every KEPT entry we + # didn't already see above (entries outside wheels-v*/sdists-v*/pypi + # that still resolve into archive-v0, if any such mounts exist). + archives_removed = 0 + for archive_root in cache_dir.glob("archive-v0"): + if not archive_root.is_dir(): + continue + for archive_dir in archive_root.iterdir(): + if archive_dir.name in reachable: + continue + archives_removed += 1 + if not dry_run: + shutil.rmtree(archive_dir, ignore_errors=True) + + return entries_removed, archives_removed + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lock", type=Path, required=True, help="Path to uv.lock") + parser.add_argument( + "--cache-dir", type=Path, required=True, help="UV_CACHE_DIR to prune" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Report what would be removed without removing it", + ) + # Sanity floor: if parsing uv.lock somehow yields a near-empty keep set + # (e.g. a future uv.lock format change this script doesn't understand), + # refuse to prune rather than silently deleting almost everything. + parser.add_argument("--min-keep", type=int, default=100) + args = parser.parse_args() + + if not args.lock.is_file(): + print(f"error: {args.lock} does not exist", file=sys.stderr) + return 1 + if not args.cache_dir.is_dir(): + print(f"error: {args.cache_dir} does not exist", file=sys.stderr) + return 1 + + keep = load_keep_set(args.lock) + if len(keep) < args.min_keep: + print( + f"error: only parsed {len(keep)} pinned packages from {args.lock} " + f"(expected at least {args.min_keep}) -- refusing to prune, " + "uv.lock's format may have changed in a way this script doesn't understand", + file=sys.stderr, + ) + return 1 + + entries_removed, archives_removed = prune( + args.cache_dir, keep, dry_run=args.dry_run + ) + verb = "Would remove" if args.dry_run else "Removed" + print( + f"{verb} {entries_removed} stale pypi cache entries and {archives_removed} " + f"orphaned archives (kept {len(keep)} packages pinned in {args.lock})" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/docker.md b/docs/docker.md index 62baf8a5c64..6f6a1c537cf 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -33,6 +33,12 @@ docker buildx build --build-context nemo-rl=. -f docker/Dockerfile \ > [!NOTE] > The `--tag /nemo-rl:latest --push` flags are not necessary if you just want to build locally. +## uv Cache Pruning + +The release image ships `uv`'s download cache (`/opt/nemo_rl_cache/uv`) so an unprivileged runtime user can later run `uv run --extra ` fully offline. Because the shared build cache this is seeded from persists across many builds, routine dependency-version bumps (torch, vLLM, SGLang, ...) leave each predecessor version's wheel behind -- `uv` only ever adds to its cache, it never removes a version a prior `uv.lock` pinned and the current one doesn't. + +Before the image ships, `docker/prune_uv_cache.py` removes exactly those superseded entries: anything in the cache that isn't a `(package, version)` pair the current `uv.lock` still pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. + ## Skipping vLLM or SGLang Dependencies If you don't need vLLM, SGLang, or TRT-LLM support, you can skip building those dependencies to reduce build time and image size. Use the `SKIP_VLLM_BUILD`, `SKIP_SGLANG_BUILD`, and/or `SKIP_TRTLLM_BUILD` build arguments: diff --git a/tests/unit/tools/test_prune_uv_cache.py b/tests/unit/tools/test_prune_uv_cache.py new file mode 100644 index 00000000000..99697aa8c9c --- /dev/null +++ b/tests/unit/tools/test_prune_uv_cache.py @@ -0,0 +1,122 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).parents[3] + +# docker/ is not a package (no __init__.py, by design -- it mirrors a Docker +# build context, not Python source), so import the script directly by path +# rather than adding docker/ to sys.path. +_SPEC = importlib.util.spec_from_file_location( + "prune_uv_cache", REPO_ROOT / "docker" / "prune_uv_cache.py" +) +prune_uv_cache = importlib.util.module_from_spec(_SPEC) +sys.modules["prune_uv_cache"] = prune_uv_cache +_SPEC.loader.exec_module(prune_uv_cache) + + +def test_normalize_is_pep503() -> None: + assert prune_uv_cache.normalize("Foo_Bar.Baz") == "foo-bar-baz" + assert prune_uv_cache.normalize("foo--bar") == "foo-bar" + assert prune_uv_cache.normalize("UPPER") == "upper" + + +def test_load_keep_set_parses_uv_lock(tmp_path: Path) -> None: + lock = tmp_path / "uv.lock" + lock.write_text( + """ +version = 1 + +[[package]] +name = "Orjson" +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "torch" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +""" + ) + keep = prune_uv_cache.load_keep_set(lock) + assert keep == {("orjson", "3.12.0"), ("torch", "2.9.0")} + + +def _make_wheel_entry( + cache_dir: Path, name: str, version: str, tag: str, archive_id: str +) -> None: + pypi = cache_dir / "wheels-v6" / "pypi" / name + pypi.mkdir(parents=True, exist_ok=True) + archive = cache_dir / "archive-v0" / archive_id + archive.mkdir(parents=True, exist_ok=True) + (archive / "payload.txt").write_text("real wheel content") + (pypi / f"{version}-{tag}").symlink_to(archive, target_is_directory=True) + (pypi / f"{version}-{tag}.http").write_text("cache-control: ...") + + +def test_prune_keeps_locked_version_drops_stale_ones(tmp_path: Path) -> None: + cache_dir = tmp_path / "uv-cache" + # Three versions of the same package on disk, as if orjson got bumped + # 3.11.3 -> 3.11.9 -> 3.12.0 across three separate dependency-bump PRs + # while BASE_IMAGE/UV_VERSION/HYBRID_EP_MULTINODE stayed the same. + _make_wheel_entry( + cache_dir, "orjson", "3.11.3", "cp313-cp313-linux_x86_64", "archive-old-1" + ) + _make_wheel_entry( + cache_dir, "orjson", "3.11.9", "cp313-cp313-linux_x86_64", "archive-old-2" + ) + _make_wheel_entry( + cache_dir, "orjson", "3.12.0", "cp313-cp313-linux_x86_64", "archive-kept" + ) + + keep = {("orjson", "3.12.0")} + entries_removed, archives_removed = prune_uv_cache.prune( + cache_dir, keep, dry_run=False + ) + + assert entries_removed == 4 # 2 stale versions x (symlink + .http sibling) + assert archives_removed == 2 # the 2 stale archives, not the kept one + + pypi_dir = cache_dir / "wheels-v6" / "pypi" / "orjson" + remaining = {p.name for p in pypi_dir.iterdir()} + assert remaining == { + "3.12.0-cp313-cp313-linux_x86_64", + "3.12.0-cp313-cp313-linux_x86_64.http", + } + + archive_dir = cache_dir / "archive-v0" + assert {p.name for p in archive_dir.iterdir()} == {"archive-kept"} + assert ( + archive_dir / "archive-kept" / "payload.txt" + ).read_text() == "real wheel content" + + +def test_prune_dry_run_reports_without_removing(tmp_path: Path) -> None: + cache_dir = tmp_path / "uv-cache" + _make_wheel_entry( + cache_dir, "orjson", "3.11.3", "cp313-cp313-linux_x86_64", "archive-old" + ) + + entries_removed, archives_removed = prune_uv_cache.prune( + cache_dir, keep=set(), dry_run=True + ) + + assert entries_removed == 2 + assert archives_removed == 1 + # Nothing actually removed in dry-run mode. + assert (cache_dir / "wheels-v6" / "pypi" / "orjson").exists() + assert (cache_dir / "archive-v0" / "archive-old").exists() From 55dd90f41a51a9faa109e29450e043ff2d417316 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Thu, 22 Oct 2026 17:41:35 -0500 Subject: [PATCH 2/8] fix(ci): stop skipping the uv cache seed refresh on main update-uv-cache and update-uv-cache-gb200 had no status function in their if:, so GitHub implied success(), which also requires every transitive need to succeed. Since cicd-wait-in-queue (and later prepare-megatron-lock) became needs of the build jobs, and both are skipped on main pushes, the refresh has been skipped on every main run. The seed went stale, its cache key no longer matches, and every build that reruns the dependency layer starts from an empty uv cache. Use the always() ... !cancelled() pattern the build jobs already use, with explicit result checks on the jobs these depend on. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- .github/workflows/cicd-main.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index c7bad64967a..6c001201021 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -641,11 +641,17 @@ jobs: update-uv-cache: name: Update uv build cache needs: [build-container, org-member-pre-flight] + # always() is required: without a status function GitHub implies success(), which also + # requires every transitive need to succeed. cicd-wait-in-queue and prepare-megatron-lock + # are skipped on main pushes, so this job was silently skipped on every run. if: >- ${{ + always() && github.ref == 'refs/heads/main' && vars.UV_BUILD_CACHE == 'enabled' && - needs.build-container.result == 'success' + needs.org-member-pre-flight.result == 'success' && + needs.build-container.result == 'success' && + !cancelled() }} runs-on: ${{ startsWith(needs.org-member-pre-flight.outputs.runner_prefix, 'ephe-v2-') && format('{0}-a{1}', needs.org-member-pre-flight.outputs.runner_prefix, github.run_attempt) || needs.org-member-pre-flight.outputs.runner_prefix }} env: @@ -674,11 +680,15 @@ jobs: update-uv-cache-gb200: name: Update GB200 uv build cache needs: [build-container-gb200, gb200-config] + # always(): see update-uv-cache. if: >- ${{ + always() && github.ref == 'refs/heads/main' && vars.UV_BUILD_CACHE == 'enabled' && - needs.build-container-gb200.result == 'success' + needs.gb200-config.result == 'success' && + needs.build-container-gb200.result == 'success' && + !cancelled() }} runs-on: ${{ vars.GB200_RUNNER }} env: From f2d2057228c47c0b9cad0c992932f2e87746bdc6 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Thu, 22 Oct 2026 19:46:46 -0500 Subject: [PATCH 3/8] fix(docker): make uv cache prune cover every source and keep live archives - Garbage-collect archive-v0 by scanning every remaining link under wheels-v*/ and sdists-v*/ (as uv cache prune does), instead of only kept pypi entries. The old scan deleted archives still referenced by index/, url/ and built-from-source entries, leaving dangling links that break offline installs of e.g. torch. - Prune alternative-index trees (index//, used for torch and flashinfer) by (name, version), not just pypi/. - Prune git source builds and checkouts whose commit uv.lock no longer pins. - Judge each cache file by its own parsed version; the old prefix glob removed 1.0.1's files when pruning 1.0. - Keep the truncated form of long versions, matching uv's wheel cache keys. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- docker/prune_uv_cache.py | 266 ++++++++++++++---------- docs/docker.md | 2 +- tests/unit/tools/test_prune_uv_cache.py | 259 +++++++++++++++++------ 3 files changed, 355 insertions(+), 172 deletions(-) diff --git a/docker/prune_uv_cache.py b/docker/prune_uv_cache.py index a98d6d1ac33..db665161222 100644 --- a/docker/prune_uv_cache.py +++ b/docker/prune_uv_cache.py @@ -13,129 +13,178 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Prune uv's PyPI wheel/sdist cache down to what uv.lock still pins. - -uv.lock pins exactly one version per package across the WHOLE project (every -extra, installed in this build or not -- see the offline-extras comment on -UV_CACHE_DIR in docker/Dockerfile). But uv's cache is append-only: `uv sync` -only ever downloads what a build needs, it never removes a version a prior -lockfile revision pinned and this one no longer does. Every routine -dependency bump (torch, vllm, sglang, ...) therefore leaves its predecessor's -wheel sitting in the cache forever, as long as BASE_IMAGE/UV_VERSION/ -HYBRID_EP_MULTINODE haven't changed (see docker/Dockerfile's dependency-install -RUN for why those three are the only things that ever reset the cache). - -This script removes exactly those superseded entries -- never anything -`uv.lock` still references, installed or not -- so it is safe to run before -shipping UV_CACHE_DIR in the release image: it cannot remove a wheel the -offline `uv run --extra ` promise depends on, because that promise -is itself "every wheel uv.lock could need," which is exactly the keep set -computed here. - -Scope: only the `pypi/` subtree of `wheels-v*`/`sdists-v*` (the dominant -case -- every normal PyPI-sourced dependency). Git/URL/local-path/custom-index -sources (`wheels-v*/url/`, `wheels-v*/index/`, `git-v*/`) are deliberately -left untouched: their directory naming does not reliably encode a PEP 503 -package name + exact version the way the pypi/ tree does, so matching them -against uv.lock risks a false match/miss. They are a small fraction of the -cache's total size (see the before/after `du -sh` this script prints). +"""Prune uv's cache down to what uv.lock still pins. + +uv.lock pins one version (or one git commit) per package across the WHOLE +project -- every extra, installed in this build or not. uv's cache is +append-only: `uv sync` downloads what a build needs and never removes a version +an older lockfile pinned. The CI seed persists the cache across builds for as +long as BASE_IMAGE/UV_VERSION/HYBRID_EP_MULTINODE are unchanged (see the +dependency-install RUN in docker/Dockerfile), so every dependency bump would +otherwise leave its predecessor in the seed and in every shipped image. +`uv cache prune` does not help: it removes stale cache-format buckets, cached +environments and unlinked archives, never a valid entry for an older version. + +Removed, matched against uv.lock (layout per uv's crates/uv-cache/src/wheel.rs): + wheels-v*/{pypi,index/}//-[.http|.msgpack] + sdists-v*/{pypi,index/}//[...] + kept iff (, ) is pinned by uv.lock. + sdists-v*/git// + git-v*/checkouts// + kept iff the commit prefixes a git commit uv.lock pins. + archive-v0/ + kept iff any link left under wheels-v*/ or sdists-v*/ resolves to it -- + the same reachability scan uv's own `uv cache prune` uses, so archives + behind entries this script never touches (url/, path/, editable/) stay. + +Never removed: url/, path/, editable/ trees (keyed by URL digest, no version to +match), git-v*/db, and every other bucket. That keeps it safe to run before +shipping UV_CACHE_DIR: it cannot remove anything offline +`uv run --extra ` needs, because that is itself "what uv.lock pins". """ from __future__ import annotations import argparse +import os import re import shutil import sys import tomllib from pathlib import Path -# PEP 503 normalization: case-insensitive, runs of -_. collapse to one '-'. -# uv.lock package names and the cache's on-disk directory names are not -# guaranteed to use the same casing/separator, so both sides must go through -# this before comparing. +# PEP 503: uv.lock and the cache's directory names may differ in case/separator. _NORMALIZE_RE = re.compile(r"[-_.]+") +_GIT_SHA_RE = re.compile(r"#([0-9a-f]{40})$") +_METADATA_SUFFIXES = (".http", ".msgpack", ".lock", ".rev") +# uv's archive-reference scan skips these; they are never links into archive-v0. +_SCAN_SKIP_SUFFIXES = (".lock", ".whl", ".http", ".rev", ".msgpack") def normalize(name: str) -> str: return _NORMALIZE_RE.sub("-", name).lower() -def load_keep_set(lock_path: Path) -> set[tuple[str, str]]: +def load_lock(lock_path: Path) -> tuple[set[tuple[str, str]], set[str]]: + """Returns ({(normalized name, version)}, {full git commit SHA}).""" with lock_path.open("rb") as f: - data = tomllib.load(f) - packages = data.get("package", []) - keep = { - (normalize(p["name"]), p["version"]) - for p in packages - if "name" in p and "version" in p - } - return keep - - -def _entry_name_version(entry: Path) -> tuple[str, str] | None: - # entry is e.g. wheels-v6/pypi//- - # or sdists-v*/pypi// - name = normalize(entry.parent.name) - version = entry.name.split("-", 1)[0] - if not version: - return None - return name, version + packages = tomllib.load(f).get("package", []) + versions = set() + for p in packages: + if "name" in p and "version" in p: + name, version = normalize(p["name"]), p["version"] + versions.add((name, version)) + # uv's WheelFilename::cache_key truncates long versions to 47 chars. + versions.add((name, version[:47].rstrip(".+"))) + shas = set() + for p in packages: + git = p.get("source", {}).get("git") + if git and (m := _GIT_SHA_RE.search(git)): + shas.add(m.group(1)) + return versions, shas + + +def entry_version(entry_name: str) -> str | None: + """`-[.http]` or `` -> ``; None if not a version.""" + for suffix in _METADATA_SUFFIXES: + if entry_name.endswith(suffix): + entry_name = entry_name[: -len(suffix)] + break + version = entry_name.split("-", 1)[0] + # PEP 440 versions (including epochs) start with a digit. Anything else is + # not ours to judge, so it is kept. + return version if version[:1].isdigit() else None + + +def _registry_roots(cache_dir: Path) -> list[Path]: + roots = [] + for bucket in ("wheels-v*", "sdists-v*"): + for bucket_dir in cache_dir.glob(bucket): + roots.extend(p for p in [bucket_dir / "pypi"] if p.is_dir()) + index = bucket_dir / "index" + if index.is_dir(): + roots.extend(p for p in index.iterdir() if p.is_dir()) + return roots + + +def _sha_kept(prefix: str, shas: set[str]) -> bool: + # Unparseable names are kept: only a hex commit prefix can be matched. + if len(prefix) < 7 or not all(c in "0123456789abcdef" for c in prefix): + return True + return any(sha.startswith(prefix) for sha in shas) + + +def plan_removals( + cache_dir: Path, versions: set[tuple[str, str]], shas: set[str] +) -> list[Path]: + removals = [] + for root in _registry_roots(cache_dir): + for name_dir in root.iterdir(): + if not name_dir.is_dir() or name_dir.is_symlink(): + continue + name = normalize(name_dir.name) + for entry in name_dir.iterdir(): + version = entry_version(entry.name) + if version is not None and (name, version) not in versions: + removals.append(entry) + commit_dirs = [ + *cache_dir.glob("sdists-v*/git/*/*"), + *cache_dir.glob("git-v*/checkouts/*/*"), + ] + for commit_dir in commit_dirs: + if commit_dir.is_dir() and not _sha_kept(commit_dir.name, shas): + removals.append(commit_dir) + return removals + + +def _reachable_archives(cache_dir: Path, removed: set[Path]) -> set[str]: + archive_root = (cache_dir / "archive-v0").resolve() + reachable = set() + for bucket in [*cache_dir.glob("wheels-v*"), *cache_dir.glob("sdists-v*")]: + for dirpath, dirnames, filenames in os.walk(bucket): + here = Path(dirpath) + dirnames[:] = [ + d for d in dirnames if d != "src" and here / d not in removed + ] + # os.walk lists directory symlinks under dirnames without following them. + for name in [*filenames, *dirnames]: + link = here / name + if link in removed or name.endswith(_SCAN_SKIP_SUFFIXES): + continue + if link.is_symlink(): + target = link.resolve() + if target.parent == archive_root: + reachable.add(target.name) + return reachable def prune( - cache_dir: Path, keep: set[tuple[str, str]], *, dry_run: bool + cache_dir: Path, + versions: set[tuple[str, str]], + shas: set[str], + *, + dry_run: bool, + verbose: bool = False, ) -> tuple[int, int]: """Returns (entries_removed, archives_removed).""" - reachable: set[str] = set() - entries_removed = 0 - - pypi_roots = [ - p - for pattern in ("wheels-v*/pypi", "sdists-v*/pypi") - for p in cache_dir.glob(pattern) - ] - - for pypi_root in pypi_roots: - for name_dir in pypi_root.iterdir(): - if not name_dir.is_dir(): - continue - for entry in name_dir.iterdir(): - parsed = _entry_name_version(entry) - if parsed is None: - continue - if parsed in keep: - if entry.is_symlink(): - reachable.add(Path(entry.resolve()).name) - continue - entries_removed += 1 - if dry_run: - continue - # Remove the pointer directory/symlink and its sibling - # metadata files (.msgpack, .http). - for sibling in name_dir.glob(f"{entry.name}*"): - if sibling.is_symlink() or sibling.is_file(): - sibling.unlink() - elif sibling.is_dir(): - shutil.rmtree(sibling) - if not dry_run and not any(name_dir.iterdir()): - name_dir.rmdir() - - # Second pass: also mark reachable archives for every KEPT entry we - # didn't already see above (entries outside wheels-v*/sdists-v*/pypi - # that still resolve into archive-v0, if any such mounts exist). - archives_removed = 0 - for archive_root in cache_dir.glob("archive-v0"): - if not archive_root.is_dir(): + removals = plan_removals(cache_dir, versions, shas) + reachable = _reachable_archives(cache_dir, set(removals)) + archive_root = cache_dir / "archive-v0" + orphaned = ( + [a for a in archive_root.iterdir() if a.name not in reachable] + if archive_root.is_dir() + else [] + ) + for path in [*removals, *orphaned]: + if verbose: + print(f"{'would remove' if dry_run else 'remove'}: {path}") + if dry_run: continue - for archive_dir in archive_root.iterdir(): - if archive_dir.name in reachable: - continue - archives_removed += 1 - if not dry_run: - shutil.rmtree(archive_dir, ignore_errors=True) - - return entries_removed, archives_removed + if path.is_symlink() or path.is_file(): + path.unlink() + else: + shutil.rmtree(path) + return len(removals), len(orphaned) def main() -> int: @@ -149,9 +198,11 @@ def main() -> int: action="store_true", help="Report what would be removed without removing it", ) - # Sanity floor: if parsing uv.lock somehow yields a near-empty keep set - # (e.g. a future uv.lock format change this script doesn't understand), - # refuse to prune rather than silently deleting almost everything. + parser.add_argument( + "--verbose", action="store_true", help="Print every removed path" + ) + # Sanity floor: a near-empty keep set (e.g. a uv.lock format change this + # script does not understand) must not turn into deleting the whole cache. parser.add_argument("--min-keep", type=int, default=100) args = parser.parse_args() @@ -162,10 +213,10 @@ def main() -> int: print(f"error: {args.cache_dir} does not exist", file=sys.stderr) return 1 - keep = load_keep_set(args.lock) - if len(keep) < args.min_keep: + versions, shas = load_lock(args.lock) + if len(versions) < args.min_keep: print( - f"error: only parsed {len(keep)} pinned packages from {args.lock} " + f"error: only parsed {len(versions)} pinned packages from {args.lock} " f"(expected at least {args.min_keep}) -- refusing to prune, " "uv.lock's format may have changed in a way this script doesn't understand", file=sys.stderr, @@ -173,12 +224,13 @@ def main() -> int: return 1 entries_removed, archives_removed = prune( - args.cache_dir, keep, dry_run=args.dry_run + args.cache_dir, versions, shas, dry_run=args.dry_run, verbose=args.verbose ) verb = "Would remove" if args.dry_run else "Removed" print( - f"{verb} {entries_removed} stale pypi cache entries and {archives_removed} " - f"orphaned archives (kept {len(keep)} packages pinned in {args.lock})" + f"{verb} {entries_removed} stale cache entries and {archives_removed} " + f"orphaned archives (kept {len(versions)} packages and {len(shas)} git " + f"commits pinned in {args.lock})" ) return 0 diff --git a/docs/docker.md b/docs/docker.md index 6f6a1c537cf..15fa1428b95 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -37,7 +37,7 @@ docker buildx build --build-context nemo-rl=. -f docker/Dockerfile \ The release image ships `uv`'s download cache (`/opt/nemo_rl_cache/uv`) so an unprivileged runtime user can later run `uv run --extra ` fully offline. Because the shared build cache this is seeded from persists across many builds, routine dependency-version bumps (torch, vLLM, SGLang, ...) leave each predecessor version's wheel behind -- `uv` only ever adds to its cache, it never removes a version a prior `uv.lock` pinned and the current one doesn't. -Before the image ships, `docker/prune_uv_cache.py` removes exactly those superseded entries: anything in the cache that isn't a `(package, version)` pair the current `uv.lock` still pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. +Before the image ships, `docker/prune_uv_cache.py` removes exactly those superseded entries: index wheels and source builds whose `(package, version)` the current `uv.lock` no longer pins, and git builds/checkouts for commits it no longer pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. ## Skipping vLLM or SGLang Dependencies diff --git a/tests/unit/tools/test_prune_uv_cache.py b/tests/unit/tools/test_prune_uv_cache.py index 99697aa8c9c..64b47932a70 100644 --- a/tests/unit/tools/test_prune_uv_cache.py +++ b/tests/unit/tools/test_prune_uv_cache.py @@ -13,14 +13,15 @@ # limitations under the License. import importlib.util +import os import sys from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).parents[3] -# docker/ is not a package (no __init__.py, by design -- it mirrors a Docker -# build context, not Python source), so import the script directly by path -# rather than adding docker/ to sys.path. +# docker/ is not a package, so import the script by path. _SPEC = importlib.util.spec_from_file_location( "prune_uv_cache", REPO_ROOT / "docker" / "prune_uv_cache.py" ) @@ -28,6 +29,42 @@ sys.modules["prune_uv_cache"] = prune_uv_cache _SPEC.loader.exec_module(prune_uv_cache) +SHA_KEPT = "17cfb817bccec3a9c247013360cc550c2bac441e" +SHA_STALE = "0123456789abcdef0123456789abcdef01234567" +TAG = "cp313-cp313-linux_x86_64" + + +def _archive(cache: Path, archive_id: str) -> Path: + archive = cache / "archive-v0" / archive_id + archive.mkdir(parents=True, exist_ok=True) + (archive / "payload.txt").write_text(archive_id) + return archive + + +def _link(link: Path, archive: Path) -> None: + # uv writes relative links into archive-v0. + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(os.path.relpath(archive, link.parent), target_is_directory=True) + + +def _wheel(root: Path, name: str, entry: str, archive_id: str) -> None: + cache = root + while not (cache / "archive-v0").exists() and cache.name != "cache": + cache = cache.parent + _link(root / name / entry, _archive(cache, archive_id)) + (root / name / f"{entry}.http").write_text("cache-control") + + +def _names(path: Path) -> set[str]: + return {p.name for p in path.iterdir()} if path.exists() else set() + + +@pytest.fixture +def cache(tmp_path: Path) -> Path: + c = tmp_path / "cache" + (c / "archive-v0").mkdir(parents=True) + return c + def test_normalize_is_pep503() -> None: assert prune_uv_cache.normalize("Foo_Bar.Baz") == "foo-bar-baz" @@ -35,88 +72,182 @@ def test_normalize_is_pep503() -> None: assert prune_uv_cache.normalize("UPPER") == "upper" -def test_load_keep_set_parses_uv_lock(tmp_path: Path) -> None: +def test_load_lock_reads_versions_and_git_commits(tmp_path: Path) -> None: + long_version = "1.0.0+" + "a" * 60 lock = tmp_path / "uv.lock" lock.write_text( - """ + f""" version = 1 [[package]] name = "Orjson" version = "3.12.0" -source = { registry = "https://pypi.org/simple" } +source = {{ registry = "https://pypi.org/simple" }} [[package]] -name = "torch" -version = "2.9.0" -source = { registry = "https://pypi.org/simple" } +name = "deep-ep" +version = "1.2.1" +source = {{ git = "https://github.com/deepseek-ai/DeepEP.git?rev={SHA_KEPT}#{SHA_KEPT}" }} + +[[package]] +name = "long" +version = "{long_version}" +source = {{ registry = "https://pypi.org/simple" }} + +[[package]] +name = "nemo-rl" +source = {{ editable = "." }} """ ) - keep = prune_uv_cache.load_keep_set(lock) - assert keep == {("orjson", "3.12.0"), ("torch", "2.9.0")} + versions, shas = prune_uv_cache.load_lock(lock) + assert ("orjson", "3.12.0") in versions + assert ("deep-ep", "1.2.1") in versions + assert ("long", long_version) in versions + # uv truncates long versions in wheel cache keys; the truncated form is kept too. + assert ("long", long_version[:47]) in versions + assert shas == {SHA_KEPT} + + +@pytest.mark.parametrize( + "entry, expected", + [ + (f"3.12.0-{TAG}", "3.12.0"), + (f"3.12.0-{TAG}.http", "3.12.0"), + (f"3.12.0-{TAG}.msgpack", "3.12.0"), + (f"2.9.0+cu130-{TAG}", "2.9.0+cu130"), + ("1!2.0-py3-none-any", "1!2.0"), + ("1.0.1", "1.0.1"), + ("metadata.msgpack", None), + (".lock", None), + ], +) +def test_entry_version(entry: str, expected: str | None) -> None: + assert prune_uv_cache.entry_version(entry) == expected -def _make_wheel_entry( - cache_dir: Path, name: str, version: str, tag: str, archive_id: str -) -> None: - pypi = cache_dir / "wheels-v6" / "pypi" / name - pypi.mkdir(parents=True, exist_ok=True) - archive = cache_dir / "archive-v0" / archive_id - archive.mkdir(parents=True, exist_ok=True) - (archive / "payload.txt").write_text("real wheel content") - (pypi / f"{version}-{tag}").symlink_to(archive, target_is_directory=True) - (pypi / f"{version}-{tag}.http").write_text("cache-control: ...") - - -def test_prune_keeps_locked_version_drops_stale_ones(tmp_path: Path) -> None: - cache_dir = tmp_path / "uv-cache" - # Three versions of the same package on disk, as if orjson got bumped - # 3.11.3 -> 3.11.9 -> 3.12.0 across three separate dependency-bump PRs - # while BASE_IMAGE/UV_VERSION/HYBRID_EP_MULTINODE stayed the same. - _make_wheel_entry( - cache_dir, "orjson", "3.11.3", "cp313-cp313-linux_x86_64", "archive-old-1" - ) - _make_wheel_entry( - cache_dir, "orjson", "3.11.9", "cp313-cp313-linux_x86_64", "archive-old-2" - ) - _make_wheel_entry( - cache_dir, "orjson", "3.12.0", "cp313-cp313-linux_x86_64", "archive-kept" - ) +def test_prunes_stale_pypi_wheels_and_their_archives(cache: Path) -> None: + pypi = cache / "wheels-v6" / "pypi" + _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old-1") + _wheel(pypi, "orjson", f"3.11.9-{TAG}", "old-2") + _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") - keep = {("orjson", "3.12.0")} - entries_removed, archives_removed = prune_uv_cache.prune( - cache_dir, keep, dry_run=False + removed, archives = prune_uv_cache.prune( + cache, {("orjson", "3.12.0")}, set(), dry_run=False ) - assert entries_removed == 4 # 2 stale versions x (symlink + .http sibling) - assert archives_removed == 2 # the 2 stale archives, not the kept one + assert (removed, archives) == (4, 2) + assert _names(pypi / "orjson") == {f"3.12.0-{TAG}", f"3.12.0-{TAG}.http"} + assert _names(cache / "archive-v0") == {"kept"} + assert (cache / "archive-v0" / "kept" / "payload.txt").read_text() == "kept" + + +def test_does_not_confuse_version_prefixes(cache: Path) -> None: + # Removing 1.0 must not touch 1.0.1's files (a prefix glob would). + pypi = cache / "wheels-v6" / "pypi" + _wheel(pypi, "pkg", "1.0-py3-none-any", "stale") + _wheel(pypi, "pkg", "1.0.1-py3-none-any", "kept") + + prune_uv_cache.prune(cache, {("pkg", "1.0.1")}, set(), dry_run=False) - pypi_dir = cache_dir / "wheels-v6" / "pypi" / "orjson" - remaining = {p.name for p in pypi_dir.iterdir()} - assert remaining == { - "3.12.0-cp313-cp313-linux_x86_64", - "3.12.0-cp313-cp313-linux_x86_64.http", + assert _names(pypi / "pkg") == { + "1.0.1-py3-none-any", + "1.0.1-py3-none-any.http", } + assert _names(cache / "archive-v0") == {"kept"} - archive_dir = cache_dir / "archive-v0" - assert {p.name for p in archive_dir.iterdir()} == {"archive-kept"} - assert ( - archive_dir / "archive-kept" / "payload.txt" - ).read_text() == "real wheel content" +def test_prunes_alternative_index_trees(cache: Path) -> None: + # torch & co. come from download.pytorch.org: wheels-v6/index//... + index = cache / "wheels-v6" / "index" / "d1g3st" + _wheel(index, "torch", f"2.8.0+cu130-{TAG}", "torch-old") + _wheel(index, "torch", f"2.9.0+cu130-{TAG}", "torch-kept") -def test_prune_dry_run_reports_without_removing(tmp_path: Path) -> None: - cache_dir = tmp_path / "uv-cache" - _make_wheel_entry( - cache_dir, "orjson", "3.11.3", "cp313-cp313-linux_x86_64", "archive-old" - ) + prune_uv_cache.prune(cache, {("torch", "2.9.0+cu130")}, set(), dry_run=False) + + assert _names(index / "torch") == { + f"2.9.0+cu130-{TAG}", + f"2.9.0+cu130-{TAG}.http", + } + assert _names(cache / "archive-v0") == {"torch-kept"} + + +def test_keeps_archives_referenced_by_untouched_trees(cache: Path) -> None: + # url/ and editable/ entries carry no version to match, so they are left + # alone -- and so must the archives they point at. + wheels = cache / "wheels-v6" + _link(wheels / "url" / "abc" / "vllm" / f"0.29.0-{TAG}", _archive(cache, "url")) + _link(wheels / "editable" / "def" / "proj" / "0.1.0", _archive(cache, "edit")) + + removed, archives = prune_uv_cache.prune(cache, set(), set(), dry_run=False) + + assert (removed, archives) == (0, 0) + assert _names(cache / "archive-v0") == {"url", "edit"} + + +def test_prunes_stale_sdist_versions_keeping_built_wheel_archives( + cache: Path, +) -> None: + sdists = cache / "sdists-v9" / "pypi" / "flash-attn" + _link(sdists / "2.7.0" / "rev1" / "wheel", _archive(cache, "built-old")) + _link(sdists / "2.8.3" / "rev1" / "wheel", _archive(cache, "built-kept")) + (sdists / "2.8.3" / "rev1" / "src").mkdir() + + prune_uv_cache.prune(cache, {("flash-attn", "2.8.3")}, set(), dry_run=False) + + assert _names(sdists) == {"2.8.3"} + assert _names(cache / "archive-v0") == {"built-kept"} + + +def test_prunes_git_builds_and_checkouts_by_commit(cache: Path) -> None: + git_builds = cache / "sdists-v9" / "git" / "repo" + _link(git_builds / SHA_KEPT[:16] / "rev" / "wheel", _archive(cache, "git-kept")) + _link(git_builds / SHA_STALE[:16] / "rev" / "wheel", _archive(cache, "git-old")) + checkouts = cache / "git-v0" / "checkouts" / "repo" + (checkouts / SHA_KEPT[:7]).mkdir(parents=True) + (checkouts / SHA_STALE[:7]).mkdir() + (checkouts / "not-a-sha").mkdir() + (cache / "git-v0" / "db" / "repo").mkdir(parents=True) - entries_removed, archives_removed = prune_uv_cache.prune( - cache_dir, keep=set(), dry_run=True + removed, archives = prune_uv_cache.prune(cache, set(), {SHA_KEPT}, dry_run=False) + + assert (removed, archives) == (2, 1) + assert _names(git_builds) == {SHA_KEPT[:16]} + assert _names(checkouts) == {SHA_KEPT[:7], "not-a-sha"} + assert (cache / "git-v0" / "db" / "repo").is_dir() + assert _names(cache / "archive-v0") == {"git-kept"} + + +def test_dry_run_reports_same_counts_without_removing(cache: Path) -> None: + pypi = cache / "wheels-v6" / "pypi" + _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old") + _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") + keep = {("orjson", "3.12.0")} + + assert prune_uv_cache.prune(cache, keep, set(), dry_run=True) == (2, 1) + assert len(_names(pypi / "orjson")) == 4 + assert _names(cache / "archive-v0") == {"old", "kept"} + assert prune_uv_cache.prune(cache, keep, set(), dry_run=False) == (2, 1) + + +def test_second_run_is_a_no_op(cache: Path) -> None: + pypi = cache / "wheels-v6" / "pypi" + _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old") + _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") + keep = {("orjson", "3.12.0")} + + prune_uv_cache.prune(cache, keep, set(), dry_run=False) + assert prune_uv_cache.prune(cache, keep, set(), dry_run=False) == (0, 0) + + +def test_main_refuses_a_suspiciously_small_lock( + cache: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _wheel(cache / "wheels-v6" / "pypi", "orjson", f"3.11.3-{TAG}", "old") + lock = tmp_path / "uv.lock" + lock.write_text('version = 1\n[[package]]\nname = "a"\nversion = "1"\n') + monkeypatch.setattr( + sys, "argv", ["prune", "--lock", str(lock), "--cache-dir", str(cache)] ) - assert entries_removed == 2 - assert archives_removed == 1 - # Nothing actually removed in dry-run mode. - assert (cache_dir / "wheels-v6" / "pypi" / "orjson").exists() - assert (cache_dir / "archive-v0" / "archive-old").exists() + assert prune_uv_cache.main() == 1 + assert _names(cache / "archive-v0") == {"old"} From a6378c128218c61b94ac0b6086dbccb31d0141f6 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Fri, 23 Oct 2026 11:12:59 -0500 Subject: [PATCH 4/8] fix(docker): prune the uv cache in the layers that write it Deleting cache files in a later RUN only hides them behind whiteouts; the layers that wrote them still ship every byte. The release-stage prune therefore left the image size unchanged (61 vs 60 layers, the extra one ~0 bytes). - Prune at the end of the dependency RUN, which seeds and fills the cache, so superseded versions never get committed to a layer. - Install Dynamo with a throwaway UV_CACHE_DIR: it syncs its own Python 3.12 lock into a --link-mode copy venv that nothing re-syncs at runtime, so its wheels were stored twice. - Drop the release-stage prune RUN, and add a test that pins the prune to the dependency RUN and the Dynamo install to a throwaway cache. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- docker/Dockerfile | 28 ++++++++++++------------- docs/docker.md | 2 +- tests/unit/tools/test_prune_uv_cache.py | 27 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4a8a739cc97..01c3b45c844 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -243,6 +243,7 @@ COPY --from=nemo-rl nemo_rl/__init__.py nemo_rl/package_info.py ./nemo_rl/ COPY --from=nemo-rl nemo_rl/distributed/actor_environments.py ./nemo_rl/distributed/ COPY --from=nemo-rl tools/build-custom-vllm.sh ./tools/build-custom-vllm.sh COPY --from=nemo-rl tools/build-custom-flashinfer.sh ./tools/build-custom-flashinfer.sh +COPY --from=nemo-rl docker/prune_uv_cache.py ./docker/prune_uv_cache.py COPY --from=nemo-rl --link research/ ./research/ COPY --from=nemo-rl --link \ --exclude=TensorRT-LLM-workspace \ @@ -354,6 +355,13 @@ find "${UV_CACHE_DIR}" -type d \ \( -path "*ray/_private/runtime_env/agent/thirdparty_files/aiohttp*" \ -o -path "*ray/_private/runtime_env/agent/thirdparty_files/yarl*" \) -exec rm -rf {} + +# Drop cache entries uv.lock no longer pins -- versions the seed carried in from older locks +# (see docker/prune_uv_cache.py). It has to run here, in the layer that writes them: deleting +# them in a later layer only hides them, and the image still ships every byte. +du -sh "${UV_CACHE_DIR}" +"${UV_PROJECT_ENVIRONMENT}/bin/python" docker/prune_uv_cache.py --lock uv.lock --cache-dir "${UV_CACHE_DIR}" +du -sh "${UV_CACHE_DIR}" + echo "${CACHE_KEY}" > "${UV_CACHE_DIR}/.cache-key" # Hand the cache and the venvs built from it to the runtime UID in the layer that creates them. @@ -630,7 +638,11 @@ if [[ "${BUILD_DYNAMO:-0}" != "1" ]]; then echo "BUILD_DYNAMO is not 1; skipping the managed Dynamo runtime" exit 0 fi -bash docker/dynamo/install.sh +# Dynamo syncs its own lock (Python 3.12) into a --link-mode copy venv that nothing re-syncs at +# runtime, so its wheels are never needed from the shared cache. A throwaway cache keeps them +# out of this layer instead of shipping them twice. +UV_CACHE_DIR=/tmp/dynamo-uv-cache bash docker/dynamo/install.sh +rm -rf /tmp/dynamo-uv-cache EOF # Unshallow the repo to get the full history (in the case it was from the scratch layer). # Potentially not necessary if the repo is passed in as a complete repository (w/ full git history), @@ -691,20 +703,6 @@ if [[ -n "${NEMO_GYM_PREFETCH_CONFIGS:-}" ]]; then fi EOF -# Drop uv cache entries no longer pinned by uv.lock -- see docker/prune_uv_cache.py's -# module docstring for the full accumulation mechanism this addresses (every -# dependency-version bump leaves its predecessor's wheel behind forever, because the -# uv cache seed's own staleness key is BASE_IMAGE+UV_VERSION+HYBRID_EP_MULTINODE, not -# the lock content -- see the dependency-install RUN above). Must run after every -# uv-writing RUN above (dependency-install, TRT-LLM, Dynamo, worker-venv prefetch, -# Gym prefetch) and before the ownership handoff below, so it can only ever remove -# entries nothing in THIS build still needs -- it keeps every (name, version) pair -# uv.lock pins across every extra, installed in this build or not, so it cannot break -# the offline `uv run --extra ` promise that UV_CACHE_DIR exists for. -RUN echo "uv cache size before pruning:" && du -sh "${UV_CACHE_DIR}" && \ - python docker/prune_uv_cache.py --lock uv.lock --cache-dir "${UV_CACHE_DIR}" && \ - echo "uv cache size after pruning:" && du -sh "${UV_CACHE_DIR}" - # ray's bundled Java worker jar ships an outdated httpcore5 with a known CVE # (GHSA-hf6x-8p5f-cgmf). NeMo-RL never uses Ray's Java cross-language worker, # and removing the jar doesn't affect ray.init(), remote tasks, actors, or diff --git a/docs/docker.md b/docs/docker.md index 15fa1428b95..e4b0cc3d6f7 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -37,7 +37,7 @@ docker buildx build --build-context nemo-rl=. -f docker/Dockerfile \ The release image ships `uv`'s download cache (`/opt/nemo_rl_cache/uv`) so an unprivileged runtime user can later run `uv run --extra ` fully offline. Because the shared build cache this is seeded from persists across many builds, routine dependency-version bumps (torch, vLLM, SGLang, ...) leave each predecessor version's wheel behind -- `uv` only ever adds to its cache, it never removes a version a prior `uv.lock` pinned and the current one doesn't. -Before the image ships, `docker/prune_uv_cache.py` removes exactly those superseded entries: index wheels and source builds whose `(package, version)` the current `uv.lock` no longer pins, and git builds/checkouts for commits it no longer pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. +At the end of the dependency layer that fills the cache, `docker/prune_uv_cache.py` removes exactly those superseded entries (it must run in that same layer: deleting files in a later layer only hides them, and the image still ships them): index wheels and source builds whose `(package, version)` the current `uv.lock` no longer pins, and git builds/checkouts for commits it no longer pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. ## Skipping vLLM or SGLang Dependencies diff --git a/tests/unit/tools/test_prune_uv_cache.py b/tests/unit/tools/test_prune_uv_cache.py index 64b47932a70..0e3fb02cd4d 100644 --- a/tests/unit/tools/test_prune_uv_cache.py +++ b/tests/unit/tools/test_prune_uv_cache.py @@ -239,6 +239,33 @@ def test_second_run_is_a_no_op(cache: Path) -> None: assert prune_uv_cache.prune(cache, keep, set(), dry_run=False) == (0, 0) +def test_dockerfile_prunes_in_the_layers_that_fill_the_cache() -> None: + # Deleting cache files in a later layer only hides them; the image still ships + # them. So the prune must run inside the dependency RUN that seeds and fills + # the cache, and Dynamo's wheels must never land in the shared cache. + lines = (REPO_ROOT / "docker" / "Dockerfile").read_text().splitlines() + code = [(i, line) for i, line in enumerate(lines) if not line.startswith("#")] + + copy_script = next( + i for i, l in code if l.startswith("COPY") and "prune_uv_cache.py" in l + ) + seed_run = next(i for i, l in code if "from=uv-cache-seed" in l) + run_end = next(i for i, l in code if i > seed_run and l == "EOF") + prefetch_end = lines.index("done < /opt/actor_venvs.tsv") + cache_key = next(i for i, l in code if '.cache-key"' in l and l.startswith("echo")) + prunes = [ + i for i, l in code if "prune_uv_cache.py" in l and not l.startswith("COPY") + ] + + assert copy_script < seed_run + assert len(prunes) == 1, "prune must run exactly once, in the dependency RUN" + assert seed_run < prefetch_end < prunes[0] < cache_key < run_end + + dynamo = next(i for i, l in code if "docker/dynamo/install.sh" in l and "bash" in l) + assert lines[dynamo].startswith("UV_CACHE_DIR=/tmp/") + assert lines[dynamo + 1].startswith("rm -rf /tmp/") + + def test_main_refuses_a_suspiciously_small_lock( cache: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 0d8214ebffc87f807e8fcb203724b6303378ce58 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Fri, 23 Oct 2026 15:25:32 -0500 Subject: [PATCH 5/8] fix(docker): prune only seed content, before the dependency syncs Pruning at the end of the dependency RUN also deleted the build-isolation deps the syncs resolve outside uv.lock (setuptools, setuptools-scm, poetry-core, a build-time torch 2.14.1 + CUDA wheels, ...), which the TRT-LLM step then downloaded again. Run the prune right after the seed is copied, so it only judges what older locks left behind. Also, from review: - copy the script from the recipe's context, so building an older NRL_GIT_REF that lacks it still works - compare sdist version directories verbatim instead of stripping a metadata suffix from the version - keep archives of a package named `src` - type-check the script with pyrefly Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- docker/Dockerfile | 21 +++++++---- docker/prune_uv_cache.py | 21 ++++++++++- docs/docker.md | 2 +- pyrefly.toml | 1 + tests/unit/tools/test_prune_uv_cache.py | 49 ++++++++++++++++++++++--- 5 files changed, 77 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 01c3b45c844..ea2f94ab717 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -243,7 +243,8 @@ COPY --from=nemo-rl nemo_rl/__init__.py nemo_rl/package_info.py ./nemo_rl/ COPY --from=nemo-rl nemo_rl/distributed/actor_environments.py ./nemo_rl/distributed/ COPY --from=nemo-rl tools/build-custom-vllm.sh ./tools/build-custom-vllm.sh COPY --from=nemo-rl tools/build-custom-flashinfer.sh ./tools/build-custom-flashinfer.sh -COPY --from=nemo-rl docker/prune_uv_cache.py ./docker/prune_uv_cache.py +# Part of this recipe, not of the selected source: an older NRL_GIT_REF has no such script. +COPY docker/prune_uv_cache.py /usr/local/bin/prune_uv_cache.py COPY --from=nemo-rl --link research/ ./research/ COPY --from=nemo-rl --link \ --exclude=TensorRT-LLM-workspace \ @@ -260,9 +261,20 @@ elif [ -n "$SEED_KEY" ]; then echo "Seeding uv cache (key=${CACHE_KEY})" mkdir -p "${UV_CACHE_DIR}" rsync -a --exclude='.cache-key' /tmp/uv-cache-seed/ "${UV_CACHE_DIR}/" 2>/dev/null || true + SEEDED=1 fi uv venv --seed +# Drop what the seed carried in that uv.lock no longer pins -- versions from older locks +# (see docker/prune_uv_cache.py). It runs before any sync, so it only ever judges seed +# content: build-isolation deps the syncs below resolve outside uv.lock (setuptools, a +# build-time torch, ...) are kept. And it runs in this RUN because deleting files in a +# later layer only hides them; the image would still ship every byte. +if [[ -n "${SEEDED:-}" ]]; then + du -sh "${UV_CACHE_DIR}" + "${UV_PROJECT_ENVIRONMENT}/bin/python" /usr/local/bin/prune_uv_cache.py --lock uv.lock --cache-dir "${UV_CACHE_DIR}" + du -sh "${UV_CACHE_DIR}" +fi # The custom build scripts will alter the pyproject.toml and uv.lock if [[ -n "${BUILD_CUSTOM_VLLM:-}" ]]; then UV_LINK_MODE=hardlink bash tools/build-custom-vllm.sh ${BUILD_CUSTOM_VLLM_URL:-} ${BUILD_CUSTOM_VLLM_REF:-} ${BUILD_CUSTOM_VLLM_PRECOMPILED_WHEEL_LOCATION:-} @@ -355,13 +367,6 @@ find "${UV_CACHE_DIR}" -type d \ \( -path "*ray/_private/runtime_env/agent/thirdparty_files/aiohttp*" \ -o -path "*ray/_private/runtime_env/agent/thirdparty_files/yarl*" \) -exec rm -rf {} + -# Drop cache entries uv.lock no longer pins -- versions the seed carried in from older locks -# (see docker/prune_uv_cache.py). It has to run here, in the layer that writes them: deleting -# them in a later layer only hides them, and the image still ships every byte. -du -sh "${UV_CACHE_DIR}" -"${UV_PROJECT_ENVIRONMENT}/bin/python" docker/prune_uv_cache.py --lock uv.lock --cache-dir "${UV_CACHE_DIR}" -du -sh "${UV_CACHE_DIR}" - echo "${CACHE_KEY}" > "${UV_CACHE_DIR}/.cache-key" # Hand the cache and the venvs built from it to the runtime UID in the layer that creates them. diff --git a/docker/prune_uv_cache.py b/docker/prune_uv_cache.py index db665161222..ab6a27ad796 100644 --- a/docker/prune_uv_cache.py +++ b/docker/prune_uv_cache.py @@ -124,7 +124,14 @@ def plan_removals( continue name = normalize(name_dir.name) for entry in name_dir.iterdir(): - version = entry_version(entry.name) + if root.relative_to(cache_dir).parts[0].startswith("sdists-v") and ( + entry.is_dir() and not entry.is_symlink() + ): + # An sdist version directory is the bare version, compared verbatim: + # stripping a metadata suffix would change a version like `1.0+foo.http`. + version = entry.name if entry.name[:1].isdigit() else None + else: + version = entry_version(entry.name) if version is not None and (name, version) not in versions: removals.append(entry) commit_dirs = [ @@ -139,12 +146,22 @@ def plan_removals( def _reachable_archives(cache_dir: Path, removed: set[Path]) -> set[str]: archive_root = (cache_dir / "archive-v0").resolve() + registry_roots = set(_registry_roots(cache_dir)) reachable = set() for bucket in [*cache_dir.glob("wheels-v*"), *cache_dir.glob("sdists-v*")]: for dirpath, dirnames, filenames in os.walk(bucket): here = Path(dirpath) + # Skip the unpacked source trees inside sdist builds (`/src`), but not + # a package that happens to be named `src`. dirnames[:] = [ - d for d in dirnames if d != "src" and here / d not in removed + d + for d in dirnames + if here / d not in removed + and ( + d != "src" + or bucket.name.startswith("wheels-v") + or here in registry_roots + ) ] # os.walk lists directory symlinks under dirnames without following them. for name in [*filenames, *dirnames]: diff --git a/docs/docker.md b/docs/docker.md index e4b0cc3d6f7..59cbfa4bae4 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -37,7 +37,7 @@ docker buildx build --build-context nemo-rl=. -f docker/Dockerfile \ The release image ships `uv`'s download cache (`/opt/nemo_rl_cache/uv`) so an unprivileged runtime user can later run `uv run --extra ` fully offline. Because the shared build cache this is seeded from persists across many builds, routine dependency-version bumps (torch, vLLM, SGLang, ...) leave each predecessor version's wheel behind -- `uv` only ever adds to its cache, it never removes a version a prior `uv.lock` pinned and the current one doesn't. -At the end of the dependency layer that fills the cache, `docker/prune_uv_cache.py` removes exactly those superseded entries (it must run in that same layer: deleting files in a later layer only hides them, and the image still ships them): index wheels and source builds whose `(package, version)` the current `uv.lock` no longer pins, and git builds/checkouts for commits it no longer pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. +Right after the seed is copied in, before any `uv sync`, `docker/prune_uv_cache.py` removes exactly those superseded entries. Running first means it only judges what the seed carried in, so the build-isolation dependencies the syncs resolve outside `uv.lock` (setuptools, a build-time torch, ...) are kept. It runs in the same layer the syncs fill, because deleting files in a later layer only hides them and the image still ships them. It removes index wheels and source builds whose `(package, version)` the current `uv.lock` no longer pins, and git builds/checkouts for commits it no longer pins, across every extra (installed in this build or not). It never removes a wheel the offline-install promise depends on, since that promise is itself "every wheel `uv.lock` could need." This runs automatically on every build; there is no flag to disable it. ## Skipping vLLM or SGLang Dependencies diff --git a/pyrefly.toml b/pyrefly.toml index ca658115f82..d2285e1f3b6 100644 --- a/pyrefly.toml +++ b/pyrefly.toml @@ -25,6 +25,7 @@ replace-imports-with-any = [ "zstandard.*", ] project-includes = [ + "docker/prune_uv_cache.py", "nemo_rl/data/captured_media.py", "nemo_rl/experience/reward_penalties.py", # TODO: enable these once we have 100 correctness diff --git a/tests/unit/tools/test_prune_uv_cache.py b/tests/unit/tools/test_prune_uv_cache.py index 0e3fb02cd4d..709af1a611a 100644 --- a/tests/unit/tools/test_prune_uv_cache.py +++ b/tests/unit/tools/test_prune_uv_cache.py @@ -239,10 +239,11 @@ def test_second_run_is_a_no_op(cache: Path) -> None: assert prune_uv_cache.prune(cache, keep, set(), dry_run=False) == (0, 0) -def test_dockerfile_prunes_in_the_layers_that_fill_the_cache() -> None: +def test_dockerfile_prunes_the_seed_before_any_sync() -> None: # Deleting cache files in a later layer only hides them; the image still ships - # them. So the prune must run inside the dependency RUN that seeds and fills - # the cache, and Dynamo's wheels must never land in the shared cache. + # them. So the prune runs inside the dependency RUN -- and before its first sync, + # so it only judges seed content and never deletes the build-isolation deps + # (setuptools, a build-time torch, ...) the syncs resolve outside uv.lock. lines = (REPO_ROOT / "docker" / "Dockerfile").read_text().splitlines() code = [(i, line) for i, line in enumerate(lines) if not line.startswith("#")] @@ -251,21 +252,57 @@ def test_dockerfile_prunes_in_the_layers_that_fill_the_cache() -> None: ) seed_run = next(i for i, l in code if "from=uv-cache-seed" in l) run_end = next(i for i, l in code if i > seed_run and l == "EOF") - prefetch_end = lines.index("done < /opt/actor_venvs.tsv") - cache_key = next(i for i, l in code if '.cache-key"' in l and l.startswith("echo")) + seed_copy = next(i for i, l in code if "/tmp/uv-cache-seed/" in l and "rsync" in l) + venv = next(i for i, l in code if i > seed_run and l == "uv venv --seed") + first_sync = next(i for i, l in code if i > seed_run and "uv sync" in l) prunes = [ i for i, l in code if "prune_uv_cache.py" in l and not l.startswith("COPY") ] + # The script comes from the recipe's own context: an older NRL_GIT_REF lacks it. + assert "--from=" not in lines[copy_script] assert copy_script < seed_run assert len(prunes) == 1, "prune must run exactly once, in the dependency RUN" - assert seed_run < prefetch_end < prunes[0] < cache_key < run_end + assert seed_run < seed_copy < venv < prunes[0] < first_sync < run_end dynamo = next(i for i, l in code if "docker/dynamo/install.sh" in l and "bash" in l) assert lines[dynamo].startswith("UV_CACHE_DIR=/tmp/") assert lines[dynamo + 1].startswith("rm -rf /tmp/") +@pytest.mark.parametrize("suffix", [".http", ".msgpack", ".lock", ".rev"]) +def test_compares_sdist_version_directories_verbatim(cache: Path, suffix: str) -> None: + version = f"1.0+foo{suffix}" + sdists = cache / "sdists-v9" / "pypi" / "pkg" + _link(sdists / version / "rev" / "wheel", _archive(cache, "built")) + + prune_uv_cache.prune(cache, {("pkg", version)}, set(), dry_run=False) + assert _names(sdists) == {version} + assert _names(cache / "archive-v0") == {"built"} + + # Pinning only the shorter version must not keep the longer one. + prune_uv_cache.prune(cache, {("pkg", "1.0+foo")}, set(), dry_run=False) + assert _names(sdists) == set() + assert _names(cache / "archive-v0") == set() + + +def test_keeps_archives_of_a_package_named_src(cache: Path) -> None: + pypi = cache / "wheels-v6" / "pypi" + _wheel(pypi, "src", "1.0-py3-none-any", "wheel-kept") + _wheel(pypi, "src", "0.9-py3-none-any", "wheel-old") + sdists = cache / "sdists-v9" / "pypi" + _link(sdists / "src" / "1.0" / "rev" / "wheel", _archive(cache, "sdist-kept")) + _link(sdists / "src" / "0.9" / "rev" / "wheel", _archive(cache, "sdist-old")) + # A link inside an unpacked source tree does not keep its target alive. + _link(sdists / "src" / "1.0" / "rev" / "src" / "x", _archive(cache, "in-src")) + + prune_uv_cache.prune(cache, {("src", "1.0")}, set(), dry_run=False) + + assert _names(pypi / "src") == {"1.0-py3-none-any", "1.0-py3-none-any.http"} + assert _names(sdists / "src") == {"1.0"} + assert _names(cache / "archive-v0") == {"wheel-kept", "sdist-kept"} + + def test_main_refuses_a_suspiciously_small_lock( cache: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From b70c46f3625cb9ef891c3504754c17c40446c246 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Sun, 1 Nov 2026 18:52:29 -0600 Subject: [PATCH 6/8] build(docker): stop cleaning the deep-ep uv cache entry The uv cache mount id and the seed tag are now keyed by HYBRID_EP_MULTINODE, and deep_ep's match-runtime torch pin is part of uv's built-wheel cache key, so a cached DeepEP build can no longer be reused for the wrong multinode setting or torch ABI. The clean only forced a rebuild on every build. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- docker/Dockerfile | 1 - docker/uv_cache_mount.sh | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6bf94103abe..f6f15b2ac1e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -334,7 +334,6 @@ apt-get update # DeepEP's multinode link needs the exact filename libnvidia-ml.so.1; the CUDA base # only ships stubs/libnvidia-ml.so. Purged again after the syncs below. apt-get install -y --no-install-recommends libnvidia-ml-dev -uv cache clean deep-ep # Everything is hardlinked against the uv cache; this layer holds exactly one real copy of every wheel. UV_LINK_MODE=hardlink uv sync --frozen --no-install-project if [[ -z "${SKIP_VLLM_BUILD:-}" ]]; then diff --git a/docker/uv_cache_mount.sh b/docker/uv_cache_mount.sh index cb4ad9eb1b2..4601519bc7a 100644 --- a/docker/uv_cache_mount.sh +++ b/docker/uv_cache_mount.sh @@ -43,8 +43,8 @@ uv_cache_restore() { # Sync new downloads back to the durable store and, if STRIP_UV_CACHE is set, # delete the materialized copy before this RUN's layer commits. Pass "prune" -# only on the first uv-writing RUN in a stage, so cleanup performed there (uv -# cache clean, the aiohttp/yarl removal) is reflected in the store. Later +# only on the first uv-writing RUN in a stage, so cleanup performed there (the +# lockfile prune, the aiohttp/yarl removal) is reflected in the store. Later # RUNs must NOT prune: the store can hold entries this RUN never copied in, # and --delete there would discard them from the next build's seed. # From e8e33385bb43f26f19fc5e7b4570b96280a0aa22 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Mon, 9 Nov 2026 11:27:45 -0600 Subject: [PATCH 7/8] build(docker): bump uv to 0.13.6 uv 0.13 keeps git and no-build-isolation wheels (deep-ep, deep-gemm, mamba-ssm, ...) through `uv cache prune --ci`, so the exported seed no longer forces those source builds on every rebuild. With `preview = true`, 0.13 also enables the content-addressed cache: archive files are hardlinks into files-v0/. Adapt to it: - prune_uv_cache.py drops files-v0 objects nothing links to any more (link count 1, uv's own rule) and the new .ok checkout markers; otherwise removing an archive frees nothing. - rsync the cache with -H so hardlinks survive the restore/persist round trip instead of doubling the cache. Tests move to the 0.13 bucket layout, cover the files-v0 cleanup, and allowlist the fake commit SHAs for the secrets detector. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- docker/Dockerfile | 4 +- docker/prune_uv_cache.py | 33 +++++++++- docker/uv_cache_mount.sh | 6 +- tests/unit/tools/test_prune_uv_cache.py | 82 +++++++++++++++++-------- 4 files changed, 95 insertions(+), 30 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index f6f15b2ac1e..bfc2ec511d7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -159,7 +159,7 @@ RUN curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.ru # Install uv and python outside /root so an unprivileged runtime UID (for example # 65532) can execute them: /root is not traversable by other users, and the project # venv's python is a symlink into uv's managed-Python directory. -ARG UV_VERSION=0.11.28 +ARG UV_VERSION=0.13.6 ARG PYTHON_VERSION=3.13.14 ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python # One uv cache for the build and for every runtime identity. It holds every wheel the venvs are @@ -870,7 +870,7 @@ echo "Exporting uv cache for build ${UV_CACHE_EXPORT_NONCE:-local}" # RUN command, leaving this builder-local mount empty. Restore the seed only in # that case; unconditionally merging it would resurrect wheels uv evicted. if [[ -z "$(find /root/.uv-cache-store -mindepth 1 -print -quit)" ]]; then - rsync -a /tmp/uv-cache-seed/ /root/.uv-cache-store/ + rsync -aH /tmp/uv-cache-seed/ /root/.uv-cache-store/ fi mkdir -p /tmp/uv-cache-export cp -a /root/.uv-cache-store/. /tmp/uv-cache-export/ diff --git a/docker/prune_uv_cache.py b/docker/prune_uv_cache.py index ab6a27ad796..c448937d7c0 100644 --- a/docker/prune_uv_cache.py +++ b/docker/prune_uv_cache.py @@ -30,12 +30,15 @@ sdists-v*/{pypi,index/}//[...] kept iff (, ) is pinned by uv.lock. sdists-v*/git// - git-v*/checkouts// + git-v*/checkouts//[.ok] kept iff the commit prefixes a git commit uv.lock pins. archive-v0/ kept iff any link left under wheels-v*/ or sdists-v*/ resolves to it -- the same reachability scan uv's own `uv cache prune` uses, so archives behind entries this script never touches (url/, path/, editable/) stay. + files-v*// + kept iff something still hardlinks it (link count > 1), as uv's own + `uv cache prune` decides. Never removed: url/, path/, editable/ trees (keyed by URL digest, no version to match), git-v*/db, and every other bucket. That keeps it safe to run before @@ -141,6 +144,10 @@ def plan_removals( for commit_dir in commit_dirs: if commit_dir.is_dir() and not _sha_kept(commit_dir.name, shas): removals.append(commit_dir) + # uv >= 0.12 marks a finished checkout with a sibling `.ok` file. + ready = commit_dir.with_name(f"{commit_dir.name}.ok") + if ready.is_file(): + removals.append(ready) return removals @@ -175,6 +182,28 @@ def _reachable_archives(cache_dir: Path, removed: set[Path]) -> set[str]: return reachable +def _remove_unlinked_files(cache_dir: Path, *, verbose: bool) -> int: + """Drop content-addressed file objects no archive hardlinks any more. + + With uv's `content-addressed-cache` preview feature (on under `preview = true`), + every file in archive-v0// is a hardlink to files-v0//, so + removing an archive frees nothing until its objects go too. An object with a + link count of 1 has no archive (and no hardlinked venv) left -- the same rule + as uv's own `uv cache prune` (Cache::prune_archive_files). + """ + removed = 0 + for obj in cache_dir.glob("files-v*/*/*"): + if not obj.is_symlink() and obj.is_file() and obj.stat().st_nlink == 1: + if verbose: + print(f"remove: {obj}") + obj.unlink() + removed += 1 + for shard in cache_dir.glob("files-v*/*"): + if shard.is_dir() and not shard.is_symlink() and not any(shard.iterdir()): + shard.rmdir() + return removed + + def prune( cache_dir: Path, versions: set[tuple[str, str]], @@ -201,6 +230,8 @@ def prune( path.unlink() else: shutil.rmtree(path) + if not dry_run: + _remove_unlinked_files(cache_dir, verbose=verbose) return len(removals), len(orphaned) diff --git a/docker/uv_cache_mount.sh b/docker/uv_cache_mount.sh index 4601519bc7a..c151298bee4 100644 --- a/docker/uv_cache_mount.sh +++ b/docker/uv_cache_mount.sh @@ -37,7 +37,7 @@ uv_cache_restore() { if [ "${1:-}" = "force" ] || [ ! -d "${UV_CACHE_DIR}" ]; then mkdir -p "${UV_CACHE_DIR}" - rsync -a /root/.uv-cache-store/ "${UV_CACHE_DIR}/" + rsync -aH /root/.uv-cache-store/ "${UV_CACHE_DIR}/" fi } @@ -54,9 +54,9 @@ uv_cache_restore() { # should degrade the next build's cache hit rate, not fail this build. uv_cache_persist() { if [ "${1:-}" = "prune" ]; then - rsync -a --delete "${UV_CACHE_DIR}/" /root/.uv-cache-store/ 2>/dev/null || true + rsync -aH --delete "${UV_CACHE_DIR}/" /root/.uv-cache-store/ 2>/dev/null || true else - rsync -a "${UV_CACHE_DIR}/" /root/.uv-cache-store/ 2>/dev/null || true + rsync -aH "${UV_CACHE_DIR}/" /root/.uv-cache-store/ 2>/dev/null || true fi if [ -n "${STRIP_UV_CACHE:-}" ]; then rm -rf "${UV_CACHE_DIR}"; fi } diff --git a/tests/unit/tools/test_prune_uv_cache.py b/tests/unit/tools/test_prune_uv_cache.py index eb5aa7263f3..9bf1559d099 100644 --- a/tests/unit/tools/test_prune_uv_cache.py +++ b/tests/unit/tools/test_prune_uv_cache.py @@ -29,8 +29,8 @@ sys.modules["prune_uv_cache"] = prune_uv_cache _SPEC.loader.exec_module(prune_uv_cache) -SHA_KEPT = "17cfb817bccec3a9c247013360cc550c2bac441e" -SHA_STALE = "0123456789abcdef0123456789abcdef01234567" +SHA_KEPT = "17cfb817bccec3a9c247013360cc550c2bac441e" # pragma: allowlist secret +SHA_STALE = "0123456789abcdef0123456789abcdef01234567" # pragma: allowlist secret TAG = "cp313-cp313-linux_x86_64" @@ -126,7 +126,7 @@ def test_entry_version(entry: str, expected: str | None) -> None: def test_prunes_stale_pypi_wheels_and_their_archives(cache: Path) -> None: - pypi = cache / "wheels-v6" / "pypi" + pypi = cache / "wheels-v7" / "pypi" _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old-1") _wheel(pypi, "orjson", f"3.11.9-{TAG}", "old-2") _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") @@ -143,7 +143,7 @@ def test_prunes_stale_pypi_wheels_and_their_archives(cache: Path) -> None: def test_does_not_confuse_version_prefixes(cache: Path) -> None: # Removing 1.0 must not touch 1.0.1's files (a prefix glob would). - pypi = cache / "wheels-v6" / "pypi" + pypi = cache / "wheels-v7" / "pypi" _wheel(pypi, "pkg", "1.0-py3-none-any", "stale") _wheel(pypi, "pkg", "1.0.1-py3-none-any", "kept") @@ -157,8 +157,8 @@ def test_does_not_confuse_version_prefixes(cache: Path) -> None: def test_prunes_alternative_index_trees(cache: Path) -> None: - # torch & co. come from download.pytorch.org: wheels-v6/index//... - index = cache / "wheels-v6" / "index" / "d1g3st" + # torch & co. come from download.pytorch.org: wheels-v7/index//... + index = cache / "wheels-v7" / "index" / "d1g3st" _wheel(index, "torch", f"2.8.0+cu130-{TAG}", "torch-old") _wheel(index, "torch", f"2.9.0+cu130-{TAG}", "torch-kept") @@ -174,7 +174,7 @@ def test_prunes_alternative_index_trees(cache: Path) -> None: def test_keeps_archives_referenced_by_untouched_trees(cache: Path) -> None: # url/ and editable/ entries carry no version to match, so they are left # alone -- and so must the archives they point at. - wheels = cache / "wheels-v6" + wheels = cache / "wheels-v7" _link(wheels / "url" / "abc" / "vllm" / f"0.29.0-{TAG}", _archive(cache, "url")) _link(wheels / "editable" / "def" / "proj" / "0.1.0", _archive(cache, "edit")) @@ -187,7 +187,7 @@ def test_keeps_archives_referenced_by_untouched_trees(cache: Path) -> None: def test_prunes_stale_sdist_versions_keeping_built_wheel_archives( cache: Path, ) -> None: - sdists = cache / "sdists-v9" / "pypi" / "flash-attn" + sdists = cache / "sdists-v10" / "pypi" / "flash-attn" _link(sdists / "2.7.0" / "rev1" / "wheel", _archive(cache, "built-old")) _link(sdists / "2.8.3" / "rev1" / "wheel", _archive(cache, "built-kept")) (sdists / "2.8.3" / "rev1" / "src").mkdir() @@ -199,26 +199,60 @@ def test_prunes_stale_sdist_versions_keeping_built_wheel_archives( def test_prunes_git_builds_and_checkouts_by_commit(cache: Path) -> None: - git_builds = cache / "sdists-v9" / "git" / "repo" - _link(git_builds / SHA_KEPT[:16] / "rev" / "wheel", _archive(cache, "git-kept")) - _link(git_builds / SHA_STALE[:16] / "rev" / "wheel", _archive(cache, "git-old")) - checkouts = cache / "git-v0" / "checkouts" / "repo" - (checkouts / SHA_KEPT[:7]).mkdir(parents=True) - (checkouts / SHA_STALE[:7]).mkdir() + # uv >= 0.13: sdists-v10/git//<16-hex commit>// + # holds the built wheel, its unzipped link and metadata.msgpack (astral-sh/uv#22404). + git_builds = cache / "sdists-v10" / "git" / "repo" + for sha, archive in ((SHA_KEPT, "git-kept"), (SHA_STALE, "git-old")): + shard = git_builds / sha[:16] / "b1nf0d1g3st" + _link(shard / "pkg-1.0-py3-none-any", _archive(cache, archive)) + (shard / "metadata.msgpack").write_text("metadata") + # git-v1/checkouts//<7-hex commit>/ plus its `<7-hex commit>.ok` marker. + checkouts = cache / "git-v1" / "checkouts" / "repo" + for sha in (SHA_KEPT, SHA_STALE): + (checkouts / sha[:7]).mkdir(parents=True, exist_ok=True) + (checkouts / f"{sha[:7]}.ok").touch() (checkouts / "not-a-sha").mkdir() - (cache / "git-v0" / "db" / "repo").mkdir(parents=True) + (cache / "git-v1" / "db" / "repo").mkdir(parents=True) removed, archives = prune_uv_cache.prune(cache, set(), {SHA_KEPT}, dry_run=False) - assert (removed, archives) == (2, 1) + assert (removed, archives) == (3, 1) assert _names(git_builds) == {SHA_KEPT[:16]} - assert _names(checkouts) == {SHA_KEPT[:7], "not-a-sha"} - assert (cache / "git-v0" / "db" / "repo").is_dir() + assert _names(git_builds / SHA_KEPT[:16] / "b1nf0d1g3st") == { + "pkg-1.0-py3-none-any", + "metadata.msgpack", + } + assert _names(checkouts) == {SHA_KEPT[:7], f"{SHA_KEPT[:7]}.ok", "not-a-sha"} + assert (cache / "git-v1" / "db" / "repo").is_dir() assert _names(cache / "archive-v0") == {"git-kept"} +def test_removes_content_addressed_files_no_archive_links(cache: Path) -> None: + # uv's content-addressed-cache preview (on under `preview = true`) hardlinks every + # archive file to files-v0//; dropping the archive alone frees nothing. + pypi = cache / "wheels-v7" / "pypi" + _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old") + _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") + files = cache / "files-v0" + for archive in ("old", "kept"): + obj = files / archive[:2] / f"{archive}-digest" + obj.parent.mkdir(parents=True, exist_ok=True) + os.link(cache / "archive-v0" / archive / "payload.txt", obj) + # An object restored without its hardlink (e.g. `rsync -a` without -H) is unlinked too. + (files / "zz").mkdir() + (files / "zz" / "split-copy").write_text("x") + + assert prune_uv_cache.prune( + cache, {("orjson", "3.12.0")}, set(), dry_run=False + ) == (2, 1) + + assert _names(files) == {"ke"} + assert _names(files / "ke") == {"kept-digest"} + assert (cache / "archive-v0" / "kept" / "payload.txt").stat().st_nlink == 2 + + def test_dry_run_reports_same_counts_without_removing(cache: Path) -> None: - pypi = cache / "wheels-v6" / "pypi" + pypi = cache / "wheels-v7" / "pypi" _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old") _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") keep = {("orjson", "3.12.0")} @@ -230,7 +264,7 @@ def test_dry_run_reports_same_counts_without_removing(cache: Path) -> None: def test_second_run_is_a_no_op(cache: Path) -> None: - pypi = cache / "wheels-v6" / "pypi" + pypi = cache / "wheels-v7" / "pypi" _wheel(pypi, "orjson", f"3.11.3-{TAG}", "old") _wheel(pypi, "orjson", f"3.12.0-{TAG}", "kept") keep = {("orjson", "3.12.0")} @@ -285,7 +319,7 @@ def test_dockerfile_prunes_the_seed_before_any_sync() -> None: @pytest.mark.parametrize("suffix", [".http", ".msgpack", ".lock", ".rev"]) def test_compares_sdist_version_directories_verbatim(cache: Path, suffix: str) -> None: version = f"1.0+foo{suffix}" - sdists = cache / "sdists-v9" / "pypi" / "pkg" + sdists = cache / "sdists-v10" / "pypi" / "pkg" _link(sdists / version / "rev" / "wheel", _archive(cache, "built")) prune_uv_cache.prune(cache, {("pkg", version)}, set(), dry_run=False) @@ -299,10 +333,10 @@ def test_compares_sdist_version_directories_verbatim(cache: Path, suffix: str) - def test_keeps_archives_of_a_package_named_src(cache: Path) -> None: - pypi = cache / "wheels-v6" / "pypi" + pypi = cache / "wheels-v7" / "pypi" _wheel(pypi, "src", "1.0-py3-none-any", "wheel-kept") _wheel(pypi, "src", "0.9-py3-none-any", "wheel-old") - sdists = cache / "sdists-v9" / "pypi" + sdists = cache / "sdists-v10" / "pypi" _link(sdists / "src" / "1.0" / "rev" / "wheel", _archive(cache, "sdist-kept")) _link(sdists / "src" / "0.9" / "rev" / "wheel", _archive(cache, "sdist-old")) # A link inside an unpacked source tree does not keep its target alive. @@ -318,7 +352,7 @@ def test_keeps_archives_of_a_package_named_src(cache: Path) -> None: def test_main_refuses_a_suspiciously_small_lock( cache: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - _wheel(cache / "wheels-v6" / "pypi", "orjson", f"3.11.3-{TAG}", "old") + _wheel(cache / "wheels-v7" / "pypi", "orjson", f"3.11.3-{TAG}", "old") lock = tmp_path / "uv.lock" lock.write_text('version = 1\n[[package]]\nname = "a"\nversion = "1"\n') monkeypatch.setattr( From e4dce7e736daac88065022d0bb0f03cf47045db7 Mon Sep 17 00:00:00 2001 From: Kajal Jain Date: Mon, 9 Nov 2026 12:05:43 -0600 Subject: [PATCH 8/8] test(docker): shorten the prune-placement test comment The Dockerfile already explains why the prune runs where it does. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Kajal Jain --- tests/unit/tools/test_prune_uv_cache.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/unit/tools/test_prune_uv_cache.py b/tests/unit/tools/test_prune_uv_cache.py index 9bf1559d099..7190c57908a 100644 --- a/tests/unit/tools/test_prune_uv_cache.py +++ b/tests/unit/tools/test_prune_uv_cache.py @@ -274,12 +274,7 @@ def test_second_run_is_a_no_op(cache: Path) -> None: def test_dockerfile_prunes_the_seed_before_any_sync() -> None: - # Deleting cache files in a later layer only hides them; the image still ships - # them. So the prune runs inside the dependency RUN -- right after the cache - # mount's content is restored and before the first sync, so it only judges - # restored content and never deletes the build-isolation deps (setuptools, a - # build-time torch, ...) the syncs resolve outside uv.lock. The RUN's - # `uv_cache_persist prune` trap then mirrors the pruned cache into the mount. + # Prune restored content in its importing layer, before syncs add build dependencies. lines = (REPO_ROOT / "docker" / "Dockerfile").read_text().splitlines() code = [(i, line) for i, line in enumerate(lines) if not line.startswith("#")]