Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 23 additions & 15 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ RUN curl --proto '=https' --tlsv1.2 --retry 3 --retry-delay 2 -sSf https://sh.ru
# Install uv and python outside /root so an unprivileged runtime UID (for example
# 65532) can execute them: /root is not traversable by other users, and the project
# venv's python is a symlink into uv's managed-Python directory.
ARG UV_VERSION=0.11.28
ARG UV_VERSION=0.13.6
ARG PYTHON_VERSION=3.13.14
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
# One uv cache for the build and for every runtime identity. It holds every wheel the venvs are
Expand Down Expand Up @@ -271,6 +271,8 @@ COPY --from=nemo-rl nemo_rl/__init__.py nemo_rl/package_info.py ./nemo_rl/
COPY --from=nemo-rl nemo_rl/distributed/actor_environments.py ./nemo_rl/distributed/
COPY --from=nemo-rl tools/build-custom-vllm.sh ./tools/build-custom-vllm.sh
COPY --from=nemo-rl tools/build-custom-flashinfer.sh ./tools/build-custom-flashinfer.sh
# Part of this recipe, not of the selected source: an older NRL_GIT_REF has no such script.
COPY docker/prune_uv_cache.py /usr/local/bin/prune_uv_cache.py
COPY --from=nemo-rl --link research/ ./research/
COPY --from=nemo-rl --link \
--exclude=TensorRT-LLM-workspace \
Expand All @@ -290,7 +292,7 @@ RUN --mount=type=ssh \
# downloads persist for the next build. This is the first uv-writing RUN in
# the stage, so it force-copies the whole store in and prunes on sync-back
# (see docker/uv_cache_mount.sh). Every later RUN in this stage that also
# invokes uv (TRT-LLM build, Dynamo install, worker-venv prefetch phase 2,
# invokes uv (TRT-LLM build, worker-venv prefetch phase 2,
# Gym prefetch) sources the same helper but only tops up what it adds,
# instead of re-shipping the whole store again in its own layer.
# Whether the materialized copy also ships in the final image (for the
Expand All @@ -301,6 +303,17 @@ uv_cache_restore force
trap 'uv_cache_persist prune' EXIT

uv venv --seed
# Drop what the restored cache carried in that uv.lock no longer pins -- versions from
# older locks, which neither the seed export's `uv cache prune --ci` (it keeps every
# source-built version) nor the durable cache mount ever evict (see
# docker/prune_uv_cache.py). It runs before any sync, so it only ever judges restored
# content: build-isolation deps the syncs below resolve outside uv.lock (setuptools, a
# build-time torch, ...) are kept. It runs in this RUN because deleting files in a later
# layer only hides them, and the image would still ship every byte; and the
# `uv_cache_persist prune` trap above mirrors the result back into the cache mount.
du -sh "${UV_CACHE_DIR}"
"${UV_PROJECT_ENVIRONMENT}/bin/python" /usr/local/bin/prune_uv_cache.py --lock uv.lock --cache-dir "${UV_CACHE_DIR}"
du -sh "${UV_CACHE_DIR}"
# The custom build scripts will alter the pyproject.toml and uv.lock
if [[ -n "${BUILD_CUSTOM_VLLM:-}" ]]; then
UV_LINK_MODE=hardlink bash tools/build-custom-vllm.sh ${BUILD_CUSTOM_VLLM_URL:-} ${BUILD_CUSTOM_VLLM_REF:-} ${BUILD_CUSTOM_VLLM_PRECOMPILED_WHEEL_LOCATION:-}
Expand All @@ -321,7 +334,6 @@ apt-get update
# DeepEP's multinode link needs the exact filename libnvidia-ml.so.1; the CUDA base
# only ships stubs/libnvidia-ml.so. Purged again after the syncs below.
apt-get install -y --no-install-recommends libnvidia-ml-dev
uv cache clean deep-ep
# Everything is hardlinked against the uv cache; this layer holds exactly one real copy of every wheel.
UV_LINK_MODE=hardlink uv sync --frozen --no-install-project
if [[ -z "${SKIP_VLLM_BUILD:-}" ]]; then
Expand Down Expand Up @@ -680,21 +692,17 @@ COPY --from=nemo-rl --exclude=pyproject.toml --exclude=uv.lock . /opt/nemo-rl

# Keep Dynamo's Python 3.12 dependency graph isolated from NeMo-RL's normal
# Ray/vLLM environments. Default image builds do not create this environment.
RUN --mount=from=uv-cache-helper,src=/,dst=/tmp/uv-cache-helper \
--mount=type=cache,id=uv-cache-${TARGETARCH}-${BASE_IMAGE}-${UV_VERSION}-${HYBRID_EP_MULTINODE},from=uv-cache-seed,source=.,target=/root/.uv-cache-store,sharing=locked <<"EOF" bash -exu -o pipefail
RUN <<"EOF" bash -exu -o pipefail
if [[ "${BUILD_DYNAMO:-0}" != "1" ]]; then
echo "BUILD_DYNAMO is not 1; skipping the managed Dynamo runtime"
exit 0
fi
# Same materialize/sync-back/strip dance as the other uv-writing RUNs --
# docker/dynamo/install.sh calls uv sync with no explicit UV_CACHE_DIR override, so
# it uses the image-wide UV_CACHE_DIR (set in base), which must be a plain
# same-filesystem directory for hardlink installs. Already exists from the
# hermetic stage unless STRIP_UV_CACHE removed it there.
source /tmp/uv-cache-helper/uv_cache_mount.sh
uv_cache_restore
trap 'uv_cache_persist' EXIT
bash docker/dynamo/install.sh
# Dynamo syncs its own lock (Python 3.12) into a --link-mode copy venv that nothing re-syncs at
# runtime, so its wheels are never needed from the shared cache. A throwaway cache keeps them
# out of this layer instead of shipping them twice, and out of the durable cache mount, so this
# RUN needs neither the mount nor the uv_cache_mount.sh restore/persist.
UV_CACHE_DIR=/tmp/dynamo-uv-cache bash docker/dynamo/install.sh
rm -rf /tmp/dynamo-uv-cache
EOF
# Unshallow the repo to get the full history (in the case it was from the scratch layer).
# Potentially not necessary if the repo is passed in as a complete repository (w/ full git history),
Expand Down Expand Up @@ -862,7 +870,7 @@ echo "Exporting uv cache for build ${UV_CACHE_EXPORT_NONCE:-local}"
# RUN command, leaving this builder-local mount empty. Restore the seed only in
# that case; unconditionally merging it would resurrect wheels uv evicted.
if [[ -z "$(find /root/.uv-cache-store -mindepth 1 -print -quit)" ]]; then
rsync -a /tmp/uv-cache-seed/ /root/.uv-cache-store/
rsync -aH /tmp/uv-cache-seed/ /root/.uv-cache-store/
fi
mkdir -p /tmp/uv-cache-export
cp -a /root/.uv-cache-store/. /tmp/uv-cache-export/
Expand Down
287 changes: 287 additions & 0 deletions docker/prune_uv_cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,287 @@
#!/usr/bin/env python3
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Prune uv's cache down to what uv.lock still pins.
Comment thread
kajalj22 marked this conversation as resolved.
Resolved
Comment thread
kajalj22 marked this conversation as resolved.
Resolved

uv.lock pins one version (or one git commit) per package across the WHOLE
project -- every extra, installed in this build or not. uv's cache is
append-only: `uv sync` downloads what a build needs and never removes a version
an older lockfile pinned. The CI seed persists the cache across builds for as
long as BASE_IMAGE/UV_VERSION/HYBRID_EP_MULTINODE are unchanged (see the
dependency-install RUN in docker/Dockerfile), so every dependency bump would
otherwise leave its predecessor in the seed and in every shipped image.
`uv cache prune` does not help: it removes stale cache-format buckets, cached
environments and unlinked archives, never a valid entry for an older version.

Removed, matched against uv.lock (layout per uv's crates/uv-cache/src/wheel.rs):
wheels-v*/{pypi,index/<digest>}/<name>/<version>-<tags>[.http|.msgpack]
sdists-v*/{pypi,index/<digest>}/<name>/<version>[...]
kept iff (<name>, <version>) is pinned by uv.lock.
sdists-v*/git/<digest>/<first 16 hex of commit>
git-v*/checkouts/<digest>/<short commit>[.ok]
kept iff the commit prefixes a git commit uv.lock pins.
archive-v0/<id>
kept iff any link left under wheels-v*/ or sdists-v*/ resolves to it --
the same reachability scan uv's own `uv cache prune` uses, so archives
behind entries this script never touches (url/, path/, editable/) stay.
files-v*/<shard>/<digest>
kept iff something still hardlinks it (link count > 1), as uv's own
`uv cache prune` decides.

Never removed: url/, path/, editable/ trees (keyed by URL digest, no version to
match), git-v*/db, and every other bucket. That keeps it safe to run before
shipping UV_CACHE_DIR: it cannot remove anything offline
`uv run --extra <backend>` needs, because that is itself "what uv.lock pins".
"""

from __future__ import annotations

import argparse
import os
import re
import shutil
import sys
import tomllib
from pathlib import Path

# PEP 503: uv.lock and the cache's directory names may differ in case/separator.
_NORMALIZE_RE = re.compile(r"[-_.]+")
_GIT_SHA_RE = re.compile(r"#([0-9a-f]{40})$")
_METADATA_SUFFIXES = (".http", ".msgpack", ".lock", ".rev")
# uv's archive-reference scan skips these; they are never links into archive-v0.
_SCAN_SKIP_SUFFIXES = (".lock", ".whl", ".http", ".rev", ".msgpack")


def normalize(name: str) -> str:
return _NORMALIZE_RE.sub("-", name).lower()


def load_lock(lock_path: Path) -> tuple[set[tuple[str, str]], set[str]]:
"""Returns ({(normalized name, version)}, {full git commit SHA})."""
with lock_path.open("rb") as f:
packages = tomllib.load(f).get("package", [])
versions = set()
for p in packages:
if "name" in p and "version" in p:
name, version = normalize(p["name"]), p["version"]
versions.add((name, version))
# uv's WheelFilename::cache_key truncates long versions to 47 chars.
versions.add((name, version[:47].rstrip(".+")))
shas = set()
for p in packages:
git = p.get("source", {}).get("git")
if git and (m := _GIT_SHA_RE.search(git)):
shas.add(m.group(1))
return versions, shas


def entry_version(entry_name: str) -> str | None:
"""`<version>-<tags>[.http]` or `<version>` -> `<version>`; None if not a version."""
for suffix in _METADATA_SUFFIXES:
if entry_name.endswith(suffix):
entry_name = entry_name[: -len(suffix)]
break
version = entry_name.split("-", 1)[0]
# PEP 440 versions (including epochs) start with a digit. Anything else is
# not ours to judge, so it is kept.
return version if version[:1].isdigit() else None


def _registry_roots(cache_dir: Path) -> list[Path]:
roots = []
for bucket in ("wheels-v*", "sdists-v*"):
for bucket_dir in cache_dir.glob(bucket):
roots.extend(p for p in [bucket_dir / "pypi"] if p.is_dir())
index = bucket_dir / "index"
if index.is_dir():
roots.extend(p for p in index.iterdir() if p.is_dir())
return roots


def _sha_kept(prefix: str, shas: set[str]) -> bool:
# Unparseable names are kept: only a hex commit prefix can be matched.
if len(prefix) < 7 or not all(c in "0123456789abcdef" for c in prefix):
return True
return any(sha.startswith(prefix) for sha in shas)


def plan_removals(
cache_dir: Path, versions: set[tuple[str, str]], shas: set[str]
) -> list[Path]:
removals = []
for root in _registry_roots(cache_dir):
for name_dir in root.iterdir():
if not name_dir.is_dir() or name_dir.is_symlink():
continue
name = normalize(name_dir.name)
for entry in name_dir.iterdir():
if root.relative_to(cache_dir).parts[0].startswith("sdists-v") and (
entry.is_dir() and not entry.is_symlink()
):
# An sdist version directory is the bare version, compared verbatim:
# stripping a metadata suffix would change a version like `1.0+foo.http`.
version = entry.name if entry.name[:1].isdigit() else None
else:
version = entry_version(entry.name)
if version is not None and (name, version) not in versions:
removals.append(entry)
commit_dirs = [
*cache_dir.glob("sdists-v*/git/*/*"),
*cache_dir.glob("git-v*/checkouts/*/*"),
]
for commit_dir in commit_dirs:
if commit_dir.is_dir() and not _sha_kept(commit_dir.name, shas):
removals.append(commit_dir)
# uv >= 0.12 marks a finished checkout with a sibling `<short sha>.ok` file.
ready = commit_dir.with_name(f"{commit_dir.name}.ok")
if ready.is_file():
removals.append(ready)
return removals


def _reachable_archives(cache_dir: Path, removed: set[Path]) -> set[str]:
archive_root = (cache_dir / "archive-v0").resolve()
registry_roots = set(_registry_roots(cache_dir))
reachable = set()
for bucket in [*cache_dir.glob("wheels-v*"), *cache_dir.glob("sdists-v*")]:
for dirpath, dirnames, filenames in os.walk(bucket):
here = Path(dirpath)
# Skip the unpacked source trees inside sdist builds (`<rev>/src`), but not
# a package that happens to be named `src`.
dirnames[:] = [
d
for d in dirnames
if here / d not in removed
and (
d != "src"
or bucket.name.startswith("wheels-v")
or here in registry_roots
)
]
# os.walk lists directory symlinks under dirnames without following them.
for name in [*filenames, *dirnames]:
link = here / name
if link in removed or name.endswith(_SCAN_SKIP_SUFFIXES):
continue
if link.is_symlink():
target = link.resolve()
if target.parent == archive_root:
reachable.add(target.name)
return reachable


def _remove_unlinked_files(cache_dir: Path, *, verbose: bool) -> int:
"""Drop content-addressed file objects no archive hardlinks any more.

With uv's `content-addressed-cache` preview feature (on under `preview = true`),
every file in archive-v0/<id>/ is a hardlink to files-v0/<shard>/<digest>, so
removing an archive frees nothing until its objects go too. An object with a
link count of 1 has no archive (and no hardlinked venv) left -- the same rule
as uv's own `uv cache prune` (Cache::prune_archive_files).
"""
removed = 0
for obj in cache_dir.glob("files-v*/*/*"):
if not obj.is_symlink() and obj.is_file() and obj.stat().st_nlink == 1:
if verbose:
print(f"remove: {obj}")
obj.unlink()
removed += 1
for shard in cache_dir.glob("files-v*/*"):
if shard.is_dir() and not shard.is_symlink() and not any(shard.iterdir()):
shard.rmdir()
return removed


def prune(
cache_dir: Path,
versions: set[tuple[str, str]],
shas: set[str],
*,
dry_run: bool,
verbose: bool = False,
) -> tuple[int, int]:
"""Returns (entries_removed, archives_removed)."""
removals = plan_removals(cache_dir, versions, shas)
reachable = _reachable_archives(cache_dir, set(removals))
archive_root = cache_dir / "archive-v0"
orphaned = (
[a for a in archive_root.iterdir() if a.name not in reachable]
if archive_root.is_dir()
else []
)
for path in [*removals, *orphaned]:
if verbose:
print(f"{'would remove' if dry_run else 'remove'}: {path}")
if dry_run:
continue
if path.is_symlink() or path.is_file():
path.unlink()
else:
shutil.rmtree(path)
if not dry_run:
_remove_unlinked_files(cache_dir, verbose=verbose)
return len(removals), len(orphaned)


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--lock", type=Path, required=True, help="Path to uv.lock")
parser.add_argument(
"--cache-dir", type=Path, required=True, help="UV_CACHE_DIR to prune"
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Report what would be removed without removing it",
)
parser.add_argument(
"--verbose", action="store_true", help="Print every removed path"
)
# Sanity floor: a near-empty keep set (e.g. a uv.lock format change this
# script does not understand) must not turn into deleting the whole cache.
parser.add_argument("--min-keep", type=int, default=100)
args = parser.parse_args()

if not args.lock.is_file():
print(f"error: {args.lock} does not exist", file=sys.stderr)
return 1
if not args.cache_dir.is_dir():
print(f"error: {args.cache_dir} does not exist", file=sys.stderr)
return 1

versions, shas = load_lock(args.lock)
if len(versions) < args.min_keep:
print(
f"error: only parsed {len(versions)} pinned packages from {args.lock} "
f"(expected at least {args.min_keep}) -- refusing to prune, "
"uv.lock's format may have changed in a way this script doesn't understand",
file=sys.stderr,
)
return 1

entries_removed, archives_removed = prune(
args.cache_dir, versions, shas, dry_run=args.dry_run, verbose=args.verbose
)
verb = "Would remove" if args.dry_run else "Removed"
print(
f"{verb} {entries_removed} stale cache entries and {archives_removed} "
f"orphaned archives (kept {len(versions)} packages and {len(shas)} git "
f"commits pinned in {args.lock})"
)
return 0


if __name__ == "__main__":
sys.exit(main())
Loading
Loading