Skip to content

Commit b9c14d9

Browse files
Wauplinclaude
andauthored
[Download] Write cache ref file atomically in snapshot_download (#4829)
`snapshot_download` rewrote `refs/<revision>` with a plain `open(path, "w")` on every call, even when the commit hash was unchanged. `open(..., "w")` truncates the file before rewriting it, so a concurrent reader can observe an empty revision and conclude that a fully cached file is missing. Use the existing `_cache_commit_hash_for_specific_revision` helper instead: it skips the write when the ref is already up to date and otherwise writes to a temporary file and `os.replace`s it into place. `hf_hub_download` and `HfApi.resolve_revision` already go through it. Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 8934ec0 commit b9c14d9

1 file changed

Lines changed: 14 additions & 7 deletions

File tree

src/huggingface_hub/_snapshot_download.py

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,13 @@
1818
RepositoryNotFoundError,
1919
RevisionNotFoundError,
2020
)
21-
from .file_download import REGEX_COMMIT_HASH, DryRunFileInfo, hf_hub_download, repo_folder_name
21+
from .file_download import (
22+
REGEX_COMMIT_HASH,
23+
DryRunFileInfo,
24+
_cache_commit_hash_for_specific_revision,
25+
hf_hub_download,
26+
repo_folder_name,
27+
)
2228
from .hf_api import HfApi, RepoFile
2329
from .utils import OfflineModeIsEnabled, filter_repo_objects, logging, validate_hf_hub_args
2430
from .utils._xet_progress_reporting import (
@@ -412,14 +418,15 @@ def snapshot_download(
412418
# if passed revision is not identical to commit_hash
413419
# then revision has to be a branch name or tag name.
414420
# In that case store a ref (except if ResolvedRevision, in which case it's already done).
415-
if not isinstance(revision, ResolvedRevision) and revision != commit_hash:
416-
ref_path = os.path.join(storage_folder, "refs", revision)
421+
if not isinstance(revision, ResolvedRevision):
417422
try:
418-
os.makedirs(os.path.dirname(ref_path), exist_ok=True)
419-
with open(ref_path, "w") as f:
420-
f.write(commit_hash)
423+
# Skips the write if the ref is already up to date, and writes atomically otherwise so that
424+
# concurrent readers never observe a truncated (empty) ref file.
425+
_cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
421426
except OSError as e:
422-
logger.warning(f"Ignored error while writing commit hash to {ref_path}: {e}.")
427+
logger.warning(
428+
f"Ignored error while writing commit hash to {os.path.join(storage_folder, 'refs', revision)}: {e}."
429+
)
423430

424431
results: list[str | DryRunFileInfo] = []
425432

0 commit comments

Comments
 (0)