From aa37261578a2fd7ff35295704ba556f05d0060a4 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:31:09 +0200 Subject: [PATCH 1/6] Retry get_daily_ci_runs on stale GitHub API cache (#48374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [debug] Poll GitHub API 10x to observe cache inconsistency in schedule runs Co-Authored-By: Claude Sonnet 4.6 * Retry get_daily_ci_runs when GitHub API returns stale cache results The GitHub Actions search index (used for event=/branch= filters) can lag behind the database — different backend nodes return wildly different total_count values (190, 238, 311, 413 observed for the same URL within minutes), and the most-recent runs are missing from stale responses. Detect staleness by checking whether the current GITHUB_RUN_ID appears in the returned list; if absent, retry up to 3 times with a 30s delay. Co-Authored-By: Claude Sonnet 4.6 * [debug] Use get_daily_ci_runs in debug job; add current_run_id param Add optional `current_run_id` parameter to `get_daily_ci_runs` so callers can supply a known schedule run ID for stale-cache detection when GITHUB_RUN_ID belongs to a non-schedule-triggered job (e.g. push). Update the debug workflow job to call the Python function directly (two quick back-to-back calls) instead of raw curl, so the retry logic is exercised end-to-end and visible in the CI logs. Co-Authored-By: Claude Sonnet 4.6 * [debug] Force retry path with fake run ID to verify stale-detection logging Co-Authored-By: Claude Sonnet 4.6 * Revert "[debug] Force retry path with fake run ID to verify stale-detection logging" This reverts commit e49340f7c58ae1abc43f90e6678deedc50b1a0e5. * [debug] Run get_daily_ci_runs 20 times to catch stale cache hit Co-Authored-By: Claude Sonnet 4.6 * [debug] Sleep 60s between get_daily_ci_runs calls to hit different cache nodes Co-Authored-By: Claude Sonnet 4.6 * Increase get_daily_ci_runs max_attempts from 3 to 5 Co-Authored-By: Claude Sonnet 4.6 * Add unit tests for get_daily_ci_runs stale-cache retry logic - tests/utils/test_get_previous_daily_ci.py: 5 tests covering fresh hit (no retry), stale→fresh (one retry), all stale (max_attempts exhausted), no current_run_id (stale check skipped), and empty schedule fallback to workflow_run event - utils/get_previous_daily_ci.py: ruff formatting fix (long print line) Co-Authored-By: Claude Sonnet 4.6 * Infer stale-check eligibility from current run's workflow_id Instead of an explicit `current_run_id` argument, `get_daily_ci_runs` now always fetches the current run's metadata (GITHUB_RUN_ID) upfront and compares its `workflow_id` to the queried one: - Same workflow → stale-cache detection applies (max_attempts=5, 30s sleep) - Different workflow → skip (e.g. AMD CI querying Nvidia CI runs); left for a follow-up PR once the same-workflow case is confirmed stable Also adds a TODO comment on the event=workflow_run fallback path (AMD CI) and updates the 6 unit tests accordingly (new test for the different-workflow case; all tests account for the upfront current-run lookup call). Co-Authored-By: Claude Sonnet 4.6 * Gate stale-cache check on GITHUB_EVENT_NAME == 'schedule' Push- and dispatch-triggered runs won't appear in event=schedule results even when the API is fresh, causing spurious retries. Only enable the stale-cache retry loop when the current run is itself schedule-triggered. Adds a corresponding unit test (test_non_schedule_event_skips_stale_check). Co-Authored-By: Claude Sonnet 4.6 * Revert debug changes to self-scheduled-caller.yml Restore the workflow to its main-branch state; the debug polling job was only needed to verify the stale-cache behaviour. Co-Authored-By: Claude Sonnet 4.6 * Condense get_daily_ci_runs comments and refer to #48374 for rationale Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: ydshieh --- tests/utils/test_get_previous_daily_ci.py | 209 ++++++++++++++++++++++ utils/get_previous_daily_ci.py | 96 +++++++--- 2 files changed, 279 insertions(+), 26 deletions(-) create mode 100644 tests/utils/test_get_previous_daily_ci.py diff --git a/tests/utils/test_get_previous_daily_ci.py b/tests/utils/test_get_previous_daily_ci.py new file mode 100644 index 000000000000..f5111b0e5d24 --- /dev/null +++ b/tests/utils/test_get_previous_daily_ci.py @@ -0,0 +1,209 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest +from unittest.mock import patch + + +# utils/ is not an installable package; add it to the path so CI utility modules +# can be imported without modifying the installed transformers package. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "utils")) + +from get_previous_daily_ci import get_daily_ci_runs # noqa: E402 + + +_WORKFLOW_ID = 90575235 +_CURRENT_RUN_ID = 33038227895 + +# Simulates get_github_json("/runs/{GITHUB_RUN_ID}") — same workflow as the one being queried. +_CURRENT_RUN_RESPONSE = {"id": _CURRENT_RUN_ID, "workflow_id": _WORKFLOW_ID} + +# Simulates a current run belonging to a *different* workflow (e.g. AMD CI querying Nvidia). +_DIFFERENT_WORKFLOW_RUN_RESPONSE = {"id": 11111111111, "workflow_id": 99999} + +_STALE_RESPONSE = { + "total_count": 209, + "workflow_runs": [ + { + "id": 30781855254, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-03T03:27:23Z", + "event": "schedule", + }, + { + "id": 30730647697, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-02T03:27:19Z", + "event": "schedule", + }, + ], +} + +_FRESH_RESPONSE = { + "total_count": 413, + "workflow_runs": [ + { + "id": _CURRENT_RUN_ID, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-27T04:03:26Z", + "event": "schedule", + }, + { + "id": 32923823246, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-26T02:44:02Z", + "event": "schedule", + }, + ], +} + + +class GetDailyCiRunsRetryTest(unittest.TestCase): + """Unit tests for the stale-cache retry logic in get_daily_ci_runs. + + get_github_json is called once up-front to fetch the current run's workflow_id + (for stale-check eligibility), then once per schedule-query attempt. Tests use + side_effect lists ordered as: [current-run lookup, attempt-1, attempt-2, ...]. + """ + + def test_fresh_on_first_attempt_no_retry(self): + """When the first response contains the current run, return immediately without retrying.""" + with ( + patch("get_previous_daily_ci.get_github_json", side_effect=[_CURRENT_RUN_RESPONSE, _FRESH_RESPONSE]), + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + mock_sleep.assert_not_called() + self.assertEqual(runs[0]["id"], _CURRENT_RUN_ID) + + def test_stale_then_fresh_retries_once(self): + """A stale first response triggers one retry that returns fresh data.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE, _STALE_RESPONSE, _FRESH_RESPONSE], + ), + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + mock_sleep.assert_called_once_with(30) + self.assertEqual(runs[0]["id"], _CURRENT_RUN_ID) + + def test_all_stale_exhausts_max_attempts(self): + """When all attempts return stale data, proceed after max_attempts with the last result.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE] + [_STALE_RESPONSE] * 5, + ) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # 1 current-run lookup + 5 schedule queries (max_attempts=5) + self.assertEqual(mock_api.call_count, 6) + # sleep between attempts 1-2, 2-3, 3-4, 4-5 — not after the last attempt + self.assertEqual(mock_sleep.call_count, 4) + mock_sleep.assert_called_with(30) + # Stale data is returned as a graceful fallback + self.assertEqual(runs[0]["id"], 30781855254) + + def test_different_workflow_skips_stale_check(self): + """When workflow_id differs from the current run's, stale check is skipped entirely.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_DIFFERENT_WORKFLOW_RUN_RESPONSE, _STALE_RESPONSE], + ) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": "11111111111", "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # 1 current-run lookup + 1 schedule query (max_attempts=1, no retries) + self.assertEqual(mock_api.call_count, 2) + mock_sleep.assert_not_called() + # Returns whatever the single attempt gave (stale in this case) + self.assertEqual(runs[0]["id"], 30781855254) + + def test_non_schedule_event_skips_stale_check(self): + """When the triggering event is not 'schedule' (e.g. push), stale check is skipped.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE, _STALE_RESPONSE], + ) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "push"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # 1 current-run lookup + 1 schedule query (max_attempts=1, no retries) + self.assertEqual(mock_api.call_count, 2) + mock_sleep.assert_not_called() + self.assertEqual(runs[0]["id"], 30781855254) + + def test_no_github_run_id_skips_stale_check(self): + """With GITHUB_RUN_ID unset, the current-run lookup is skipped and no retry is done.""" + with ( + patch("get_previous_daily_ci.get_github_json", side_effect=[_STALE_RESPONSE]) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {}, clear=False), + ): + os.environ.pop("GITHUB_RUN_ID", None) + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # No current-run lookup, 1 schedule query, no retries + mock_api.assert_called_once() + mock_sleep.assert_not_called() + self.assertEqual(runs[0]["id"], 30781855254) + + def test_empty_schedule_falls_back_to_workflow_run(self): + """When event=schedule returns no runs, fall back to event=workflow_run without retrying.""" + fallback_response = { + "total_count": 5, + "workflow_runs": [ + { + "id": _CURRENT_RUN_ID, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-27T04:03:26Z", + "event": "workflow_run", + }, + ], + } + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE, {"total_count": 0, "workflow_runs": []}, fallback_response], + ), + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID)}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + mock_sleep.assert_not_called() + self.assertEqual(len(runs), 1) + self.assertEqual(runs[0]["event"], "workflow_run") diff --git a/utils/get_previous_daily_ci.py b/utils/get_previous_daily_ci.py index 290213e8a277..a8af937f4e67 100644 --- a/utils/get_previous_daily_ci.py +++ b/utils/get_previous_daily_ci.py @@ -1,50 +1,94 @@ import os +import time import zipfile from get_ci_error_statistics import download_artifact, get_artifacts_links, get_github_json def get_daily_ci_runs(token, num_runs=7, workflow_id=None): - """Get the workflow runs of the scheduled (daily) CI. + """Get the most recent workflow runs of the scheduled (daily) CI on the main branch. - This only selects the runs triggered by the `schedule` event on the `main` branch. + Queries event=schedule; falls back to event=workflow_run when no results are found + (AMD CI is triggered via workflow_run, not schedule). Retries on stale GitHub API + responses — see PR #48374 for background. """ - # The id of a workflow (not of a workflow run). - # From a given workflow run (where we have workflow run id), we can get the workflow id by going to - # https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id} - # and check the `workflow_id` key. - - if not workflow_id: - workflow_run_id = os.environ["GITHUB_RUN_ID"] - workflow_run = get_github_json( - f"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}", token=token + # Fetch current run metadata to (1) resolve workflow_id when not given, and (2) compare + # workflow IDs to decide whether stale-cache detection applies (see stale_check below). + current_run_id = int(os.environ.get("GITHUB_RUN_ID", 0)) + current_workflow_id = None + if current_run_id: + current_run = get_github_json( + f"https://api.github.com/repos/huggingface/transformers/actions/runs/{current_run_id}", token=token ) - workflow_id = workflow_run["workflow_id"] + current_workflow_id = current_run["workflow_id"] + if not workflow_id: + workflow_id = current_workflow_id url = f"https://api.github.com/repos/huggingface/transformers/actions/workflows/{workflow_id}/runs" - # On `main` branch + event being `schedule` + not returning PRs + only `num_runs` results url += f"?branch=main&exclude_pull_requests=true&per_page={num_runs}" - schedule_url = f"{url}&event=schedule" - print(f"[DEBUG get_daily_ci_runs] Querying: {schedule_url}") - result = get_github_json(schedule_url, token=token) - workflow_runs = result["workflow_runs"] - print(f"[DEBUG get_daily_ci_runs] event=schedule returned {len(workflow_runs)} runs:") - for r in workflow_runs: + # The GitHub Actions search index (used for event=/branch= filters) can lag badly: + # the same URL has returned total_count values of 190, 238, 311, and 413 within minutes + # from different backend nodes (PR #48374). We detect a stale response by checking + # that GITHUB_RUN_ID appears in the results; if absent we retry. + # + # Stale-check is only active when querying the *same* workflow as the current run AND + # the current run is schedule-triggered (so GITHUB_RUN_ID is guaranteed to be in the + # results when the API is fresh). The two remaining uncovered paths are left for a + # follow-up PR once this path is confirmed stable: + # • AMD CI querying its own history → falls into the event=workflow_run fallback below + # (AMD CI is triggered via workflow_run, not schedule). + # • AMD CI querying the matching Nvidia run (different workflow_id) → stale_check=False. + stale_check = ( + current_workflow_id is not None + and int(workflow_id) == int(current_workflow_id) + and os.environ.get("GITHUB_EVENT_NAME") == "schedule" + ) + max_attempts = 5 if stale_check else 1 + + for attempt in range(1, max_attempts + 1): + schedule_url = f"{url}&event=schedule" + print(f"[DEBUG get_daily_ci_runs] Querying (attempt {attempt}/{max_attempts}): {schedule_url}") + result = get_github_json(schedule_url, token=token) + workflow_runs = result["workflow_runs"] print( - f" id={r['id']} status={r['status']} conclusion={r.get('conclusion')} created_at={r['created_at']} event={r['event']}" + f"[DEBUG get_daily_ci_runs] event=schedule returned {len(workflow_runs)} runs (total_count={result.get('total_count')}):" ) - if len(workflow_runs) == 0: - workflow_run_url = f"{url}&event=workflow_run" - print(f"[DEBUG get_daily_ci_runs] Falling back to: {workflow_run_url}") - result = get_github_json(workflow_run_url, token=token) - workflow_runs = result["workflow_runs"] - print(f"[DEBUG get_daily_ci_runs] event=workflow_run returned {len(workflow_runs)} runs:") for r in workflow_runs: print( f" id={r['id']} status={r['status']} conclusion={r.get('conclusion')} created_at={r['created_at']} event={r['event']}" ) + if len(workflow_runs) == 0: + # AMD CI runs appear under event=workflow_run (triggered via the workflow_run + # event, not schedule). Stale-check for this path is TODO (see above). + workflow_run_url = f"{url}&event=workflow_run" + print(f"[DEBUG get_daily_ci_runs] Falling back to: {workflow_run_url}") + result = get_github_json(workflow_run_url, token=token) + workflow_runs = result["workflow_runs"] + print(f"[DEBUG get_daily_ci_runs] event=workflow_run returned {len(workflow_runs)} runs:") + for r in workflow_runs: + print( + f" id={r['id']} status={r['status']} conclusion={r.get('conclusion')} created_at={r['created_at']} event={r['event']}" + ) + break + + if stale_check and current_run_id not in {r["id"] for r in workflow_runs}: + if attempt < max_attempts: + print( + f"[WARN get_daily_ci_runs] Current run {current_run_id} not found in results — " + f"likely a stale GitHub API cache (total_count={result.get('total_count')}). " + f"Retrying in 30s..." + ) + time.sleep(30) + continue + else: + print( + f"[WARN get_daily_ci_runs] Current run {current_run_id} still not found after " + f"{max_attempts} attempts — proceeding with stale results." + ) + break + return workflow_runs From 8631167e310d56baa87f5213f524fdb5f47c4ff9 Mon Sep 17 00:00:00 2001 From: eryk-roch Date: Thu, 27 Aug 2026 12:48:55 +0000 Subject: [PATCH 2/6] Fix `safe_open` mmap memory exhaustion on Windows by using `pread` backend (#48341) * Fix `safe_open` mmap memory exhaustion on Windows by using `pread` backend On Windows, memory-mapping safetensors files reserves copy-on-write commit charge for the entire file. For large multi-shard checkpoints this exhausts virtual memory. Switch to the `pread` backend on `win32` in both `_load_pretrained_model` and `MtpModel` loading paths. * Address review: add MPS backend fix to MtpModel, hoist backend selection before loop, use targeted sys.platform patch in test * Address review: remove tests --- src/transformers/modeling_layers.py | 7 ++++++- src/transformers/modeling_utils.py | 10 +++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_layers.py b/src/transformers/modeling_layers.py index e34e5d6fccda..34bff894c429 100644 --- a/src/transformers/modeling_layers.py +++ b/src/transformers/modeling_layers.py @@ -15,6 +15,7 @@ import os import re +import sys from functools import partial from typing import TYPE_CHECKING @@ -605,8 +606,12 @@ def from_pretrained(cls, main_model: PreTrainedModel, device_map=None, **kwargs) # Open the files, get the slices corresponding only to mtp weights, rename them, and load them mtp_state_dict = {} all_pointer = set() + is_mps = device_map is not None and any( + (d.type if isinstance(d, torch.device) else d) == "mps" for d in device_map.values() + ) + backend = "pread" if is_mps or sys.platform == "win32" else "mmap" for file in mtp_files: - file_pointer = safe_open(file, framework="pt", device="cpu") + file_pointer = safe_open(file, framework="pt", device="cpu", backend=backend) all_pointer.add(file_pointer) for k in file_pointer.keys(): # It's one of the mtp weights diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e3dd16782f1c..4af2133ae118 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4512,7 +4512,15 @@ def _load_pretrained_model( (d.type if isinstance(d, torch.device) else d) == "mps" for d in load_config.device_map.values() ) - backend, device = ("pread", "mps") if is_mps else ("mmap", "cpu") + # Use pread on MPS (mmap incompatible) and Windows (mmap reserves + # copy-on-write commit charge for the entire file, exhausting memory + # for large multi-shard checkpoints). + if is_mps: + backend, device = "pread", "mps" + elif sys.platform == "win32": + backend, device = "pread", "cpu" + else: + backend, device = "mmap", "cpu" file_pointer = safe_open(file, framework="pt", device=device, backend=backend) all_pointer.add(file_pointer) for k in file_pointer.keys(): From 19876312341f42cf49467bb24d67271cf28cb599 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:55:10 +0000 Subject: [PATCH 3/6] [ONNX] Skip affected models on torch 2.13 (two dynamo regressions) (#48191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [ONNX] Skip affected models on torch >= 2.13 (two dynamo ONNX regressions) torch 2.13.0 introduced two regressions in dynamo ONNX export: - pytorch/pytorch#194381: aten.sub type-promotion failure for scalar - int_tensor - pytorch/pytorch#194382: aten.mul.Scalar missing ONNX decomposition Skip all 35 affected model classes under `EXPORT_SKIPS["onnx"]` when torch >= 2.13, using the existing skip infrastructure. Will be removed once the upstream PyTorch fixes land. Co-Authored-By: Claude Sonnet 4.6 * Scope ONNX skip to torch == 2.13.x only (auto-runs on 2.14+) Co-Authored-By: Claude Sonnet 4.6 * Add missing BigBird subclasses to torch 2.13 ONNX skip list Co-Authored-By: Claude Sonnet 4.6 * [ONNX] Clarify skip guard is torch == 2.13.x only (not >=) Update comment and skip reason strings to say `torch == 2.13` instead of `torch >= 2.13`, making it clear the guard auto-lifts on 2.14+. Co-Authored-By: Claude Sonnet 4.6 * [ONNX] Fix the torch 2.13 scalar regressions instead of skipping the models Both regressions are a Python float meeting an integral tensor, whose promotion torch 2.13 mishandles: `1.0 - int_mask` crashes the decomposition pass (pytorch/pytorch#194381) and `int_mask * 2.0` reaches translation with no registered ONNX decomposition (pytorch/pytorch#194382). On the same torch, `1.0 - float_tensor` and `float_tensor * 2.0` export fine — so promoting the tensor operand up front, to the dtype the op already produces, is enough. The op and its overload are left alone, and the inserted cast carries the op's own `meta` because for these elementwise cases it is the same value. Decomposition also emits `mul.Scalar` with a *symbolic* second operand (a division result, not a literal). There is no constant to promote there, so that one is rewritten to `mul.Tensor`, which has the two-operand translation — the same rewrite `_fix_remainder_scalar` makes for the same reason. Reachable because the FX fixes run again right after `run_decompositions`. This drops the 39 skip entries: the 15 affected families export again on 2.13, including the vision models (Sam, SamHQ, GotOcr2, GroundingDino, SegGpt, EfficientLoFTR, DeepseekOcr2). Co-Authored-By: Claude Opus 5 * Resolve the promoted-op set on first use, not at import `exporter_onnx` is importable without torch — the CI job that imports transformers with PIL only proved this the hard way — and naming `torch.ops.aten.*` overloads in a module-level frozenset broke that with a `NameError` before anything ran. Co-Authored-By: Claude Opus 5 * Say what the `mul.Scalar` operand is, and check it Review raised two things. The `Node` check assumed a tensor operand: it is never one. Across the affected families all 33 sites are an `operator.truediv` result, i.e. a `SymFloat`, which `mul.Tensor`'s translation does take — so the rewrite stands, but the guard now names the forms that op accepts instead of trusting the node type, and anything else keeps the `mul.Scalar` overload and fails visibly in translation. The fixes are also deliberately not version-gated, which the docstring implied they were. Both rewrites are semantics-preserving on any torch — a cast to the dtype the op already produces, and an overload swap with the same meaning — so gating would only decide which torch exercises the path. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: ydshieh Co-authored-by: Claude Sonnet 4.6 Co-authored-by: IlyasMoutawwakil Co-authored-by: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com> --- src/transformers/exporters/exporter_onnx.py | 102 ++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/transformers/exporters/exporter_onnx.py b/src/transformers/exporters/exporter_onnx.py index 504ac4085d5a..2da0a286c316 100644 --- a/src/transformers/exporters/exporter_onnx.py +++ b/src/transformers/exporters/exporter_onnx.py @@ -727,6 +727,108 @@ def _fix_sort_stable(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: return True +@functools.cache +def _integral_scalar_promotion_ops() -> frozenset: + """Ops where a Python float meeting an integral tensor needs the tensor promoted first. + + Named for torch 2.13, where the mishandling appeared, but applied on every version: both rewrites are + semantics-preserving — a cast to the dtype the op already produces, and an overload swap with the same + meaning — so gating them on a version would add a branch that changes nothing except which torch the + path is exercised on. + + `sub`/`rsub` and `mul` are what the affected models spell (`1.0 - attention_mask`, `mask * 2.0`); both + overloads appear, since decomposition rewrites `.Tensor` to `.Scalar` when the operand is a constant. + + Resolved on first use rather than at import: this module is importable without torch, and naming an + `OpOverload` at module scope breaks that. + """ + return frozenset( + { + torch.ops.aten.rsub.Scalar, + torch.ops.aten.sub.Scalar, + torch.ops.aten.sub.Tensor, + torch.ops.aten.mul.Scalar, + torch.ops.aten.mul.Tensor, + } + ) + + +@register_fx_node_fix("onnx") +def _fix_integral_tensor_float_scalar(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: + """Promote an integral tensor before it meets a Python float, which torch 2.13 mishandles. + + Two torch 2.13 regressions have the same shape — a float scalar against an *integral* tensor, whose + promotion the export pipeline no longer gets right: + + - `1.0 - int_mask` (`aten.rsub.Scalar`) crashes the decomposition pass (pytorch/pytorch#194381), + - `int_mask * 2.0` (`aten.mul.Tensor`, `aten.mul.Scalar` after decomposition) reaches translation with + no ONNX decomposition registered for it (pytorch/pytorch#194382). + + Both go away once the tensor is already the dtype the op produces: `1.0 - float_tensor` and + `float_tensor * 2.0` export fine on the same torch. So rather than rewriting the op — which would mean + building the constant as a tensor and picking the right overload — cast its tensor operand up front and + leave the op alone. The cast's value is the op's own output for these elementwise cases (same shape, + the promoted dtype), so it carries `node.meta` unchanged. + + Self-limiting: once the operand is floating point the predicate no longer matches, so the walk cannot + revisit it. + """ + if node.target not in _integral_scalar_promotion_ops(): + return False + if len(node.args) < 2: + return False + tensor_arg, scalar_arg = node.args[0], node.args[1] + # A tensor on the left, a Python float on the right — a `Node` there is already a real tensor operand. + if not isinstance(tensor_arg, torch.fx.Node) or not isinstance(scalar_arg, float): + return False + operand, result = tensor_arg.meta.get("val"), node.meta.get("val") + if operand is None or result is None: + return False + if operand.dtype.is_floating_point or not result.dtype.is_floating_point: + return False + with gm.graph.inserting_before(node): + promoted = gm.graph.call_function( + torch.ops.aten._to_copy.default, args=(tensor_arg,), kwargs={"dtype": result.dtype} + ) + promoted.meta.update(node.meta) + node.replace_input_with(tensor_arg, promoted) + return True + + +@register_fx_node_fix("onnx") +def _fix_mul_scalar_symbolic(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: + """Rewrite `mul.Scalar` to `mul.Tensor` when its 'scalar' is a graph node. + + The other half of pytorch/pytorch#194382: torchlib registers no real-valued `aten.mul.Scalar` + translation at all, and decomposition also produces that overload with a *symbolic* second operand — a + division result rather than a literal (`mul.Scalar(x, %truediv_1)`) — which the promotion fix above + cannot address, since there is no Python constant to promote against. `mul.Tensor` has the two-operand + translation, which is the same rewrite `_fix_remainder_scalar` makes for the same reason. + + That operand is a `SymFloat`, not a tensor: across the affected families every one of the 33 sites is + an `operator.truediv` result. `mul.Tensor`'s translation takes a symbolic scalar there — it becomes a + graph value like any other — so the rewrite is sound, but the guard below names both accepted forms + rather than trusting that a `Node` implies a tensor. Anything else (a nested list, an unbacked value + with no `val`) is left as `mul.Scalar` to fail visibly in translation instead of silently here. + + Reached because the FX fixes run a second time right after `run_decompositions`, where this overload + appears. + """ + if node.target is not torch.ops.aten.mul.Scalar: + return False + if len(node.args) < 2 or not isinstance(node.args[1], torch.fx.Node): + return False + other = node.args[1].meta.get("val") + if not isinstance(other, (torch.Tensor, torch.SymFloat, torch.SymInt, torch.SymBool)): + return False + with gm.graph.inserting_before(node): + new = gm.graph.call_function(torch.ops.aten.mul.Tensor, args=node.args) + new.meta.update(node.meta) + node.replace_all_uses_with(new) + gm.graph.erase_node(node) + return True + + @register_fx_node_fix("onnx") def _fix_remainder_scalar(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: """Rewrite remainder.Scalar to remainder.Tensor when the 'scalar' arg is actually a tensor. From 155b89935a648278dd38c78184cbc40e6a65f14b Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:15:19 +0200 Subject: [PATCH 4/6] [Qwen3VLMoe] Update `test_small_model_integration_test_batch` expected output (value drift) (#48376) Update Qwen3VLMoe batch integration test expected output (value drift) Co-authored-by: ydshieh --- tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py b/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py index d796ee6182ad..b77626a5b4c9 100644 --- a/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py +++ b/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py @@ -442,8 +442,8 @@ def test_small_model_integration_test_batch(self): output = model.generate(**inputs, max_new_tokens=30, do_sample=False) EXPECTED_DECODED_TEXT = [ - "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a small wild cat native to the grasslands and montane regions", - "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a small wild cat native to the grasslands and montane regions" + "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a wild cat species native to the grasslands and steppes", + "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a wild cat species native to the grasslands and steppes" ] # fmt: skip self.assertEqual( self.processor.batch_decode(output, skip_special_tokens=True), From 83d024e1bfed0d425d20bcde2b46a56b2333906e Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:45:22 +0200 Subject: [PATCH 5/6] [LongcatFlash] Fix test_longcat_generation_cpu: use device_map="cpu" to avoid MoE disk offload issue (#48377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [LongcatFlash] Fix test_longcat_generation_cpu by using device_map="cpu" `device_map="auto"` causes accelerate to offload MoE expert weights to disk, which then fails to reload them due to an internal weight format incompatibility. Since the test already requires large CPU RAM, use `device_map="cpu"` to keep all weights in memory and avoid disk offloading entirely. Co-Authored-By: Claude Sonnet 4.6 * [LongcatFlash] Update golden string and skip test_longcat_generation_cpu on small runners - `test_shortcat_generation`: update expected output to current model output (value drift) - `test_longcat_generation_cpu`: replace `@require_large_cpu_ram` with `@require_torch_accelerator_memory(memory=1100)` — the 562B parameter model requires ~1,047 GiB of bfloat16 weights, far exceeding the CI runner budget (84 GiB single / 168 GiB dual), and disk offloading fails due to MoE weight format incompatibility with accelerate Co-Authored-By: Claude Sonnet 4.6 * remove unused require_large_cpu_ram import Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: ydshieh --- .../longcat_flash/test_modeling_longcat_flash.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/models/longcat_flash/test_modeling_longcat_flash.py b/tests/models/longcat_flash/test_modeling_longcat_flash.py index 25556e6c4165..8721b217801e 100644 --- a/tests/models/longcat_flash/test_modeling_longcat_flash.py +++ b/tests/models/longcat_flash/test_modeling_longcat_flash.py @@ -23,9 +23,9 @@ from transformers.testing_utils import ( require_bitsandbytes, require_flash_attn, - require_large_cpu_ram, require_torch, require_torch_accelerator, + require_torch_accelerator_memory, slow, torch_device, ) @@ -342,12 +342,19 @@ def test_shortcat_generation(self): outputs = self.model.generate(inputs["input_ids"], max_new_tokens=10, do_sample=False) response = self.tokenizer.batch_decode(outputs, skip_special_tokens=False)[0] - expected_output = "[Round 0] USER:Paris is... ASSISTANT: dig年车龄juanaheast稍achaotingupebarebones" + expected_output = "[Round 0] USER:Paris is... ASSISTANT: dig affidavhens酒店的coco toughoireFFIXilateraldrawal" self.assertEqual(response, expected_output) + # `meituan-longcat/LongCat-Flash-Chat` is 562B parameters (~18.6–31.3B activated per token) -- + # ~1,047 GiB of bfloat16 weights. + # + # That far exceeds the budget `device_map="auto"` plans against on the daily CI `a10` runners -- + # 24 GiB of accelerator plus the 60 GiB `CI_CPU_MEMORY_LIMIT_GB` allowance on the single-accelerator + # runner, and 48 + 120 on the two-accelerator one -- so a large portion of the model is placed on + # `"disk"`, and loading dies due to MoE weight format incompatibility with accelerate's disk offload. @slow - @require_large_cpu_ram + @require_torch_accelerator_memory(memory=1100) def test_longcat_generation_cpu(self): # takes absolutely forever and a lot RAM, but allows to test the output in the CI model = LongcatFlashForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) From 6db238a401c38bcd21eb1502f5b0d39f6ae9b3ce Mon Sep 17 00:00:00 2001 From: Qubitium Date: Fri, 28 Aug 2026 05:27:02 +0000 Subject: [PATCH 6/6] chore(pypcre): use pcre for regex in vibevoice conversion script --- src/transformers/models/vibevoice/convert_vibevoice_to_hf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py b/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py index c00c02af9da9..59ab2a9c083f 100644 --- a/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py +++ b/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py @@ -17,10 +17,10 @@ import json import logging import os -import re from pathlib import Path from typing import Any +import pcre as re import torch from huggingface_hub import snapshot_download from safetensors.torch import load_file