diff --git a/src/transformers/exporters/exporter_onnx.py b/src/transformers/exporters/exporter_onnx.py index 504ac4085d5a..2da0a286c316 100644 --- a/src/transformers/exporters/exporter_onnx.py +++ b/src/transformers/exporters/exporter_onnx.py @@ -727,6 +727,108 @@ def _fix_sort_stable(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: return True +@functools.cache +def _integral_scalar_promotion_ops() -> frozenset: + """Ops where a Python float meeting an integral tensor needs the tensor promoted first. + + Named for torch 2.13, where the mishandling appeared, but applied on every version: both rewrites are + semantics-preserving — a cast to the dtype the op already produces, and an overload swap with the same + meaning — so gating them on a version would add a branch that changes nothing except which torch the + path is exercised on. + + `sub`/`rsub` and `mul` are what the affected models spell (`1.0 - attention_mask`, `mask * 2.0`); both + overloads appear, since decomposition rewrites `.Tensor` to `.Scalar` when the operand is a constant. + + Resolved on first use rather than at import: this module is importable without torch, and naming an + `OpOverload` at module scope breaks that. + """ + return frozenset( + { + torch.ops.aten.rsub.Scalar, + torch.ops.aten.sub.Scalar, + torch.ops.aten.sub.Tensor, + torch.ops.aten.mul.Scalar, + torch.ops.aten.mul.Tensor, + } + ) + + +@register_fx_node_fix("onnx") +def _fix_integral_tensor_float_scalar(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: + """Promote an integral tensor before it meets a Python float, which torch 2.13 mishandles. + + Two torch 2.13 regressions have the same shape — a float scalar against an *integral* tensor, whose + promotion the export pipeline no longer gets right: + + - `1.0 - int_mask` (`aten.rsub.Scalar`) crashes the decomposition pass (pytorch/pytorch#194381), + - `int_mask * 2.0` (`aten.mul.Tensor`, `aten.mul.Scalar` after decomposition) reaches translation with + no ONNX decomposition registered for it (pytorch/pytorch#194382). + + Both go away once the tensor is already the dtype the op produces: `1.0 - float_tensor` and + `float_tensor * 2.0` export fine on the same torch. So rather than rewriting the op — which would mean + building the constant as a tensor and picking the right overload — cast its tensor operand up front and + leave the op alone. The cast's value is the op's own output for these elementwise cases (same shape, + the promoted dtype), so it carries `node.meta` unchanged. + + Self-limiting: once the operand is floating point the predicate no longer matches, so the walk cannot + revisit it. + """ + if node.target not in _integral_scalar_promotion_ops(): + return False + if len(node.args) < 2: + return False + tensor_arg, scalar_arg = node.args[0], node.args[1] + # A tensor on the left, a Python float on the right — a `Node` there is already a real tensor operand. + if not isinstance(tensor_arg, torch.fx.Node) or not isinstance(scalar_arg, float): + return False + operand, result = tensor_arg.meta.get("val"), node.meta.get("val") + if operand is None or result is None: + return False + if operand.dtype.is_floating_point or not result.dtype.is_floating_point: + return False + with gm.graph.inserting_before(node): + promoted = gm.graph.call_function( + torch.ops.aten._to_copy.default, args=(tensor_arg,), kwargs={"dtype": result.dtype} + ) + promoted.meta.update(node.meta) + node.replace_input_with(tensor_arg, promoted) + return True + + +@register_fx_node_fix("onnx") +def _fix_mul_scalar_symbolic(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: + """Rewrite `mul.Scalar` to `mul.Tensor` when its 'scalar' is a graph node. + + The other half of pytorch/pytorch#194382: torchlib registers no real-valued `aten.mul.Scalar` + translation at all, and decomposition also produces that overload with a *symbolic* second operand — a + division result rather than a literal (`mul.Scalar(x, %truediv_1)`) — which the promotion fix above + cannot address, since there is no Python constant to promote against. `mul.Tensor` has the two-operand + translation, which is the same rewrite `_fix_remainder_scalar` makes for the same reason. + + That operand is a `SymFloat`, not a tensor: across the affected families every one of the 33 sites is + an `operator.truediv` result. `mul.Tensor`'s translation takes a symbolic scalar there — it becomes a + graph value like any other — so the rewrite is sound, but the guard below names both accepted forms + rather than trusting that a `Node` implies a tensor. Anything else (a nested list, an unbacked value + with no `val`) is left as `mul.Scalar` to fail visibly in translation instead of silently here. + + Reached because the FX fixes run a second time right after `run_decompositions`, where this overload + appears. + """ + if node.target is not torch.ops.aten.mul.Scalar: + return False + if len(node.args) < 2 or not isinstance(node.args[1], torch.fx.Node): + return False + other = node.args[1].meta.get("val") + if not isinstance(other, (torch.Tensor, torch.SymFloat, torch.SymInt, torch.SymBool)): + return False + with gm.graph.inserting_before(node): + new = gm.graph.call_function(torch.ops.aten.mul.Tensor, args=node.args) + new.meta.update(node.meta) + node.replace_all_uses_with(new) + gm.graph.erase_node(node) + return True + + @register_fx_node_fix("onnx") def _fix_remainder_scalar(gm: torch.fx.GraphModule, node: torch.fx.Node) -> bool: """Rewrite remainder.Scalar to remainder.Tensor when the 'scalar' arg is actually a tensor. diff --git a/src/transformers/modeling_layers.py b/src/transformers/modeling_layers.py index 8cda87ca489b..933e81b5249f 100644 --- a/src/transformers/modeling_layers.py +++ b/src/transformers/modeling_layers.py @@ -14,6 +14,7 @@ from __future__ import annotations import os +import sys from functools import partial from typing import TYPE_CHECKING @@ -605,8 +606,12 @@ def from_pretrained(cls, main_model: PreTrainedModel, device_map=None, **kwargs) # Open the files, get the slices corresponding only to mtp weights, rename them, and load them mtp_state_dict = {} all_pointer = set() + is_mps = device_map is not None and any( + (d.type if isinstance(d, torch.device) else d) == "mps" for d in device_map.values() + ) + backend = "pread" if is_mps or sys.platform == "win32" else "mmap" for file in mtp_files: - file_pointer = safe_open(file, framework="pt", device="cpu") + file_pointer = safe_open(file, framework="pt", device="cpu", backend=backend) all_pointer.add(file_pointer) for k in file_pointer.keys(): # It's one of the mtp weights diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 36eda5cc67dd..8ff9134e0662 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -4512,7 +4512,15 @@ def _load_pretrained_model( (d.type if isinstance(d, torch.device) else d) == "mps" for d in load_config.device_map.values() ) - backend, device = ("pread", "mps") if is_mps else ("mmap", "cpu") + # Use pread on MPS (mmap incompatible) and Windows (mmap reserves + # copy-on-write commit charge for the entire file, exhausting memory + # for large multi-shard checkpoints). + if is_mps: + backend, device = "pread", "mps" + elif sys.platform == "win32": + backend, device = "pread", "cpu" + else: + backend, device = "mmap", "cpu" file_pointer = safe_open(file, framework="pt", device=device, backend=backend) all_pointer.add(file_pointer) for k in file_pointer.keys(): diff --git a/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py b/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py index c00c02af9da9..59ab2a9c083f 100644 --- a/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py +++ b/src/transformers/models/vibevoice/convert_vibevoice_to_hf.py @@ -17,10 +17,10 @@ import json import logging import os -import re from pathlib import Path from typing import Any +import pcre as re import torch from huggingface_hub import snapshot_download from safetensors.torch import load_file diff --git a/tests/models/longcat_flash/test_modeling_longcat_flash.py b/tests/models/longcat_flash/test_modeling_longcat_flash.py index 25556e6c4165..8721b217801e 100644 --- a/tests/models/longcat_flash/test_modeling_longcat_flash.py +++ b/tests/models/longcat_flash/test_modeling_longcat_flash.py @@ -23,9 +23,9 @@ from transformers.testing_utils import ( require_bitsandbytes, require_flash_attn, - require_large_cpu_ram, require_torch, require_torch_accelerator, + require_torch_accelerator_memory, slow, torch_device, ) @@ -342,12 +342,19 @@ def test_shortcat_generation(self): outputs = self.model.generate(inputs["input_ids"], max_new_tokens=10, do_sample=False) response = self.tokenizer.batch_decode(outputs, skip_special_tokens=False)[0] - expected_output = "[Round 0] USER:Paris is... ASSISTANT: dig年车龄juanaheast稍achaotingupebarebones" + expected_output = "[Round 0] USER:Paris is... ASSISTANT: dig affidavhens酒店的coco toughoireFFIXilateraldrawal" self.assertEqual(response, expected_output) + # `meituan-longcat/LongCat-Flash-Chat` is 562B parameters (~18.6–31.3B activated per token) -- + # ~1,047 GiB of bfloat16 weights. + # + # That far exceeds the budget `device_map="auto"` plans against on the daily CI `a10` runners -- + # 24 GiB of accelerator plus the 60 GiB `CI_CPU_MEMORY_LIMIT_GB` allowance on the single-accelerator + # runner, and 48 + 120 on the two-accelerator one -- so a large portion of the model is placed on + # `"disk"`, and loading dies due to MoE weight format incompatibility with accelerate's disk offload. @slow - @require_large_cpu_ram + @require_torch_accelerator_memory(memory=1100) def test_longcat_generation_cpu(self): # takes absolutely forever and a lot RAM, but allows to test the output in the CI model = LongcatFlashForCausalLM.from_pretrained(self.model_id, device_map="auto", dtype=torch.bfloat16) diff --git a/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py b/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py index d796ee6182ad..b77626a5b4c9 100644 --- a/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py +++ b/tests/models/qwen3_vl_moe/test_modeling_qwen3_vl_moe.py @@ -442,8 +442,8 @@ def test_small_model_integration_test_batch(self): output = model.generate(**inputs, max_new_tokens=30, do_sample=False) EXPECTED_DECODED_TEXT = [ - "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a small wild cat native to the grasslands and montane regions", - "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a small wild cat native to the grasslands and montane regions" + "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a wild cat species native to the grasslands and steppes", + "user\nWhat kind of dog is this?\nassistant\nThis is a Pallas's cat, also known as the manul. It's a wild cat species native to the grasslands and steppes" ] # fmt: skip self.assertEqual( self.processor.batch_decode(output, skip_special_tokens=True), diff --git a/tests/utils/test_get_previous_daily_ci.py b/tests/utils/test_get_previous_daily_ci.py new file mode 100644 index 000000000000..f5111b0e5d24 --- /dev/null +++ b/tests/utils/test_get_previous_daily_ci.py @@ -0,0 +1,209 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest +from unittest.mock import patch + + +# utils/ is not an installable package; add it to the path so CI utility modules +# can be imported without modifying the installed transformers package. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "utils")) + +from get_previous_daily_ci import get_daily_ci_runs # noqa: E402 + + +_WORKFLOW_ID = 90575235 +_CURRENT_RUN_ID = 33038227895 + +# Simulates get_github_json("/runs/{GITHUB_RUN_ID}") — same workflow as the one being queried. +_CURRENT_RUN_RESPONSE = {"id": _CURRENT_RUN_ID, "workflow_id": _WORKFLOW_ID} + +# Simulates a current run belonging to a *different* workflow (e.g. AMD CI querying Nvidia). +_DIFFERENT_WORKFLOW_RUN_RESPONSE = {"id": 11111111111, "workflow_id": 99999} + +_STALE_RESPONSE = { + "total_count": 209, + "workflow_runs": [ + { + "id": 30781855254, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-03T03:27:23Z", + "event": "schedule", + }, + { + "id": 30730647697, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-02T03:27:19Z", + "event": "schedule", + }, + ], +} + +_FRESH_RESPONSE = { + "total_count": 413, + "workflow_runs": [ + { + "id": _CURRENT_RUN_ID, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-27T04:03:26Z", + "event": "schedule", + }, + { + "id": 32923823246, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-26T02:44:02Z", + "event": "schedule", + }, + ], +} + + +class GetDailyCiRunsRetryTest(unittest.TestCase): + """Unit tests for the stale-cache retry logic in get_daily_ci_runs. + + get_github_json is called once up-front to fetch the current run's workflow_id + (for stale-check eligibility), then once per schedule-query attempt. Tests use + side_effect lists ordered as: [current-run lookup, attempt-1, attempt-2, ...]. + """ + + def test_fresh_on_first_attempt_no_retry(self): + """When the first response contains the current run, return immediately without retrying.""" + with ( + patch("get_previous_daily_ci.get_github_json", side_effect=[_CURRENT_RUN_RESPONSE, _FRESH_RESPONSE]), + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + mock_sleep.assert_not_called() + self.assertEqual(runs[0]["id"], _CURRENT_RUN_ID) + + def test_stale_then_fresh_retries_once(self): + """A stale first response triggers one retry that returns fresh data.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE, _STALE_RESPONSE, _FRESH_RESPONSE], + ), + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + mock_sleep.assert_called_once_with(30) + self.assertEqual(runs[0]["id"], _CURRENT_RUN_ID) + + def test_all_stale_exhausts_max_attempts(self): + """When all attempts return stale data, proceed after max_attempts with the last result.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE] + [_STALE_RESPONSE] * 5, + ) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # 1 current-run lookup + 5 schedule queries (max_attempts=5) + self.assertEqual(mock_api.call_count, 6) + # sleep between attempts 1-2, 2-3, 3-4, 4-5 — not after the last attempt + self.assertEqual(mock_sleep.call_count, 4) + mock_sleep.assert_called_with(30) + # Stale data is returned as a graceful fallback + self.assertEqual(runs[0]["id"], 30781855254) + + def test_different_workflow_skips_stale_check(self): + """When workflow_id differs from the current run's, stale check is skipped entirely.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_DIFFERENT_WORKFLOW_RUN_RESPONSE, _STALE_RESPONSE], + ) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": "11111111111", "GITHUB_EVENT_NAME": "schedule"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # 1 current-run lookup + 1 schedule query (max_attempts=1, no retries) + self.assertEqual(mock_api.call_count, 2) + mock_sleep.assert_not_called() + # Returns whatever the single attempt gave (stale in this case) + self.assertEqual(runs[0]["id"], 30781855254) + + def test_non_schedule_event_skips_stale_check(self): + """When the triggering event is not 'schedule' (e.g. push), stale check is skipped.""" + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE, _STALE_RESPONSE], + ) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID), "GITHUB_EVENT_NAME": "push"}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # 1 current-run lookup + 1 schedule query (max_attempts=1, no retries) + self.assertEqual(mock_api.call_count, 2) + mock_sleep.assert_not_called() + self.assertEqual(runs[0]["id"], 30781855254) + + def test_no_github_run_id_skips_stale_check(self): + """With GITHUB_RUN_ID unset, the current-run lookup is skipped and no retry is done.""" + with ( + patch("get_previous_daily_ci.get_github_json", side_effect=[_STALE_RESPONSE]) as mock_api, + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {}, clear=False), + ): + os.environ.pop("GITHUB_RUN_ID", None) + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + # No current-run lookup, 1 schedule query, no retries + mock_api.assert_called_once() + mock_sleep.assert_not_called() + self.assertEqual(runs[0]["id"], 30781855254) + + def test_empty_schedule_falls_back_to_workflow_run(self): + """When event=schedule returns no runs, fall back to event=workflow_run without retrying.""" + fallback_response = { + "total_count": 5, + "workflow_runs": [ + { + "id": _CURRENT_RUN_ID, + "status": "completed", + "conclusion": "failure", + "created_at": "2026-08-27T04:03:26Z", + "event": "workflow_run", + }, + ], + } + with ( + patch( + "get_previous_daily_ci.get_github_json", + side_effect=[_CURRENT_RUN_RESPONSE, {"total_count": 0, "workflow_runs": []}, fallback_response], + ), + patch("get_previous_daily_ci.time.sleep") as mock_sleep, + patch.dict("os.environ", {"GITHUB_RUN_ID": str(_CURRENT_RUN_ID)}), + ): + runs = get_daily_ci_runs(token="tok", workflow_id=_WORKFLOW_ID) + + mock_sleep.assert_not_called() + self.assertEqual(len(runs), 1) + self.assertEqual(runs[0]["event"], "workflow_run") diff --git a/utils/get_previous_daily_ci.py b/utils/get_previous_daily_ci.py index 290213e8a277..a8af937f4e67 100644 --- a/utils/get_previous_daily_ci.py +++ b/utils/get_previous_daily_ci.py @@ -1,50 +1,94 @@ import os +import time import zipfile from get_ci_error_statistics import download_artifact, get_artifacts_links, get_github_json def get_daily_ci_runs(token, num_runs=7, workflow_id=None): - """Get the workflow runs of the scheduled (daily) CI. + """Get the most recent workflow runs of the scheduled (daily) CI on the main branch. - This only selects the runs triggered by the `schedule` event on the `main` branch. + Queries event=schedule; falls back to event=workflow_run when no results are found + (AMD CI is triggered via workflow_run, not schedule). Retries on stale GitHub API + responses — see PR #48374 for background. """ - # The id of a workflow (not of a workflow run). - # From a given workflow run (where we have workflow run id), we can get the workflow id by going to - # https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id} - # and check the `workflow_id` key. - - if not workflow_id: - workflow_run_id = os.environ["GITHUB_RUN_ID"] - workflow_run = get_github_json( - f"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}", token=token + # Fetch current run metadata to (1) resolve workflow_id when not given, and (2) compare + # workflow IDs to decide whether stale-cache detection applies (see stale_check below). + current_run_id = int(os.environ.get("GITHUB_RUN_ID", 0)) + current_workflow_id = None + if current_run_id: + current_run = get_github_json( + f"https://api.github.com/repos/huggingface/transformers/actions/runs/{current_run_id}", token=token ) - workflow_id = workflow_run["workflow_id"] + current_workflow_id = current_run["workflow_id"] + if not workflow_id: + workflow_id = current_workflow_id url = f"https://api.github.com/repos/huggingface/transformers/actions/workflows/{workflow_id}/runs" - # On `main` branch + event being `schedule` + not returning PRs + only `num_runs` results url += f"?branch=main&exclude_pull_requests=true&per_page={num_runs}" - schedule_url = f"{url}&event=schedule" - print(f"[DEBUG get_daily_ci_runs] Querying: {schedule_url}") - result = get_github_json(schedule_url, token=token) - workflow_runs = result["workflow_runs"] - print(f"[DEBUG get_daily_ci_runs] event=schedule returned {len(workflow_runs)} runs:") - for r in workflow_runs: + # The GitHub Actions search index (used for event=/branch= filters) can lag badly: + # the same URL has returned total_count values of 190, 238, 311, and 413 within minutes + # from different backend nodes (PR #48374). We detect a stale response by checking + # that GITHUB_RUN_ID appears in the results; if absent we retry. + # + # Stale-check is only active when querying the *same* workflow as the current run AND + # the current run is schedule-triggered (so GITHUB_RUN_ID is guaranteed to be in the + # results when the API is fresh). The two remaining uncovered paths are left for a + # follow-up PR once this path is confirmed stable: + # • AMD CI querying its own history → falls into the event=workflow_run fallback below + # (AMD CI is triggered via workflow_run, not schedule). + # • AMD CI querying the matching Nvidia run (different workflow_id) → stale_check=False. + stale_check = ( + current_workflow_id is not None + and int(workflow_id) == int(current_workflow_id) + and os.environ.get("GITHUB_EVENT_NAME") == "schedule" + ) + max_attempts = 5 if stale_check else 1 + + for attempt in range(1, max_attempts + 1): + schedule_url = f"{url}&event=schedule" + print(f"[DEBUG get_daily_ci_runs] Querying (attempt {attempt}/{max_attempts}): {schedule_url}") + result = get_github_json(schedule_url, token=token) + workflow_runs = result["workflow_runs"] print( - f" id={r['id']} status={r['status']} conclusion={r.get('conclusion')} created_at={r['created_at']} event={r['event']}" + f"[DEBUG get_daily_ci_runs] event=schedule returned {len(workflow_runs)} runs (total_count={result.get('total_count')}):" ) - if len(workflow_runs) == 0: - workflow_run_url = f"{url}&event=workflow_run" - print(f"[DEBUG get_daily_ci_runs] Falling back to: {workflow_run_url}") - result = get_github_json(workflow_run_url, token=token) - workflow_runs = result["workflow_runs"] - print(f"[DEBUG get_daily_ci_runs] event=workflow_run returned {len(workflow_runs)} runs:") for r in workflow_runs: print( f" id={r['id']} status={r['status']} conclusion={r.get('conclusion')} created_at={r['created_at']} event={r['event']}" ) + if len(workflow_runs) == 0: + # AMD CI runs appear under event=workflow_run (triggered via the workflow_run + # event, not schedule). Stale-check for this path is TODO (see above). + workflow_run_url = f"{url}&event=workflow_run" + print(f"[DEBUG get_daily_ci_runs] Falling back to: {workflow_run_url}") + result = get_github_json(workflow_run_url, token=token) + workflow_runs = result["workflow_runs"] + print(f"[DEBUG get_daily_ci_runs] event=workflow_run returned {len(workflow_runs)} runs:") + for r in workflow_runs: + print( + f" id={r['id']} status={r['status']} conclusion={r.get('conclusion')} created_at={r['created_at']} event={r['event']}" + ) + break + + if stale_check and current_run_id not in {r["id"] for r in workflow_runs}: + if attempt < max_attempts: + print( + f"[WARN get_daily_ci_runs] Current run {current_run_id} not found in results — " + f"likely a stale GitHub API cache (total_count={result.get('total_count')}). " + f"Retrying in 30s..." + ) + time.sleep(30) + continue + else: + print( + f"[WARN get_daily_ci_runs] Current run {current_run_id} still not found after " + f"{max_attempts} attempts — proceeding with stale results." + ) + break + return workflow_runs