Fangyangci/add failed type (#1357)

Author: fangyangci

.github/workflows/docs.yml

@@ -28,7 +28,7 @@ jobs:
         with:
           python-version: "3.11"
 
-      - run: uv sync --locked --only-group docs
+      - run: uv sync --only-group docs
 
       - name: Configure git for mike
         run: |

.github/workflows/lint.yml

@@ -27,7 +27,7 @@ jobs:
         run: uv python install 3.11
 
       - name: Install dependencies
-        run: uv sync --locked --all-extras --all-groups
+        run: uv sync --all-extras --all-groups
 
       - name: Check license headers
         run: |

.github/workflows/modelkit-ci.yml

@@ -58,7 +58,7 @@ jobs:
           node-version: 22
 
       - name: Install dependencies
-        run: uv sync --locked --all-extras
+        run: uv sync --all-extras
 
       - name: Run tests (${{ matrix.group }})
         shell: pwsh

.gitignore

@@ -280,3 +280,4 @@ src/winml/modelkit/analyze/rules/runtime_check_rules/**/*.parquet
 
 # Generated by mike (docs versioning)
 docs/versions.json
+/uv.lock

CONTRIBUTING.md

@@ -16,7 +16,7 @@ See the [README](./README.md#getting-started) for prerequisites and installation
 ```bash
 git clone https://github.com/microsoft/winml-cli.git
 cd winml-cli
-uv sync --locked
+uv sync
 uv run pre-commit install
 ```
 

docs/README.md

@@ -6,7 +6,7 @@ This folder hosts the source for the [winml-cli](https://github.com/microsoft/wi
 
 | Task | Command |
 |---|---|
-| Install dev deps | `uv sync --locked` |
+| Install dev deps | `uv sync` |
 | Live preview | `uv run mkdocs serve` |
 | Build for CI | `uv run mkdocs build --strict` |
 | Publish (one-shot from laptop) | `uv run mkdocs gh-deploy --force` |
@@ -48,7 +48,7 @@ Python 3.10+ and [uv](https://github.com/astral-sh/uv).
 
 ```bash
 # from the repo root
-uv sync --locked
+uv sync
 uv run mkdocs serve
 ```
 

docs/contributing.md

@@ -8,7 +8,7 @@ For the full contributing guide — development setup, coding conventions, testi
 # Clone and set up
 git clone https://github.com/microsoft/winml-cli.git
 cd winml-cli
-uv sync --locked
+uv sync
 uv run pre-commit install
 
 # Download runtime check rules (required for `winml analyze`)

docs/getting-started/installation.md

@@ -27,7 +27,7 @@ uv pip install winml-cli
 
     ```powershell
     uv python install cpython-3.11-windows-x86_64-none
-    uv sync --locked --python cpython-3.11-windows-x86_64-none
+    uv sync --python cpython-3.11-windows-x86_64-none
     ```
 
     The x64 interpreter runs under Windows emulation on Arm64 hardware; `winml sys` still reports the Arm64 machine, and the NPU/GPU/CPU providers work normally.

pyproject.toml

@@ -51,12 +51,14 @@ dependencies = [
   # form of what WinML's wasdk metadata pins as `1.24.5.post202604171637` (PEP 440
   # normalized) — an exact `==` can't reconcile the two. A bounded range keeps the
   # 1.24.x series while resolving to the published wheel.
-  "onnxruntime-windowsml>=1.24.5,<1.25 ; sys_platform == 'win32'",
+  "onnxruntime-windowsml==1.24.5.202604171637 ; sys_platform == 'win32'",
   "onnxscript>=0.2",
   "opentelemetry-sdk>=1.39.1",
   "optimum>=2",
   "optimum-onnx>=0.1",
-  "plotext>=5.3.2",
+  # plotext 6.0 was recently released with breaking API changes that conflict
+  # with the current live chart code. Upgrade the code and dependency to >=6 together.
+  "plotext>=5.3.2,<6",
   # PINNED EXACTLY ON PURPOSE — do NOT relax to a range.
   # pandas version drift changes parquet string typing/metadata, producing
   # non-functional but huge diffs across every generated rule artifact. If you
@@ -88,7 +90,9 @@ dependencies = [
 optional-dependencies.audio = [ "soundfile>=0.13" ]
 optional-dependencies.openvino = [ "openvino>=2023" ]
 optional-dependencies.qnn = [
-  "onnxruntime-qnn>=1.24.1; python_version>='3.11'",
+  # 2.1+ is a plugin-only wheel under ``onnxruntime_qnn``. Older releases
+  # also install the ``onnxruntime`` package and overwrite Windows ML files.
+  "onnxruntime-qnn>=2.1; python_version>='3.11'",
 ]
 urls."Bug Tracker" = "https://github.com/microsoft/winml-cli/issues"
 urls.Documentation = "https://github.com/microsoft/winml-cli/blob/main/README.md"
@@ -119,19 +123,12 @@ environments = [
   "sys_platform == 'linux' and platform_machine == 'x86_64'",
   "sys_platform == 'win32' and platform_machine == 'AMD64'",
 ]
-# Overrides:
-# 1. Exercise transformers 5.x in development while keeping the published
-#    `transformers>=4.57` metadata compatible with optimum-onnx 0.1.0's
-#    `transformers<4.58.0` ceiling. The transformers_compat shim bridges the
-#    few 4.x internals optimum-onnx reaches for under transformers 5.
-# 2. Neutralize onnxruntime-qnn's transitive standard `onnxruntime` requirement.
-#    onnxruntime-windowsml ships the same `onnxruntime/` module; installing plain
-#    onnxruntime alongside it overwrites windowsml's files on disk and kills the
-#    DirectML EP. The `sys_platform == 'unobtainium'` marker is never true, so uv
-#    never installs plain onnxruntime while still keeping onnxruntime-qnn.
-override-dependencies = [
-  "transformers>=5,<6",
-  "onnxruntime ; sys_platform == 'unobtainium'",
+# The QNN plugin declares standard ``onnxruntime`` as a runtime dependency,
+# but WinML CLI uses the API-compatible ``onnxruntime-windowsml`` build.
+# Installing both distributions corrupts their shared ``onnxruntime`` package
+# tree and makes the active CPU/DML runtime depend on installation order.
+exclude-dependencies = [
+  { package = { name = "onnxruntime-qnn" }, dependencies = [ "onnxruntime" ] },
 ]
 
 [[tool.uv.index]]

scripts/e2e_eval/README.md

@@ -95,8 +95,8 @@ uv run python scripts/e2e_eval/run_eval.py --continue
 # Backfill accuracy onto an existing perf-only batch (reuses cached perf)
 uv run python scripts/e2e_eval/run_eval.py --eval-type both --continue
 
-# Retry only ENVIRONMENT failures (disk/network issues)
-uv run python scripts/e2e_eval/run_eval.py --retry-failed ENVIRONMENT UNKNOWN
+# Retry only retryable infra/download failures
+uv run python scripts/e2e_eval/run_eval.py --retry-failed HF_FETCH_FAIL ENVIRONMENT UNKNOWN
 
 # Retry ALL failed jobs
 uv run python scripts/e2e_eval/run_eval.py --retry-failed
@@ -121,11 +121,12 @@ uv run python scripts/e2e_eval/run_eval.py --update-baseline --eval-type accurac
 | `--device` | `auto` | Target device |
 | `--ep` | — | Execution provider (e.g. `qnn`, `dml`, `openvino`); applied at perf/eval time |
 | `--timeout` | 600 | Per-subprocess timeout (seconds) |
+| `--clean-cache [TARGET ...]` | off | Clean caches after each job. `TARGET`: `winml`, `huggingface`, `others` (others = VitisAI cache + temp/cwd leaked scratch files). Use `--clean-cache` without TARGET to clear all (legacy behavior). |
 | `--update-baseline` | off | Offline mode: refresh `cache/baseline_cache.json` via the PyTorch baseline, then exit (no build/perf/eval) |
 | `--list` | off | List models and exit |
 | `--verbose` | off | Print stderr for failed models |
 | `--continue` | off | Skip jobs with existing results (but backfill accuracy onto perf-only results when `--eval-type` wants it) |
-| `--retry-failed [TYPE ...]` | — | Re-run failed jobs (implies `--continue`) |
+| `--retry-failed [TYPE ...]` | — | Re-run failed jobs (implies `--continue`); unknown types are rejected as argument errors. Retry criteria are not mutually exclusive: `HF_FETCH_FAIL` also checks failed perf and accuracy logs for `WinError 10060`, `we couldn't connect to 'https://huggingface.co'`, or `thrown while requesting HEAD https://huggingface.co`, even when the primary perf classification is another type or accuracy is `FAIL`. |
 | `--build-only` | off | Build with `--no-compile`, writing each stage's ONNX (no EP needed). Loops the EP matrix when `--ep`/`--device` omitted |
 
 ### `run_llm_eval.py` — Run GenAI Context Sweep
@@ -334,7 +335,8 @@ Groups (`Foundry Toolkit`, `Benchmark`, `ISV`, `microsoft`, `Top200`, …) categ
 
 ### Failure Classification
 
-Failures are classified from `stdout + stderr` pattern matching (ordered by pipeline stage):
+Perf failures receive one primary classification from `stdout + stderr` pattern
+matching (ordered by pipeline stage; first match wins):
 
 | Type | Stage |
 |---|---|
@@ -343,13 +345,20 @@ Failures are classified from `stdout + stderr` pattern matching (ordered by pipe
 | `OPT_FAIL` | Graph optimization |
 | `COMPILE_FAIL` | Compilation / quantization |
 | `RUNTIME_FAIL` | Inference |
+| `HF_FETCH_FAIL` | Hugging Face model/config fetch or cache miss |
 | `ENVIRONMENT` | Disk / network (retryable) |
 | `TIMEOUT` | Exceeded time limit |
 | `UNKNOWN` | No pattern matched |
 
 Classification is **derived on-the-fly** from stored facts — updating classifier
 rules automatically reclassifies all historical results.
 
+`--retry-failed` criteria are not mutually exclusive. In addition to matching
+`HF_FETCH_FAIL` when it is the primary perf classification, the runner scans failed
+perf and accuracy logs for supplemental Hugging Face fetch markers. A job can
+therefore match both `EXPORT_FAIL` and `HF_FETCH_FAIL`, or both accuracy `FAIL` and
+`HF_FETCH_FAIL`; requesting either applicable retry type re-runs it.
+
 ### Output Structure
 
 ```