From 520b750e47335fe5ddf50f4ef62d7c4f42ead7eb Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Mon, 28 Sep 2026 19:09:52 +0200 Subject: [PATCH 1/3] [CI] Run nightly PyTorch model hub scopes on related PRs and on dispatch Nightly PyTorch Models Scope1/Scope2 jobs ran only on schedule, so changes to their model lists, test code or requirements were validated only after merge. Run each scope in pre-commit when a PR changes its files (new Smart CI labels with no dependents), and allow starting them via workflow_dispatch with the pytorch-models-nightly-scope input. Same runner, image and settings. --- .github/components.yml | 8 ++++++ .github/labeler.yml | 31 ++++++++++++++++++++++ .github/workflows/ubuntu_22.yml | 19 +++++++++++-- docs/dev/ci/github_actions/adding_tests.md | 14 ++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/components.yml b/.github/components.yml index 54d1e08d1316da..0341b174281ee3 100644 --- a/.github/components.yml +++ b/.github/components.yml @@ -160,6 +160,14 @@ PyTorch_FE: - Python_API - TOKENIZERS # PyTorch_FE tests depend on tokenizers build +PyTorch_Models_Scope1: + revalidate: [] + build: [] + +PyTorch_Models_Scope2: + revalidate: [] + build: [] + JAX_FE: revalidate: - OVC diff --git a/.github/labeler.yml b/.github/labeler.yml index bb69e95aed0b39..46fd7db545e7a8 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -189,6 +189,37 @@ '!tests/model_hub_tests/jax/**/*', '!tests/model_hub_tests/gguf/**/*'] +# Nightly PyTorch model hub scopes, run in pre-commit only when these files change +'category: PyTorch Models Scope1': +- '.github/workflows/job_pytorch_models_tests.yml' +- 'tests/requirements_pytorch' +- 'tests/model_hub_tests/models_hub_common/**/*' +- 'tests/model_hub_tests/pytorch/conftest.py' +- 'tests/model_hub_tests/pytorch/torch_utils.py' +- 'tests/model_hub_tests/pytorch/scripts/**/*' +- 'tests/model_hub_tests/pytorch/test_timm.py' +- 'tests/model_hub_tests/pytorch/timm_models' +- 'tests/model_hub_tests/pytorch/envs/timm.txt' +- 'tests/model_hub_tests/pytorch/test_torchvision_models.py' +- 'tests/model_hub_tests/pytorch/torchvision_models' +- 'tests/model_hub_tests/pytorch/envs/torchvision.txt' +- 'tests/model_hub_tests/pytorch/test_edsr.py' +- 'tests/model_hub_tests/pytorch/envs/edsr.txt' + +'category: PyTorch Models Scope2': +- '.github/workflows/job_pytorch_models_tests.yml' +- 'tests/requirements_pytorch' +- 'tests/model_hub_tests/models_hub_common/**/*' +- any: ['tests/model_hub_tests/pytorch/**/*', + '!tests/model_hub_tests/pytorch/test_timm.py', + '!tests/model_hub_tests/pytorch/timm_models', + '!tests/model_hub_tests/pytorch/envs/timm.txt', + '!tests/model_hub_tests/pytorch/test_torchvision_models.py', + '!tests/model_hub_tests/pytorch/torchvision_models', + '!tests/model_hub_tests/pytorch/envs/torchvision.txt', + '!tests/model_hub_tests/pytorch/test_edsr.py', + '!tests/model_hub_tests/pytorch/envs/edsr.txt'] + 'category: JAX FE': - 'src/frontends/jax/**/*' - 'src/bindings/python/src/openvino/frontend/jax/**/*' diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 2bac36dbf420a8..4f38789fb2354f 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -12,6 +12,15 @@ on: description: 'Target branch for the build; taken from event context by default' type: string required: false + pytorch-models-nightly-scope: + description: 'Nightly PyTorch model hub scope to run' + type: choice + options: + - none + - scope1 + - scope2 + - all + default: none pull_request: types: - opened @@ -469,7 +478,10 @@ jobs: PyTorch_Models_Tests_Nightly_Scope1: name: PyTorch Models Nightly Scope1 tests - if: ${{ github.event_name == 'schedule' }} + # Scheduled nightly, manual dispatch, or a PR that changes this scope's tests + if: ${{ github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && contains(fromJSON('["scope1", "all"]'), inputs.pytorch-models-nightly-scope)) || + (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Scope1) }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: @@ -479,7 +491,10 @@ jobs: PyTorch_Models_Tests_Nightly_Scope2: name: PyTorch Models Nightly Scope2 tests - if: ${{ github.event_name == 'schedule' }} + # Scheduled nightly, manual dispatch, or a PR that changes this scope's tests + if: ${{ github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && contains(fromJSON('["scope2", "all"]'), inputs.pytorch-models-nightly-scope)) || + (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Scope2) }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: diff --git a/docs/dev/ci/github_actions/adding_tests.md b/docs/dev/ci/github_actions/adding_tests.md index 0463aad4ac7e3f..96c182401c2699 100644 --- a/docs/dev/ci/github_actions/adding_tests.md +++ b/docs/dev/ci/github_actions/adding_tests.md @@ -192,6 +192,20 @@ instead of pre-commit. Refer to the [Overview of the OpenVINO GitHub Actions CI](./overview.md#workflows-triggers-and-schedule) to learn more about triggers and schedules. Additionally, [using a more powerful runner](./runners.md) can help shorten the execution time. +A nightly job can also run in pre-commit only for pull requests that change its tests, and on +manual dispatch. For example, `PyTorch_Models_Tests_Nightly_Scope1/2` in +[ubuntu_22.yml](../../../../.github/workflows/ubuntu_22.yml): +* a dedicated label in [labeler.yml](../../../../.github/labeler.yml) + (`category: PyTorch Models Scope1`) lists the files of the scope; +* the component is added to [components.yml](../../../../.github/components.yml) with empty + `revalidate` and `build`, so it does not extend the scope of other jobs; +* the job condition is + `github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).`. + Use `changed_components`, not `affected_components`. `affected_components` enables every + component for full-scope runs, for example when `.github` files change; +* a `workflow_dispatch` input selects the job for manual runs: + `gh workflow run ubuntu_22.yml --ref -f pytorch-models-nightly-scope=scope1`. + ### Adding a Workflow When adding a new workflow, check the execution times of existing workflows. The new workflow From 0d918a7896bd41d255b1a760583751e7d044b315 Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Mon, 28 Sep 2026 19:18:05 +0200 Subject: [PATCH 2/3] [CI] Use a single component for nightly PyTorch model hub triggers Replace the two scope labels with one PyTorch_Models_Nightly component, use a boolean workflow_dispatch input, and drop the docs change. --- .github/components.yml | 6 +---- .github/labeler.yml | 30 ++-------------------- .github/workflows/ubuntu_22.yml | 25 ++++++------------ docs/dev/ci/github_actions/adding_tests.md | 14 ---------- 4 files changed, 11 insertions(+), 64 deletions(-) diff --git a/.github/components.yml b/.github/components.yml index 0341b174281ee3..6b1bca3fd30128 100644 --- a/.github/components.yml +++ b/.github/components.yml @@ -160,11 +160,7 @@ PyTorch_FE: - Python_API - TOKENIZERS # PyTorch_FE tests depend on tokenizers build -PyTorch_Models_Scope1: - revalidate: [] - build: [] - -PyTorch_Models_Scope2: +PyTorch_Models_Nightly: revalidate: [] build: [] diff --git a/.github/labeler.yml b/.github/labeler.yml index 46fd7db545e7a8..224255e741d79a 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -189,36 +189,10 @@ '!tests/model_hub_tests/jax/**/*', '!tests/model_hub_tests/gguf/**/*'] -# Nightly PyTorch model hub scopes, run in pre-commit only when these files change -'category: PyTorch Models Scope1': +'category: PyTorch Models Nightly': - '.github/workflows/job_pytorch_models_tests.yml' -- 'tests/requirements_pytorch' -- 'tests/model_hub_tests/models_hub_common/**/*' -- 'tests/model_hub_tests/pytorch/conftest.py' -- 'tests/model_hub_tests/pytorch/torch_utils.py' -- 'tests/model_hub_tests/pytorch/scripts/**/*' -- 'tests/model_hub_tests/pytorch/test_timm.py' -- 'tests/model_hub_tests/pytorch/timm_models' -- 'tests/model_hub_tests/pytorch/envs/timm.txt' -- 'tests/model_hub_tests/pytorch/test_torchvision_models.py' -- 'tests/model_hub_tests/pytorch/torchvision_models' -- 'tests/model_hub_tests/pytorch/envs/torchvision.txt' -- 'tests/model_hub_tests/pytorch/test_edsr.py' -- 'tests/model_hub_tests/pytorch/envs/edsr.txt' - -'category: PyTorch Models Scope2': -- '.github/workflows/job_pytorch_models_tests.yml' -- 'tests/requirements_pytorch' - 'tests/model_hub_tests/models_hub_common/**/*' -- any: ['tests/model_hub_tests/pytorch/**/*', - '!tests/model_hub_tests/pytorch/test_timm.py', - '!tests/model_hub_tests/pytorch/timm_models', - '!tests/model_hub_tests/pytorch/envs/timm.txt', - '!tests/model_hub_tests/pytorch/test_torchvision_models.py', - '!tests/model_hub_tests/pytorch/torchvision_models', - '!tests/model_hub_tests/pytorch/envs/torchvision.txt', - '!tests/model_hub_tests/pytorch/test_edsr.py', - '!tests/model_hub_tests/pytorch/envs/edsr.txt'] +- 'tests/model_hub_tests/pytorch/**/*' 'category: JAX FE': - 'src/frontends/jax/**/*' diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 4f38789fb2354f..7ccfb87c9dfb49 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -12,15 +12,10 @@ on: description: 'Target branch for the build; taken from event context by default' type: string required: false - pytorch-models-nightly-scope: - description: 'Nightly PyTorch model hub scope to run' - type: choice - options: - - none - - scope1 - - scope2 - - all - default: none + pytorch-models-nightly: + description: 'Run nightly PyTorch model hub tests' + type: boolean + default: false pull_request: types: - opened @@ -478,10 +473,8 @@ jobs: PyTorch_Models_Tests_Nightly_Scope1: name: PyTorch Models Nightly Scope1 tests - # Scheduled nightly, manual dispatch, or a PR that changes this scope's tests - if: ${{ github.event_name == 'schedule' || - (github.event_name == 'workflow_dispatch' && contains(fromJSON('["scope1", "all"]'), inputs.pytorch-models-nightly-scope)) || - (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Scope1) }} + if: ${{ github.event_name == 'schedule' || inputs.pytorch-models-nightly || + (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Nightly) }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: @@ -491,10 +484,8 @@ jobs: PyTorch_Models_Tests_Nightly_Scope2: name: PyTorch Models Nightly Scope2 tests - # Scheduled nightly, manual dispatch, or a PR that changes this scope's tests - if: ${{ github.event_name == 'schedule' || - (github.event_name == 'workflow_dispatch' && contains(fromJSON('["scope2", "all"]'), inputs.pytorch-models-nightly-scope)) || - (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Scope2) }} + if: ${{ github.event_name == 'schedule' || inputs.pytorch-models-nightly || + (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Nightly) }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: diff --git a/docs/dev/ci/github_actions/adding_tests.md b/docs/dev/ci/github_actions/adding_tests.md index 96c182401c2699..0463aad4ac7e3f 100644 --- a/docs/dev/ci/github_actions/adding_tests.md +++ b/docs/dev/ci/github_actions/adding_tests.md @@ -192,20 +192,6 @@ instead of pre-commit. Refer to the [Overview of the OpenVINO GitHub Actions CI](./overview.md#workflows-triggers-and-schedule) to learn more about triggers and schedules. Additionally, [using a more powerful runner](./runners.md) can help shorten the execution time. -A nightly job can also run in pre-commit only for pull requests that change its tests, and on -manual dispatch. For example, `PyTorch_Models_Tests_Nightly_Scope1/2` in -[ubuntu_22.yml](../../../../.github/workflows/ubuntu_22.yml): -* a dedicated label in [labeler.yml](../../../../.github/labeler.yml) - (`category: PyTorch Models Scope1`) lists the files of the scope; -* the component is added to [components.yml](../../../../.github/components.yml) with empty - `revalidate` and `build`, so it does not extend the scope of other jobs; -* the job condition is - `github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).`. - Use `changed_components`, not `affected_components`. `affected_components` enables every - component for full-scope runs, for example when `.github` files change; -* a `workflow_dispatch` input selects the job for manual runs: - `gh workflow run ubuntu_22.yml --ref -f pytorch-models-nightly-scope=scope1`. - ### Adding a Workflow When adding a new workflow, check the execution times of existing workflows. The new workflow From 17d298d4aeed4e1d861caa0c4042a7732c2701bb Mon Sep 17 00:00:00 2001 From: Maxim Vafin Date: Tue, 29 Sep 2026 11:57:30 +0200 Subject: [PATCH 3/3] [CI] Detect nightly PyTorch model test changes without a new label Check the PR's changed files in the Smart_CI job instead of adding a Smart CI label and component. --- .github/components.yml | 4 ---- .github/labeler.yml | 5 ----- .github/workflows/ubuntu_22.yml | 18 ++++++++++++++++-- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/components.yml b/.github/components.yml index 6b1bca3fd30128..54d1e08d1316da 100644 --- a/.github/components.yml +++ b/.github/components.yml @@ -160,10 +160,6 @@ PyTorch_FE: - Python_API - TOKENIZERS # PyTorch_FE tests depend on tokenizers build -PyTorch_Models_Nightly: - revalidate: [] - build: [] - JAX_FE: revalidate: - OVC diff --git a/.github/labeler.yml b/.github/labeler.yml index 224255e741d79a..bb69e95aed0b39 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -189,11 +189,6 @@ '!tests/model_hub_tests/jax/**/*', '!tests/model_hub_tests/gguf/**/*'] -'category: PyTorch Models Nightly': -- '.github/workflows/job_pytorch_models_tests.yml' -- 'tests/model_hub_tests/models_hub_common/**/*' -- 'tests/model_hub_tests/pytorch/**/*' - 'category: JAX FE': - 'src/frontends/jax/**/*' - 'src/bindings/python/src/openvino/frontend/jax/**/*' diff --git a/.github/workflows/ubuntu_22.yml b/.github/workflows/ubuntu_22.yml index 7ccfb87c9dfb49..eabd9361ac24ad 100644 --- a/.github/workflows/ubuntu_22.yml +++ b/.github/workflows/ubuntu_22.yml @@ -49,6 +49,7 @@ jobs: changed_components: "${{ steps.smart_ci.outputs.changed_components }}" skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}" use_latest_onednn: "${{ steps.onednn_day.outputs.use_latest_onednn }}" + pytorch_models_nightly: "${{ steps.pytorch_models_nightly.outputs.changed }}" steps: - name: checkout action uses: ababushk/checkout@dd591a6a2ac25618db4eda86e7e0d938f88cf01b # cherry_pick_retries @@ -86,6 +87,19 @@ jobs: echo "use_latest_onednn=false" >> "$GITHUB_OUTPUT" fi + - name: Check if nightly PyTorch model tests are changed + id: pytorch_models_nightly + if: ${{ github.event_name == 'pull_request' }} + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.number }} + run: | + files=$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" --jq '.[].filename') + if grep -qE '^(tests/model_hub_tests/(pytorch|models_hub_common)/|\.github/workflows/job_pytorch_models_tests\.yml$)' <<< "$files"; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + Docker: needs: Smart_CI runs-on: aks-linux-4-cores-16gb-docker-build @@ -474,7 +488,7 @@ jobs: PyTorch_Models_Tests_Nightly_Scope1: name: PyTorch Models Nightly Scope1 tests if: ${{ github.event_name == 'schedule' || inputs.pytorch-models-nightly || - (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Nightly) }} + needs.smart_ci.outputs.pytorch_models_nightly == 'true' }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: @@ -485,7 +499,7 @@ jobs: PyTorch_Models_Tests_Nightly_Scope2: name: PyTorch Models Nightly Scope2 tests if: ${{ github.event_name == 'schedule' || inputs.pytorch-models-nightly || - (github.event_name == 'pull_request' && fromJSON(needs.smart_ci.outputs.changed_components).PyTorch_Models_Nightly) }} + needs.smart_ci.outputs.pytorch_models_nightly == 'true' }} needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: