diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aefbef55e..12ea88153 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,19 +47,44 @@ jobs: run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s pylint -- --output-format=github test: - name: Test on ${{ matrix.os }} (${{ matrix.python_version }}) + name: Test on ${{ matrix.os }} (${{ matrix.python_version }}) ${{ matrix.test_select }} needs: lint runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15] - python_version: ['3.13'] include: + # Min Python - os: ubuntu-latest python_version: '3.11' + # Max Python - os: ubuntu-latest python_version: '3.14' + - os: ubuntu-latest + python_version: '3.13' + test_select: android + - os: ubuntu-24.04-arm + python_version: '3.13' + - os: windows-latest + python_version: '3.13' + - os: windows-11-arm + python_version: '3.13' + - os: macos-13 + python_version: '3.13' + - os: macos-15 + python_version: '3.13' + - os: macos-13 + python_version: '3.13' + test_select: ios + - os: macos-15 + python_version: '3.13' + test_select: ios + - os: macos-13 + python_version: '3.13' + test_select: android + - os: macos-15 + python_version: '3.13' + test_select: android timeout-minutes: 180 steps: - uses: actions/checkout@v5 @@ -129,10 +154,12 @@ jobs: output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_BUILD_FRONTEND: 'build[uv]' + CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }} + CIBW_PLATFORM: ${{ matrix.test_select }} - name: Run a sample build (GitHub Action, only) uses: ./ + if: matrix.test_select == '' with: package-dir: sample_proj output-dir: wheelhouse_only @@ -152,6 +179,8 @@ jobs: - name: Run a sample build (GitHub Action, config-file) uses: ./ + env: + CIBW_PLATFORM: ${{ matrix.test_select }} with: package-dir: sample_proj output-dir: wheelhouse_config_file @@ -161,9 +190,14 @@ jobs: shell: bash run: | test $(find wheelhouse -name '*.whl' | wc -l) -ge 1 - test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1 test $(find wheelhouse_config_file -name '*.whl' | wc -l) -eq 1 + - name: Check Action artifacts (native build only) + if: matrix.test_select == '' + shell: bash + run: | + test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1 + - uses: actions/upload-artifact@v4 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} @@ -171,7 +205,7 @@ jobs: - name: Test cibuildwheel run: | - uv run --no-sync bin/run_tests.py ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} + uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} emulated-archs: name: Get qemu emulated architectures diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 67dfac458..29dcf60b4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,62 +16,61 @@ pr: - noxfile.py jobs: -- job: linux_311 +- job: tests + strategy: + matrix: + linux_311: + imageName: "ubuntu-latest" + pythonVersion: "3.11" + testSelect: "native" + android_311: + imageName: "ubuntu-latest" + pythonVersion: "3.11" + testSelect: "android" + macos_311: + imageName: "macos-latest" + pythonVersion: "3.11" + testSelect: "native" + ios_311: + imageName: "macos-latest" + pythonVersion: "3.11" + testSelect: "ios" + android_macos_311: + imageName: "macos-latest" + pythonVersion: "3.11" + testSelect: "android" + windows_311: + imageName: "windows-latest" + pythonVersion: "3.11" + testSelect: "native" timeoutInMinutes: 180 - pool: {vmImage: 'ubuntu-latest'} + pool: + vmImage: $(imageName) + steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.11' + versionSpec: $(pythonVersion) + - task: JavaToolInstaller@0 + condition: and(eq(variables['testSelect'], 'android'), eq(variables['Agent.OS'], 'Linux')) inputs: versionSpec: '17' jdkArchitectureOption: 'x64' jdkSourceOption: 'PreInstalled' - - bash: | - docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all - python -m pip install -U pip - python -m pip install -e. --group test - if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then - echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." - export CIBW_ENABLE=all - else - echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))." - fi - python ./bin/run_tests.py -- job: macos_311 - pool: {vmImage: 'macOS-latest'} - timeoutInMinutes: 120 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.11' - - bash: | - python -m pip install -U pip - python -m pip install -e. --group test - if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then - echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." - export CIBW_ENABLE=all - else - echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))." - fi - python ./bin/run_tests.py + - bash: docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all + condition: and(eq(variables['imageName'], 'ubuntu-latest'), eq(variables['testSelect'], 'native')) + displayName: 'Install binfmt on Linux' + + - bash: python -m pip install -U pip && python -m pip install -e. --group test + displayName: 'Update pip and install cibuildwheel --group test' + + - bash: echo "##vso[task.setvariable variable=CIBW_ENABLE;]all" + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + displayName: Set CIBW_ENABLE to all (main branch) -- job: windows_311 - pool: {vmImage: 'windows-latest'} - timeoutInMinutes: 180 - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.11' - bash: | - python -m pip install -U pip - python -m pip install -e. --group test - if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then - echo "INFO: Exporting CIBW_ENABLE=all for main branch test run." - export CIBW_ENABLE=all - else - echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))." - fi - python ./bin/run_tests.py + echo "CIBW_ENABLE = $CIBW_ENABLE" + python ./bin/run_tests.py --test-select $(testSelect) + displayName: 'Run tests' diff --git a/bin/run_tests.py b/bin/run_tests.py index 5c0ad1494..4e742e44b 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -27,6 +27,12 @@ default=default_cpu_count, help="number of processes to use for testing", ) + parser.add_argument( + "--test-select", + choices={"all", "native", "android", "ios", "pyodide"}, + default="all", + help="Either 'native' or 'android'/'ios'/'pyodide'", + ) args = parser.parse_args() # move cwd to the project root @@ -50,20 +56,37 @@ ) # unit tests + print( + "\n\n================================== UNIT TESTS ==================================", + flush=True, + ) unit_test_args = [sys.executable, "-m", "pytest", "unit_test"] - if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux": + if ( + sys.platform.startswith("linux") + and os.environ.get("CIBW_PLATFORM", "linux") == "linux" + and args.test_select in ["all", "native"] + ): # run the docker unit tests only on Linux unit_test_args += ["--run-docker"] if args.run_podman: unit_test_args += ["--run-podman"] + subprocess.run(unit_test_args, check=True) + print( - "\n\n================================== UNIT TESTS ==================================", + "\n\n=========================== SERIAL INTEGRATION TESTS ===========================", flush=True, ) - subprocess.run(unit_test_args, check=True) + + match args.test_select: + case "all": + marks = [] + case "native": + marks = ["not pyodide", "not android", "not ios"] + case mark: + marks = [f"{mark}"] # Run the serial integration tests without multiple processes serial_integration_test_args = [ @@ -71,7 +94,7 @@ "-m", "pytest", "-m", - "serial", + f"{' and '.join(['serial', *marks])}", "-x", "--durations", "0", @@ -79,19 +102,19 @@ "test", "-vv", ] + + subprocess.run(serial_integration_test_args, check=True) + print( - "\n\n=========================== SERIAL INTEGRATION TESTS ===========================", + "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================", flush=True, ) - subprocess.run(serial_integration_test_args, check=True) - - # Non-serial integration tests integration_test_args = [ sys.executable, "-m", "pytest", "-m", - "not serial", + f"{' and '.join(['not serial', *marks])}", f"--numprocesses={args.num_processes}", "-x", "--durations", @@ -104,8 +127,4 @@ if sys.platform.startswith("linux") and args.run_podman: integration_test_args += ["--run-podman"] - print( - "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================", - flush=True, - ) subprocess.run(integration_test_args, check=True) diff --git a/test/test_android.py b/test/test_android.py index bd9e1a761..8b719d433 100644 --- a/test/test_android.py +++ b/test/test_android.py @@ -90,8 +90,17 @@ def test_android_home(tmp_path, capfd): assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err -# Can fail to setup +# the first build can fail to setup - mark as flaky, and serial to make sure it runs first +@pytest.mark.serial @pytest.mark.flaky(reruns=2) +def test_expected_wheels(tmp_path): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"}) + assert wheels == expected_wheels( + "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi + ) + + def test_frontend_good(tmp_path): new_c_project().generate(tmp_path) wheels = cibuildwheel_run( @@ -112,14 +121,6 @@ def test_frontend_bad(frontend, tmp_path, capfd): assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err -def test_expected_wheels(tmp_path): - new_c_project().generate(tmp_path) - wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"}) - assert wheels == expected_wheels( - "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi - ) - - @needs_emulator def test_archs(tmp_path, capfd): new_c_project().generate(tmp_path)