Merge branch 'main' into pythongh-103501

Author: aisk

.azure-pipelines/ci.yml

@@ -1,4 +1,4 @@
-trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7']
+trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
@@ -11,25 +11,6 @@ jobs:
   - template: ./prebuild-checks.yml
 
 
-- job: Ubuntu_CI_Tests
-  displayName: Ubuntu CI Tests
-  dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
-  pool:
-    vmImage: ubuntu-22.04
-
-  variables:
-    testRunTitle: '$(build.sourceBranchName)-linux'
-    testRunPlatform: linux
-    openssl_version: 1.1.1u
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: apt
-
-
 - job: Windows_CI_Tests
   displayName: Windows CI Tests
   dependsOn: Prebuild

.azure-pipelines/posix-steps.yml

@@ -1,9 +1,3 @@
-parameters:
-  sudo_dependencies: sudo
-  dependencies: apt
-  patchcheck: true
-  xvfb: true
-
 steps:
 - checkout: self
   clean: true
@@ -13,7 +7,7 @@ steps:
 - script: sudo setfacl -Rb /home/vsts
   displayName: 'Workaround ACL issue'
 
-- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version)
+- script: sudo ./.azure-pipelines/posix-deps-apt.sh $(openssl_version)
   displayName: 'Install dependencies'
 
 - script: ./configure --with-pydebug
@@ -25,27 +19,8 @@ steps:
 - script: make pythoninfo
   displayName: 'Display build info'
 
-- script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml"
-  displayName: 'Tests'
-  env:
-    ${{ if eq(parameters.xvfb, 'true') }}:
-      COMMAND: xvfb-run make
-    ${{ if ne(parameters.xvfb, 'true') }}:
-      COMMAND: make
-
-- ${{ if eq(parameters.patchcheck, 'true') }}:
-  - script: |
-      git fetch origin
-      ./python Tools/patchcheck/patchcheck.py --ci true
-    displayName: 'Run patchcheck.py'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))
-
-
-- task: PublishTestResults@2
-  displayName: 'Publish Test Results'
-  inputs:
-    testResultsFiles: '$(build.binariesDirectory)/test-results.xml'
-    mergeTestResults: true
-    testRunTitle: $(testRunTitle)
-    platform: $(testRunPlatform)
-  condition: succeededOrFailed()
+- script: |
+    git fetch origin
+    ./python Tools/patchcheck/patchcheck.py --ci true
+  displayName: 'Run patchcheck.py'
+  condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))

.azure-pipelines/pr.yml

@@ -1,4 +1,4 @@
-pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7']
+pr: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
@@ -11,8 +11,8 @@ jobs:
   - template: ./prebuild-checks.yml
 
 
-- job: Ubuntu_PR_Tests
-  displayName: Ubuntu PR Tests
+- job: Ubuntu_Patchcheck
+  displayName: Ubuntu patchcheck
   dependsOn: Prebuild
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
@@ -26,36 +26,3 @@ jobs:
 
   steps:
   - template: ./posix-steps.yml
-    parameters:
-      dependencies: apt
-
-
-- job: Windows_PR_Tests
-  displayName: Windows PR Tests
-  dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
-
-  pool:
-    vmImage: windows-2022
-
-  strategy:
-    matrix:
-      win32:
-        arch: win32
-        buildOpt: '-p Win32'
-        testRunTitle: '$(System.PullRequest.TargetBranch)-win32'
-        testRunPlatform: win32
-      win64:
-        arch: amd64
-        buildOpt: '-p x64'
-        testRunTitle: '$(System.PullRequest.TargetBranch)-win64'
-        testRunPlatform: win64
-      winarm64:
-        arch: arm64
-        buildOpt: '-p arm64'
-    maxParallel: 4
-
-  steps:
-  - template: ./windows-steps.yml
-    parameters:
-      targetBranch: $(System.PullRequest.TargetBranch)

.editorconfig

@@ -8,5 +8,8 @@ indent_style = space
 [*.{py,c,cpp,h}]
 indent_size = 4
 
+[*.rst]
+indent_size = 3
+
 [*.yml]
 indent_size = 2

.github/workflows/build.yml

@@ -40,6 +40,7 @@ jobs:
       run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
       run_tests: ${{ steps.check.outputs.run_tests }}
       run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
+      run_cifuzz: ${{ steps.check.outputs.run_cifuzz }}
       config_hash: ${{ steps.config_hash.outputs.hash }}
     steps:
       - uses: actions/checkout@v4
@@ -76,6 +77,21 @@ jobs:
             echo "Run hypothesis tests"
             echo "run_hypothesis=true" >> $GITHUB_OUTPUT
           fi
+
+          # oss-fuzz maintains a configuration for fuzzing the main branch of
+          # CPython, so CIFuzz should be run only for code that is likely to be
+          # merged into the main branch; compatibility with older branches may
+          # be broken.
+          FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
+          if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
+            # The tests are pretty slow so they are executed only for PRs
+            # changing relevant files.
+            echo "Run CIFuzz tests"
+            echo "run_cifuzz=true" >> $GITHUB_OUTPUT
+          else
+            echo "Branch too old for CIFuzz tests; or no C files were changed"
+            echo "run_cifuzz=false" >> $GITHUB_OUTPUT
+          fi
       - name: Compute hash for config cache key
         id: config_hash
         run: |
@@ -118,6 +134,8 @@ jobs:
           path: config.cache
           key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
       - uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
       - name: Add ccache to PATH
@@ -138,9 +156,6 @@ jobs:
         run: make regen-configure
       - name: Build CPython
         run: |
-          # Deepfreeze will usually cause global objects to be added or removed,
-          # so we run it before regen-global-objects gets rum (in regen-all).
-          make regen-deepfreeze
           make -j4 regen-all
           make regen-stdlib-module-names
       - name: Check for changes
@@ -180,7 +195,7 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
 
   build_win_amd64:
     name: 'Windows (x64)'
@@ -199,7 +214,22 @@ jobs:
     - name: Display build info
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
+      run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
+
+  build_win_arm64:
+    name: 'Windows (arm64)'
+    runs-on: windows-latest
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_tests == 'true'
+    env:
+       IncludeUwp: 'true'
+    steps:
+    - uses: actions/checkout@v4
+    - name: Register MSVC problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
+    - name: Build CPython
+      run: .\PCbuild\build.bat -e -d -p arm64
 
   build_macos:
     name: 'macOS'
@@ -235,7 +265,7 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: make test
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -244,7 +274,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -302,7 +332,7 @@ jobs:
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
     - name: Tests
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make test
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
@@ -313,7 +343,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1v, 3.0.10, 3.1.2]
+        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -365,7 +395,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -474,7 +504,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 1.1.1v
+      OPENSSL_VER: 3.0.11
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
@@ -518,7 +548,47 @@ jobs:
     - name: Display build info
       run: make pythoninfo
     - name: Tests
-      run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
+      run: xvfb-run make test
+
+  # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+  cifuzz:
+    name: CIFuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_cifuzz == 'true'
+    permissions:
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address, undefined, memory]
+    steps:
+      - name: Build fuzzers (${{ matrix.sanitizer }})
+        id: build
+        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+        with:
+          oss-fuzz-project-name: cpython3
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Run fuzzers (${{ matrix.sanitizer }})
+        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+        with:
+          fuzz-seconds: 600
+          oss-fuzz-project-name: cpython3
+          output-sarif: true
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Upload crash
+        uses: actions/upload-artifact@v3
+        if: failure() && steps.build.outcome == 'success'
+        with:
+          name: ${{ matrix.sanitizer }}-artifacts
+          path: ./out/artifacts
+      - name: Upload SARIF
+        if: always() && steps.build.outcome == 'success'
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: cifuzz-sarif/results.sarif
+          checkout_path: cifuzz-sarif
 
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass
@@ -530,11 +600,13 @@ jobs:
     - check_generated_files
     - build_win32
     - build_win_amd64
+    - build_win_arm64
     - build_macos
     - build_ubuntu
     - build_ubuntu_ssltests
     - test_hypothesis
     - build_asan
+    - cifuzz
 
     runs-on: ubuntu-latest
 
@@ -546,6 +618,8 @@ jobs:
           build_macos,
           build_ubuntu_ssltests,
           build_win32,
+          build_win_arm64,
+          cifuzz,
           test_hypothesis,
         allowed-skips: >-
           ${{
@@ -561,13 +635,21 @@ jobs:
             check_generated_files,
             build_win32,
             build_win_amd64,
+            build_win_arm64,
             build_macos,
             build_ubuntu,
             build_ubuntu_ssltests,
             build_asan,
             '
             || ''
           }}
+          ${{
+            !fromJSON(needs.check_source.outputs.run_cifuzz)
+            && '
+            cifuzz,
+            '
+            || ''
+          }}
           ${{
             !fromJSON(needs.check_source.outputs.run_hypothesis)
             && '

.github/workflows/lint.yml

@@ -7,7 +7,7 @@ permissions:
 
 env:
   FORCE_COLOR: 1
-  RUFF_FORMAT: github
+  RUFF_OUTPUT_FORMAT: github
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}