Merge branch 'main' into remove_classmethod_descriptor_chaining

Author: rhettinger

.editorconfig

@@ -8,5 +8,8 @@ indent_style = space
 [*.{py,c,cpp,h}]
 indent_size = 4
 
+[*.rst]
+indent_size = 3
+
 [*.yml]
 indent_size = 2

.github/ISSUE_TEMPLATE/bug.yml

@@ -39,6 +39,7 @@ body:
         - "3.10"
         - "3.11"
         - "3.12"
+        - "3.13"
         - "CPython main branch"
     validations:
       required: true

.github/workflows/build.yml

@@ -40,6 +40,7 @@ jobs:
       run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
       run_tests: ${{ steps.check.outputs.run_tests }}
       run_hypothesis: ${{ steps.check.outputs.run_hypothesis }}
+      run_cifuzz: ${{ steps.check.outputs.run_cifuzz }}
       config_hash: ${{ steps.config_hash.outputs.hash }}
     steps:
       - uses: actions/checkout@v4
@@ -76,6 +77,21 @@ jobs:
             echo "Run hypothesis tests"
             echo "run_hypothesis=true" >> $GITHUB_OUTPUT
           fi
+
+          # oss-fuzz maintains a configuration for fuzzing the main branch of
+          # CPython, so CIFuzz should be run only for code that is likely to be
+          # merged into the main branch; compatibility with older branches may
+          # be broken.
+          FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
+          if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
+            # The tests are pretty slow so they are executed only for PRs
+            # changing relevant files.
+            echo "Run CIFuzz tests"
+            echo "run_cifuzz=true" >> $GITHUB_OUTPUT
+          else
+            echo "Branch too old for CIFuzz tests; or no C files were changed"
+            echo "run_cifuzz=false" >> $GITHUB_OUTPUT
+          fi
       - name: Compute hash for config cache key
         id: config_hash
         run: |
@@ -534,6 +550,46 @@ jobs:
     - name: Tests
       run: xvfb-run make test
 
+  # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+  cifuzz:
+    name: CIFuzz
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    needs: check_source
+    if: needs.check_source.outputs.run_cifuzz == 'true'
+    permissions:
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [address, undefined, memory]
+    steps:
+      - name: Build fuzzers (${{ matrix.sanitizer }})
+        id: build
+        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+        with:
+          oss-fuzz-project-name: cpython3
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Run fuzzers (${{ matrix.sanitizer }})
+        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+        with:
+          fuzz-seconds: 600
+          oss-fuzz-project-name: cpython3
+          output-sarif: true
+          sanitizer: ${{ matrix.sanitizer }}
+      - name: Upload crash
+        uses: actions/upload-artifact@v3
+        if: failure() && steps.build.outcome == 'success'
+        with:
+          name: ${{ matrix.sanitizer }}-artifacts
+          path: ./out/artifacts
+      - name: Upload SARIF
+        if: always() && steps.build.outcome == 'success'
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: cifuzz-sarif/results.sarif
+          checkout_path: cifuzz-sarif
+
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass
     if: always()
@@ -550,6 +606,7 @@ jobs:
     - build_ubuntu_ssltests
     - test_hypothesis
     - build_asan
+    - cifuzz
 
     runs-on: ubuntu-latest
 
@@ -562,6 +619,7 @@ jobs:
           build_ubuntu_ssltests,
           build_win32,
           build_win_arm64,
+          cifuzz,
           test_hypothesis,
         allowed-skips: >-
           ${{
@@ -585,6 +643,13 @@ jobs:
             '
             || ''
           }}
+          ${{
+            !fromJSON(needs.check_source.outputs.run_cifuzz)
+            && '
+            cifuzz,
+            '
+            || ''
+          }}
           ${{
             !fromJSON(needs.check_source.outputs.run_hypothesis)
             && '

.github/workflows/lint.yml

@@ -7,7 +7,7 @@ permissions:
 
 env:
   FORCE_COLOR: 1
-  RUFF_FORMAT: github
+  RUFF_OUTPUT_FORMAT: github
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}

.gitignore

@@ -42,6 +42,7 @@ gmon.out
 .coverage
 .mypy_cache/
 .pytest_cache/
+.ruff_cache/
 .DS_Store
 
 *.exe

.pre-commit-config.yaml

@@ -1,14 +1,18 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.292
+    rev: v0.1.2
     hooks:
       - id: ruff
         name: Run Ruff on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
+      - id: ruff
+        name: Run Ruff on Argument Clinic
+        args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
+        files: ^Tools/clinic/|Lib/test/test_clinic.py
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-toml
         exclude: ^Lib/test/test_tomllib/
@@ -17,12 +21,16 @@ repos:
         types: [python]
         exclude: Lib/test/tokenizedata/coding20731.py
       - id: trailing-whitespace
-        types_or: [c, python, rst]
+        types_or: [c, inc, python, rst]
 
   - repo: https://github.com/sphinx-contrib/sphinx-lint
-    rev: v0.6.8
+    rev: v0.8.1
     hooks:
       - id: sphinx-lint
         args: [--enable=default-role]
         files: ^Doc/|^Misc/NEWS.d/next/
-        types: [rst]
+
+  - repo: meta
+    hooks:
+      - id: check-hooks-apply
+      - id: check-useless-excludes

Doc/Makefile

@@ -7,7 +7,6 @@
 PYTHON       = python3
 VENVDIR      = ./venv
 SPHINXBUILD  = PATH=$(VENVDIR)/bin:$$PATH sphinx-build
-SPHINXLINT   = PATH=$(VENVDIR)/bin:$$PATH sphinx-lint
 BLURB        = PATH=$(VENVDIR)/bin:$$PATH blurb
 JOBS         = auto
 PAPER        =

Doc/c-api/arg.rst

@@ -416,8 +416,10 @@ API Functions
 .. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...)
 
    Parse the parameters of a function that takes both positional and keyword
-   parameters into local variables.  The *keywords* argument is a
-   ``NULL``-terminated array of keyword parameter names.  Empty names denote
+   parameters into local variables.
+   The *keywords* argument is a ``NULL``-terminated array of keyword parameter
+   names specified as null-terminated ASCII or UTF-8 encoded C strings.
+   Empty names denote
    :ref:`positional-only parameters <positional-only_parameter>`.
    Returns true on success; on failure, it returns false and raises the
    appropriate exception.
@@ -426,6 +428,9 @@ API Functions
       Added support for :ref:`positional-only parameters
       <positional-only_parameter>`.
 
+   .. versionchanged:: 3.13
+      Added support for non-ASCII keyword parameter names.
+
 
 .. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs)
 

Doc/c-api/call.rst

@@ -108,6 +108,8 @@ This is a pointer to a function with the following signature:
    Doing so will allow callables such as bound methods to make their onward
    calls (which include a prepended *self* argument) very efficiently.
 
+   .. versionadded:: 3.8
+
 To call an object that implements vectorcall, use a :ref:`call API <capi-call>`
 function as with any other callable.
 :c:func:`PyObject_Vectorcall` will usually be most efficient.

Doc/c-api/init.rst

@@ -870,6 +870,19 @@ code, or when embedding the Python interpreter:
    When the current thread state is ``NULL``, this issues a fatal error (so that
    the caller needn't check for ``NULL``).
 
+   See also :c:func:`PyThreadState_GetUnchecked`.
+
+
+.. c:function:: PyThreadState* PyThreadState_GetUnchecked()
+
+   Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a
+   fatal error if it is NULL. The caller is responsible to check if the result
+   is NULL.
+
+   .. versionadded:: 3.13
+      In Python 3.5 to 3.12, the function was private and known as
+      ``_PyThreadState_UncheckedGet()``.
+
 
 .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate)
 

Doc/c-api/init_config.rst

@@ -716,7 +716,7 @@ PyConfig
 
       Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable.
 
-      Need a special build of Python with the ``Py_TRACE_REFS`` macro defined:
+      Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined:
       see the :option:`configure --with-trace-refs option <--with-trace-refs>`.
 
       Default: ``0``.
@@ -878,6 +878,19 @@ PyConfig
 
       .. versionadded:: 3.12
 
+   .. c:member:: int cpu_count
+
+      If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will
+      override the return values of :func:`os.cpu_count`,
+      :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`.
+
+      Configured by the :samp:`-X cpu_count={n|default}` command line
+      flag or the :envvar:`PYTHON_CPU_COUNT` environment variable.
+
+      Default: ``-1``.
+
+      .. versionadded:: 3.13
+
    .. c:member:: int isolated
 
       If greater than ``0``, enable isolated mode:
@@ -1035,7 +1048,7 @@ PyConfig
       Incremented by the :option:`-d` command line option. Set to the
       :envvar:`PYTHONDEBUG` environment variable value.
 
-      Need a :ref:`debug build of Python <debug-build>` (the ``Py_DEBUG`` macro
+      Needs a :ref:`debug build of Python <debug-build>` (the ``Py_DEBUG`` macro
       must be defined).
 
       Default: ``0``.
@@ -1087,6 +1100,7 @@ PyConfig
 
       Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and
       the :envvar:`PYTHONPYCACHEPREFIX` environment variable.
+      The command-line option takes precedence.
 
       If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``.
 
@@ -1130,13 +1144,27 @@ PyConfig
 
       Default: ``NULL``.
 
+   .. c:member:: wchar_t* run_presite
+
+      ``package.module`` path to module that should be imported before
+      ``site.py`` is run.
+
+      Set by the :option:`-X presite=package.module <-X>` command-line
+      option and the :envvar:`PYTHON_PRESITE` environment variable.
+      The command-line option takes precedence.
+
+      Needs a :ref:`debug build of Python <debug-build>` (the ``Py_DEBUG`` macro
+      must be defined).
+
+      Default: ``NULL``.
+
    .. c:member:: int show_ref_count
 
       Show total reference count at exit (excluding immortal objects)?
 
       Set to ``1`` by :option:`-X showrefcount <-X>` command line option.
 
-      Need a :ref:`debug build of Python <debug-build>` (the ``Py_REF_DEBUG``
+      Needs a :ref:`debug build of Python <debug-build>` (the ``Py_REF_DEBUG``
       macro must be defined).
 
       Default: ``0``.

Doc/c-api/memory.rst

@@ -491,18 +491,18 @@ Customize Memory Allocators
 
        :c:func:`PyMem_SetAllocator` does have the following contract:
 
-        * It can be called after :c:func:`Py_PreInitialize` and before
-          :c:func:`Py_InitializeFromConfig` to install a custom memory
-          allocator. There are no restrictions over the installed allocator
-          other than the ones imposed by the domain (for instance, the Raw
-          Domain allows the allocator to be called without the GIL held). See
-          :ref:`the section on allocator domains <allocator-domains>` for more
-          information.
-
-        * If called after Python has finish initializing (after
-          :c:func:`Py_InitializeFromConfig` has been called) the allocator
-          **must** wrap the existing allocator. Substituting the current
-          allocator for some other arbitrary one is **not supported**.
+       * It can be called after :c:func:`Py_PreInitialize` and before
+         :c:func:`Py_InitializeFromConfig` to install a custom memory
+         allocator. There are no restrictions over the installed allocator
+         other than the ones imposed by the domain (for instance, the Raw
+         Domain allows the allocator to be called without the GIL held). See
+         :ref:`the section on allocator domains <allocator-domains>` for more
+         information.
+
+       * If called after Python has finish initializing (after
+         :c:func:`Py_InitializeFromConfig` has been called) the allocator
+         **must** wrap the existing allocator. Substituting the current
+         allocator for some other arbitrary one is **not supported**.
 
    .. versionchanged:: 3.12
       All allocators must be thread-safe.
@@ -626,7 +626,8 @@ The pymalloc allocator
 
 Python has a *pymalloc* allocator optimized for small objects (smaller or equal
 to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
-with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and
+with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit
+platforms. It falls back to :c:func:`PyMem_RawMalloc` and
 :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
 
 *pymalloc* is the :ref:`default allocator <default-memory-allocators>` of the

Doc/c-api/set.rst

@@ -163,4 +163,6 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
 
 .. c:function:: int PySet_Clear(PyObject *set)
 
-   Empty an existing set of all elements.
+   Empty an existing set of all elements. Return ``0`` on
+   success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of
+   :class:`set` or its subtype.

Doc/c-api/structures.rst

@@ -406,7 +406,11 @@ Accessing attributes of extension types
 .. c:type:: PyMemberDef
 
    Structure which describes an attribute of a type which corresponds to a C
-   struct member.  Its fields are, in order:
+   struct member.
+   When defining a class, put a NULL-terminated array of these
+   structures in the :c:member:`~PyTypeObject.tp_members` slot.
+
+   Its fields are, in order:
 
    .. c:member:: const char* name