Merge branch 'add-missing-null-checks' of github.com:chgnrdv/cpython into add-missing-null-checks

Author: chgnrdv

.github/CODEOWNERS

@@ -22,6 +22,7 @@ configure*                    @erlend-aasland @corona10
 **/*hamt*                     @1st1
 Objects/set*                  @rhettinger
 Objects/dict*                 @methane @markshannon
+Objects/typevarobject.c       @JelleZijlstra
 Objects/type*                 @markshannon
 Objects/codeobject.c          @markshannon
 Objects/frameobject.c         @markshannon
@@ -33,6 +34,7 @@ Python/flowgraph.c            @markshannon @iritkatriel
 Python/ast_opt.c              @isidentical
 Lib/test/test_patma.py        @brandtbucher
 Lib/test/test_peepholer.py    @brandtbucher
+Lib/test/test_type_*.py       @JelleZijlstra
 
 # Exceptions
 Lib/traceback.py              @iritkatriel

.github/dependabot.yml

@@ -12,3 +12,10 @@ updates:
         update-types:
           - "version-update:semver-minor"
           - "version-update:semver-patch"
+  - package-ecosystem: "pip"
+    directory: "/Tools/clinic/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - "skip issue"
+      - "skip news"

.github/workflows/mypy.yml

@@ -0,0 +1,39 @@
+# Workflow to run mypy on select parts of the CPython repo
+name: mypy
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "Tools/clinic/**"
+      - ".github/workflows/mypy.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  PIP_DISABLE_PIP_VERSION_CHECK: 1
+  FORCE_COLOR: 1
+  TERM: xterm-256color  # needed for FORCE_COLOR to work on mypy on Ubuntu, see https://github.com/python/mypy/issues/13817
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    name: Run mypy on Tools/clinic/
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+          cache: pip
+          cache-dependency-path: Tools/clinic/requirements-dev.txt
+      - run: pip install -r Tools/clinic/requirements-dev.txt
+      - run: mypy --config-file Tools/clinic/mypy.ini

Doc/library/ast.rst

@@ -1724,6 +1724,7 @@ Function and class definitions
             body=[
                 FunctionDef(
                     name='f',
+                    typeparams=[],
                     args=arguments(
                         posonlyargs=[],
                         args=[
@@ -1847,6 +1848,7 @@ Function and class definitions
             body=[
                 ClassDef(
                     name='Foo',
+                    typeparams=[],
                     bases=[
                         Name(id='base1', ctx=Load()),
                         Name(id='base2', ctx=Load())],
@@ -1885,6 +1887,7 @@ Async and await
         body=[
             AsyncFunctionDef(
                 name='f',
+                typeparams=[],
                 args=arguments(
                     posonlyargs=[],
                     args=[],

Doc/library/atexit.rst

@@ -20,6 +20,9 @@ at interpreter termination time they will be run in the order ``C``, ``B``,
 program is killed by a signal not handled by Python, when a Python fatal
 internal error is detected, or when :func:`os._exit` is called.
 
+**Note:** The effect of registering or unregistering functions from within
+a cleanup function is undefined.
+
 .. versionchanged:: 3.7
     When used with C-API subinterpreters, registered functions
     are local to the interpreter they were registered in.

Doc/library/collections.abc.rst

@@ -14,10 +14,7 @@
 
 .. testsetup:: *
 
-   import warnings
-   # Ignore warning when ByteString is imported
-   with warnings.catch_warnings(action='ignore', category=DeprecationWarning):
-       from collections.abc import *
+   from collections.abc import *
    import itertools
    __name__ = '<doctest>'
 

Doc/library/functions.rst

@@ -168,6 +168,13 @@ are always available.  They are listed here in alphabetical order.
    If :func:`sys.breakpointhook` is not accessible, this function will
    raise :exc:`RuntimeError`.
 
+   By default, the behavior of :func:`breakpoint` can be changed with
+   the :envvar:`PYTHONBREAKPOINT` environment variable.
+   See :func:`sys.breakpointhook` for usage details.
+
+   Note that this is not guaranteed if :func:`sys.breakpointhook`
+   has been replaced.
+
    .. audit-event:: builtins.breakpoint breakpointhook breakpoint
 
    .. versionadded:: 3.7

Doc/library/http.client.rst

@@ -394,6 +394,17 @@ HTTPConnection Objects
       one will be automatically generated and transmitted if not provided in
       the headers argument.
 
+
+.. method:: HTTPConnection.get_proxy_response_headers()
+
+   Returns a dictionary with the headers of the response received from
+   the proxy server to the CONNECT request.
+
+   If the CONNECT request was not sent, the method returns an empty dictionary.
+
+   .. versionadded:: 3.12
+
+
 .. method:: HTTPConnection.connect()
 
    Connect to the server specified when the object was created.  By default,

Doc/library/logging.config.rst

@@ -111,7 +111,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in
                                     they or their ancestors are explicitly named
                                     in the logging configuration.
 
-    :param encoding: The encoding used to open file when *fname* is filename.
+   :param encoding: The encoding used to open file when *fname* is filename.
 
    .. versionchanged:: 3.4
       An instance of a subclass of :class:`~configparser.RawConfigParser` is

Doc/library/random.rst

@@ -334,8 +334,10 @@ be found in any statistics text.
 
 .. function:: gammavariate(alpha, beta)
 
-   Gamma distribution.  (*Not* the gamma function!)  Conditions on the
-   parameters are ``alpha > 0`` and ``beta > 0``.
+   Gamma distribution.  (*Not* the gamma function!)  The shape and
+   scale parameters, *alpha* and *beta*, must have positive values.
+   (Calling conventions vary and some sources define 'beta'
+   as the inverse of the scale).
 
    The probability distribution function is::
 
@@ -346,7 +348,8 @@ be found in any statistics text.
 
 .. function:: gauss(mu=0.0, sigma=1.0)
 
-   Normal distribution, also called the Gaussian distribution.  *mu* is the mean,
+   Normal distribution, also called the Gaussian distribution.
+   *mu* is the mean,
    and *sigma* is the standard deviation.  This is slightly faster than
    the :func:`normalvariate` function defined below.
 

Doc/library/typing.rst

@@ -19,7 +19,7 @@
 
 This module provides runtime support for type hints. The most fundamental
 support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`,
-:class:`TypeVar`, and :class:`Generic`. For a full specification, please see
+:class:`TypeVar`, and :class:`Generic`. For a specification, please see
 :pep:`484`. For a simplified introduction to type hints, see :pep:`483`.
 
 
@@ -592,7 +592,7 @@ The module defines the following classes, functions and decorators.
    when the checked program targets Python 3.9 or newer.
 
    The deprecated types will be removed from the :mod:`typing` module
-   in the first Python version released 5 years after the release of Python 3.9.0.
+   no sooner than the first Python version released 5 years after the release of Python 3.9.0.
    See details in :pep:`585`—*Type Hinting Generics In Standard Collections*.
 
 
@@ -1291,6 +1291,8 @@ These are not used in annotations. They are building blocks for creating generic
        U = TypeVar('U', bound=str|bytes)  # Can be any subtype of the union str|bytes
        V = TypeVar('V', bound=SupportsAbs)  # Can be anything with an __abs__ method
 
+.. _typing-constrained-typevar:
+
     Using a *constrained* type variable, however, means that the ``TypeVar``
     can only ever be solved as being exactly one of the constraints given::
 
@@ -1550,7 +1552,7 @@ These are not used in annotations. They are building blocks for creating generic
 
 .. data:: AnyStr
 
-   ``AnyStr`` is a :class:`constrained type variable <TypeVar>` defined as
+   ``AnyStr`` is a :ref:`constrained type variable <typing-constrained-typevar>` defined as
    ``AnyStr = TypeVar('AnyStr', str, bytes)``.
 
    It is meant to be used for functions that may accept any kind of string
@@ -2112,7 +2114,7 @@ Other concrete types
       Python 2 is no longer supported, and most type checkers also no longer
       support type checking Python 2 code. Removal of the alias is not
       currently planned, but users are encouraged to use
-      :class:`str` instead of ``Text`` wherever possible.
+      :class:`str` instead of ``Text``.
 
 Abstract Base Classes
 ---------------------

Doc/library/urllib.parse.rst

@@ -159,6 +159,10 @@ or on combining URL components into a URL string.
       ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html',
                   params='', query='', fragment='')
 
+   .. warning::
+
+      :func:`urlparse` does not perform validation.  See :ref:`URL parsing
+      security <url-parsing-security>` for details.
 
    .. versionchanged:: 3.2
       Added IPv6 URL parsing capabilities.
@@ -324,8 +328,14 @@ or on combining URL components into a URL string.
    ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is
    decomposed before parsing, no error will be raised.
 
-   Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline
-   ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL.
+   Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0
+   control and space characters are stripped from the URL. ``\n``,
+   ``\r`` and tab ``\t`` characters are removed from the URL at any position.
+
+   .. warning::
+
+      :func:`urlsplit` does not perform validation.  See :ref:`URL parsing
+      security <url-parsing-security>` for details.
 
    .. versionchanged:: 3.6
       Out-of-range port numbers now raise :exc:`ValueError`, instead of
@@ -338,6 +348,9 @@ or on combining URL components into a URL string.
    .. versionchanged:: 3.10
       ASCII newline and tab characters are stripped from the URL.
 
+   .. versionchanged:: 3.12
+      Leading WHATWG C0 control and space characters are stripped from the URL.
+
 .. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser
 
 .. function:: urlunsplit(parts)
@@ -414,6 +427,35 @@ or on combining URL components into a URL string.
    or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned
    without changes.
 
+.. _url-parsing-security:
+
+URL parsing security
+--------------------
+
+The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of
+inputs.  They may not raise errors on inputs that other applications consider
+invalid.  They may also succeed on some inputs that might not be considered
+URLs elsewhere.  Their purpose is for practical functionality rather than
+purity.
+
+Instead of raising an exception on unusual input, they may instead return some
+component parts as empty strings. Or components may contain more than perhaps
+they should.
+
+We recommend that users of these APIs where the values may be used anywhere
+with security implications code defensively. Do some verification within your
+code before trusting a returned component part.  Does that ``scheme`` make
+sense?  Is that a sensible ``path``?  Is there anything strange about that
+``hostname``?  etc.
+
+What constitutes a URL is not universally well defined.  Different applications
+have different needs and desired constraints.  For instance the living `WHATWG
+spec`_ describes what user facing web clients such as a web browser require.
+While :rfc:`3986` is more general.  These functions incorporate some aspects of
+both, but cannot be claimed compliant with either.  The APIs and existing user
+code with expectations on specific behaviors predate both standards leading us
+to be very cautious about making API behavior changes.
+
 .. _parsing-ascii-encoded-bytes:
 
 Parsing ASCII Encoded Bytes

Doc/whatsnew/3.12.rst

@@ -831,8 +831,8 @@ Pending Removal in Python 3.14
   For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`.
   (Contributed by Shantanu Jain in :gh:`91896`.)
 
-* :class:`typing.ByteString`, deprecated since Python 3.9, now causes an
-  :exc:`DeprecationWarning` to be emitted when it is used or accessed.
+* :class:`typing.ByteString`, deprecated since Python 3.9, now causes a
+  :exc:`DeprecationWarning` to be emitted when it is used.
 
 * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable
   bases using the C API.
@@ -1204,6 +1204,8 @@ Build Changes
 
   (Contributed by Zhang Na in :gh:`90656`.)
 
+* ``PYTHON_FOR_REGEN`` now require Python 3.10 or newer.
+
 
 C API Changes
 =============

Grammar/python.gram

@@ -112,6 +112,7 @@ simple_stmts[asdl_stmt_seq*]:
 # will throw a SyntaxError.
 simple_stmt[stmt_ty] (memo):
     | assignment
+    | &"type" type_alias
     | e=star_expressions { _PyAST_Expr(e, EXTRA) }
     | &'return' return_stmt
     | &('import' | 'from') import_stmt
@@ -252,8 +253,8 @@ class_def[stmt_ty]:
 
 class_def_raw[stmt_ty]:
     | invalid_class_def_raw
-    | 'class' a=NAME b=['(' z=[arguments] ')' { z }] ':' c=block {
-        _PyAST_ClassDef(a->v.Name.id,
+    | 'class' a=NAME t=[type_params] b=['(' z=[arguments] ')' { z }] ':' c=block {
+        _PyAST_ClassDef(a->v.Name.id, t,
                      (b) ? ((expr_ty) b)->v.Call.args : NULL,
                      (b) ? ((expr_ty) b)->v.Call.keywords : NULL,
                      c, NULL, EXTRA) }
@@ -267,16 +268,16 @@ function_def[stmt_ty]:
 
 function_def_raw[stmt_ty]:
     | invalid_def_raw
-    | 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
-        _PyAST_FunctionDef(n->v.Name.id,
+    | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
+        _PyAST_FunctionDef(n->v.Name.id, t,
                         (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)),
                         b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
-    | ASYNC 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
+    | ASYNC 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block {
         CHECK_VERSION(
             stmt_ty,
             5,
             "Async functions are",
-            _PyAST_AsyncFunctionDef(n->v.Name.id,
+            _PyAST_AsyncFunctionDef(n->v.Name.id, t,
                             (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)),
                             b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA)
         ) }
@@ -628,6 +629,39 @@ keyword_patterns[asdl_seq*]:
 keyword_pattern[KeyPatternPair*]:
     | arg=NAME '=' value=pattern { _PyPegen_key_pattern_pair(p, arg, value) }
 
+# Type statement
+# ---------------
+
+type_alias[stmt_ty]:
+    | "type" n=NAME t=[type_params] '=' b=expression {
+        CHECK_VERSION(stmt_ty, 12, "Type statement is",
+        _PyAST_TypeAlias(CHECK(expr_ty, _PyPegen_set_expr_context(p, n, Store)), t, b, EXTRA)) }
+
+# Type parameter declaration
+# --------------------------
+
+type_params[asdl_typeparam_seq*]: '[' t=type_param_seq  ']' {
+        CHECK_VERSION(asdl_typeparam_seq *, 12, "Type parameter lists are", t) }
+
+type_param_seq[asdl_typeparam_seq*]: a[asdl_typeparam_seq*]=','.type_param+ [','] { a }
+
+type_param[typeparam_ty] (memo):
+    | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) }
+    | '*' a=NAME colon=":" e=expression {
+            RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
+                ? "cannot use constraints with TypeVarTuple"
+                : "cannot use bound with TypeVarTuple")
+        }
+    | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) }
+    | '**' a=NAME colon=":" e=expression {
+            RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
+                ? "cannot use constraints with ParamSpec"
+                : "cannot use bound with ParamSpec")
+        }
+    | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) }
+
+type_param_bound[expr_ty]: ":" e=expression { e }
+
 # EXPRESSIONS
 # -----------