diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dc1939db17e4c3..a3113a390fdb8d 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,6 +7,213 @@ Initialization, Finalization, and Threads ***************************************** +.. _pre-init-safe: + +Before Python Initialization +============================ + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetPath` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * :c:func:`Py_SetStandardStreamEncoding` + +* Informative functions: + + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +.. note:: + + The following functions **should not be called** before + :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, + :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, + :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`. + + +.. _global-conf-vars: + +Global configuration variables +============================== + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + +.. c:var:: Py_BytesWarningFlag + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + +.. c:var:: Py_DebugFlag + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + +.. c:var:: Py_DontWriteBytecodeFlag + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + +.. c:var:: Py_FrozenFlag + + Suppress error messages when calculating the module search path in + :c:func:`Py_GetPath`. + + Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + +.. c:var:: Py_HashRandomizationFlag + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + +.. c:var:: Py_IgnoreEnvironmentFlag + + Ignore all :envvar:`PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + +.. c:var:: Py_InspectFlag + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + +.. c:var:: Py_InteractiveFlag + + Set by the :option:`-i` option. + +.. c:var:: Py_IsolatedFlag + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + +.. c:var:: Py_LegacyWindowsFSEncodingFlag + + If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 + encoding for the filesystem encoding. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + Availability: Windows. + +.. c:var:: Py_LegacyWindowsStdioFlag + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + Availability: Windows. + +.. c:var:: Py_NoSiteFlag + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + +.. c:var:: Py_NoUserSiteDirectory + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + +.. c:var:: Py_OptimizeFlag + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + +.. c:var:: Py_QuietFlag + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + +.. c:var:: Py_UnbufferedStdioFlag + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + +.. c:var:: Py_VerboseFlag + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + Initializing and finalizing the interpreter =========================================== @@ -27,9 +234,11 @@ Initializing and finalizing the interpreter single: PySys_SetArgvEx() single: Py_FinalizeEx() - Initialize the Python interpreter. In an application embedding Python, this - should be called before using any other Python/C API functions; with the - exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 4b1e666ef35765..2af0c46d451f05 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -100,9 +100,10 @@ The following function sets are wrappers to the system allocator. These functions are thread-safe, the :term:`GIL ` does not need to be held. -The default raw memory block allocator uses the following functions: -:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call -``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. +The :ref:`default raw memory allocator ` uses +the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` +and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting +zero bytes. .. versionadded:: 3.4 @@ -165,7 +166,8 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -By default, these functions use :ref:`pymalloc memory allocator `. +The :ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `. .. warning:: @@ -270,7 +272,8 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -By default, these functions use :ref:`pymalloc memory allocator `. +The :ref:`default object allocator ` uses the +:ref:`pymalloc memory allocator `. .. warning:: @@ -326,6 +329,31 @@ By default, these functions use :ref:`pymalloc memory allocator `. If *p* is *NULL*, no operation is performed. +.. _default-memory-allocators: + +Default Memory Allocators +========================= + +Default memory allocators: + +=============================== ==================== ================== ===================== ==================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=============================== ==================== ================== ===================== ==================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Release build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +=============================== ==================== ================== ===================== ==================== + +Legend: + +* Name: value for :envvar:`PYTHONMALLOC` environment variable +* ``malloc``: system allocators from the standard C library, C functions: + :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` +* ``pymalloc``: :ref:`pymalloc memory allocator ` +* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` + + Customize Memory Allocators =========================== @@ -431,7 +459,8 @@ Customize Memory Allocators displayed if :mod:`tracemalloc` is tracing Python memory allocations and the memory block was traced. - These hooks are installed by default if Python is compiled in debug + These hooks are :ref:`installed by default ` if + Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install debug hooks on a Python compiled in release mode. @@ -453,9 +482,9 @@ to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. -*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: -:c:func:`PyObject_Malloc`) domains. +*pymalloc* is the :ref:`default allocator ` of the +:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and +:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 6ab5942929fcce..3897fdd828216d 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -141,7 +141,8 @@ the same library that the Python runtime is using. Read and execute statements from a file associated with an interactive device until EOF is reached. The user will be prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF. + (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF or a negative + number upon failure. .. c:var:: int (*PyOS_InputHook)(void) diff --git a/Doc/conf.py b/Doc/conf.py index aaee983984bcb6..19a2f7d67ff831 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -90,11 +90,10 @@ # Options for LaTeX output # ------------------------ +latex_engine = 'xelatex' + # Get LaTeX to handle Unicode correctly latex_elements = { - 'inputenc': r'\usepackage[utf8x]{inputenc}', - 'utf8extra': '', - 'fontenc': r'\usepackage[T1,T2A]{fontenc}', } # Additional stuff for the LaTeX preamble. diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 7cde1a0701eb8f..9fce46ad266962 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -285,6 +285,10 @@ the full reference. See the :func:`setup` function for a list of keyword arguments accepted by the Distribution constructor. :func:`setup` creates a Distribution instance. + .. versionchanged:: 3.7 + :class:`~distutils.core.Distribution` now warns if ``classifiers``, + ``keywords`` and ``platforms`` fields are not specified as a list or + a string. .. class:: Command diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 38e0202e4ac1f6..542ad544843fde 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -581,17 +581,19 @@ This information includes: | | description of the | | | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``long_description`` | longer description of the | long string | \(5) | +| ``long_description`` | longer description of the | long string | \(4) | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``download_url`` | location where the | URL | \(4) | +| ``download_url`` | location where the | URL | | | | package may be downloaded | | | +----------------------+---------------------------+-----------------+--------+ -| ``classifiers`` | a list of classifiers | list of strings | \(4) | +| ``classifiers`` | a list of classifiers | list of strings | (6)(7) | +----------------------+---------------------------+-----------------+--------+ -| ``platforms`` | a list of platforms | list of strings | | +| ``platforms`` | a list of platforms | list of strings | (6)(8) | +----------------------+---------------------------+-----------------+--------+ -| ``license`` | license for the package | short string | \(6) | +| ``keywords`` | a list of keywords | list of strings | (6)(8) | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the package | short string | \(5) | +----------------------+---------------------------+-----------------+--------+ Notes: @@ -607,22 +609,30 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - These fields should not be used if your package is to be compatible with Python - versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website - `_. - -(5) The ``long_description`` field is used by PyPI when you are :ref:`registering ` a package, to :ref:`build its home page `. -(6) +(5) The ``license`` field is a text indicating the license covering the package where the license is not a selection from the "License" Trove classifiers. See the ``Classifier`` field. Notice that there's a ``licence`` distribution option which is deprecated but still acts as an alias for ``license``. +(6) + This field must be a list. + +(7) + The valid classifiers are listed on + `PyPI `_. + +(8) + To preserve backward compatibility, this field also accepts a string. If + you pass a comma-separated string ``'foo, bar'``, it will be converted to + ``['foo', 'bar']``, Otherwise, it will be converted to a list of one + string. + 'short string' A single line of text, not more than 200 characters. @@ -650,7 +660,7 @@ information is sometimes used to indicate sub-releases. These are 1.0.1a2 the second alpha release of the first patch version of 1.0 -``classifiers`` are specified in a Python list:: +``classifiers`` must be specified in a list:: setup(..., classifiers=[ @@ -671,6 +681,11 @@ information is sometimes used to indicate sub-releases. These are ], ) +.. versionchanged:: 3.7 + :class:`~distutils.core.setup` now raises a :exc:`TypeError` if + ``classifiers``, ``keywords`` and ``platforms`` fields are not specified + as a list. + .. _debug-setup-script: Debugging the setup script diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 7c273533aba5b5..ea1c29a397ed86 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -40,7 +40,7 @@ A Simple Example Let's create an extension module called ``spam`` (the favorite food of Monty Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system`. [#]_ This function takes a null-terminated +library function :c:func:`system` [#]_. This function takes a null-terminated character string as argument and returns an integer. We want this function to be callable from Python as follows:: @@ -917,7 +917,7 @@ It is also possible to :dfn:`borrow` [#]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely. [#]_ +memory and should be avoided completely [#]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1088,7 +1088,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never *NULL* --- in fact it guarantees -that it is always a tuple. [#]_ +that it is always a tuple [#]_. It is a severe error to ever let a *NULL* pointer "escape" to the Python user. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 0e36ba0aec07ae..62fbdb87a53000 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -659,7 +659,7 @@ Fortunately, Python's cyclic-garbage collector will eventually figure out that the list is garbage and free it. In the second version of the :class:`Noddy` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes. [#]_ This +object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This means that :class:`Noddy` objects can participate in cycles:: >>> import noddy2 diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index e8466ee5423c68..fa8c6939408100 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -844,7 +844,7 @@ backreferences in a RE. For example, the following RE detects doubled words in a string. :: - >>> p = re.compile(r'(\b\w+)\s+\1') + >>> p = re.compile(r'\b(\w+)\s+\1\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -943,9 +943,9 @@ number of the group. There's naturally a variant that uses the group name instead of the number. This is another Python extension: ``(?P=name)`` indicates that the contents of the group called *name* should again be matched at the current point. The regular expression for finding doubled words, -``(\b\w+)\s+\1`` can also be written as ``(?P\b\w+)\s+(?P=word)``:: +``\b(\w+)\s+\1\b`` can also be written as ``\b(?P\w+)\s+(?P=word)\b``:: - >>> p = re.compile(r'(?P\b\w+)\s+(?P=word)') + >>> p = re.compile(r'\b(?P\w+)\s+(?P=word)\b') >>> p.search('Paris in the the spring').group() 'the the' diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 0c6d412b3c4cda..0ef4765327776f 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -17,7 +17,7 @@ Shoddy_increment(Shoddy *self, PyObject *unused) static PyMethodDef Shoddy_methods[] = { {"increment", (PyCFunction)Shoddy_increment, METH_NOARGS, PyDoc_STR("increment state counter")}, - {NULL, NULL}, + {NULL}, }; static int diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index 1ab05a62ad458f..deb5e10f6ef7e9 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -345,20 +345,20 @@ and off individually. They are described here in more detail. Converts calls to various functions in the :mod:`operator` module to other, but equivalent, function calls. When needed, the appropriate ``import`` - statements are added, e.g. ``import collections``. The following mapping + statements are added, e.g. ``import collections.abc``. The following mapping are made: - ================================== ========================================== + ================================== ============================================= From To - ================================== ========================================== - ``operator.isCallable(obj)`` ``hasattr(obj, '__call__')`` + ================================== ============================================= + ``operator.isCallable(obj)`` ``callable(obj)`` ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` - ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.Sequence)`` - ``operator.isMappingType(obj)`` ``isinstance(obj, collections.Mapping)`` + ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)`` + ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)`` ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)`` ``operator.repeat(obj, n)`` ``operator.mul(obj, n)`` ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)`` - ================================== ========================================== + ================================== ============================================= .. 2to3fixer:: paren diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 1838eb95a7d522..b9735de2078d18 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -21,7 +21,9 @@ enable *debug mode*. To enable all debug checks for an application: * Enable the asyncio debug mode globally by setting the environment variable - :envvar:`PYTHONASYNCIODEBUG` to ``1``, or by calling :meth:`AbstractEventLoop.set_debug`. + :envvar:`PYTHONASYNCIODEBUG` to ``1``, using ``-X dev`` command line option + (see the :option:`-X` option), or by calling + :meth:`AbstractEventLoop.set_debug`. * Set the log level of the :ref:`asyncio logger ` to :py:data:`logging.DEBUG`. For example, call ``logging.basicConfig(level=logging.DEBUG)`` at startup. @@ -42,6 +44,11 @@ Examples debug checks: * :exc:`ResourceWarning` warnings are emitted when transports and event loops are :ref:`not closed explicitly `. +.. versionchanged:: 3.7 + + The new ``-X dev`` command line option can now also be used to enable + the debug mode. + .. seealso:: The :meth:`AbstractEventLoop.set_debug` method and the :ref:`asyncio logger @@ -209,9 +216,9 @@ The fix is to call the :func:`ensure_future` function or the Detect exceptions never consumed -------------------------------- -Python usually calls :func:`sys.displayhook` on unhandled exceptions. If +Python usually calls :func:`sys.excepthook` on unhandled exceptions. If :meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted +:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted ` when the future is deleted by the garbage collector, with the traceback where the exception was raised. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index e635cba659a36b..760640fa5e41ca 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -391,7 +391,7 @@ Creating connections :ref:`UDP echo server protocol ` examples. -.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None) +.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path=None, \*, ssl=None, sock=None, server_hostname=None) Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket @@ -403,13 +403,17 @@ Creating connections coroutine returns a ``(transport, protocol)`` pair. *path* is the name of a UNIX domain socket, and is required unless a *sock* - parameter is specified. Abstract UNIX sockets, :class:`str`, and - :class:`bytes` paths are supported. + parameter is specified. Abstract UNIX sockets, :class:`str`, + :class:`bytes`, and :class:`~pathlib.Path` paths are supported. See the :meth:`AbstractEventLoop.create_connection` method for parameters. Availability: UNIX. + .. versionchanged:: 3.7 + + The *path* parameter can now be a :class:`~pathlib.Path` object. + Creating listening connections ------------------------------ @@ -479,10 +483,18 @@ Creating listening connections Similar to :meth:`AbstractEventLoop.create_server`, but specific to the socket family :py:data:`~socket.AF_UNIX`. + *path* is the name of a UNIX domain socket, and is required unless a *sock* + parameter is specified. Abstract UNIX sockets, :class:`str`, + :class:`bytes`, and :class:`~pathlib.Path` paths are supported. + This method is a :ref:`coroutine `. Availability: UNIX. + .. versionchanged:: 3.7 + + The *path* parameter can now be a :class:`~pathlib.Path` object. + .. coroutinemethod:: BaseEventLoop.connect_accepted_socket(protocol_factory, sock, \*, ssl=None) Handle an accepted connection. @@ -501,6 +513,9 @@ Creating listening connections This method is a :ref:`coroutine `. When completed, the coroutine returns a ``(transport, protocol)`` pair. + .. versionadded:: 3.5.3 + + Watch file descriptors ---------------------- @@ -952,10 +967,7 @@ Wait until a file descriptor received some data using the :meth:`AbstractEventLoop.add_reader` method and then close the event loop:: import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair + from socket import socketpair # Create a pair of connected file descriptors rsock, wsock = socketpair() diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index cd84ae76b5d86e..af462009fa8f03 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -690,10 +690,7 @@ Wait until a socket receives data using the the event loop :: import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair + from socket import socketpair # Create a pair of connected sockets rsock, wsock = socketpair() diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 491afdd610ca08..78091d62a71631 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -426,10 +426,7 @@ Coroutine waiting until a socket receives data using the :func:`open_connection` function:: import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair + from socket import socketpair @asyncio.coroutine def wait_for_data(loop): diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 1d84d4587fd9c0..5c60b604c68301 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -21,7 +21,7 @@ program is killed by a signal not handled by Python, when a Python fatal internal error is detected, or when :func:`os._exit` is called. -.. function:: register(func, *args, **kargs) +.. function:: register(func, *args, **kwargs) Register *func* as a function to be executed at termination. Any optional arguments that are to be passed to *func* must be passed as arguments to diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cda829694a3277..4b0d8c048ae7d9 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -618,6 +618,25 @@ added elements by appending to the right and popping to the left:: d.append(elem) yield s / n +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + The :meth:`rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on the :meth:`rotate` method to position elements to be popped:: diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 19793693b7ba68..48ca0da6b95f3a 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -137,6 +137,28 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. _simplifying-support-for-single-optional-context-managers: + +.. function:: nullcontext(enter_result=None) + + Return a context manager that returns enter_result from ``__enter__``, but + otherwise does nothing. It is intended to be used as a stand-in for an + optional context manager, for example:: + + def process_file(file_or_path): + if isinstance(file_or_path, str): + # If string, open file + cm = open(file_or_path) + else: + # Caller is responsible for closing file + cm = nullcontext(file_or_path) + + with cm as file: + # Perform processing on the file + + .. versionadded:: 3.7 + + .. function:: suppress(*exceptions) Return a context manager that suppresses any of the specified exceptions @@ -433,24 +455,6 @@ statements to manage arbitrary resources that don't natively support the context management protocol. -Simplifying support for single optional context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the specific case of a single optional context manager, :class:`ExitStack` -instances can be used as a "do nothing" context manager, allowing a context -manager to easily be omitted without affecting the overall structure of -the source code:: - - def debug_trace(details): - if __debug__: - return TraceContext(details) - # Don't do anything special with the context in release mode - return ExitStack() - - with debug_trace(): - # Suite is traced in debug mode, but runs normally otherwise - - Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 9877b711a9af16..dd62cb32b9d16b 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -116,7 +116,7 @@ The :mod:`crypt` module defines the following functions: Accept ``crypt.METHOD_*`` values in addition to strings for *salt*. -.. function:: mksalt(method=None, *, log_rounds=12) +.. function:: mksalt(method=None, *, rounds=None) Return a randomly generated salt of the specified method. If no *method* is given, the strongest method available as returned by @@ -125,14 +125,18 @@ The :mod:`crypt` module defines the following functions: The return value is a string suitable for passing as the *salt* argument to :func:`crypt`. - *log_rounds* specifies the binary logarithm of the number of rounds - for ``crypt.METHOD_BLOWFISH``, and is ignored otherwise. ``8`` specifies - ``256`` rounds. + *rounds* specifies the number of rounds for ``METHOD_SHA256``, + ``METHOD_SHA512`` and ``METHOD_BLOWFISH``. + For ``METHOD_SHA256`` and ``METHOD_SHA512`` it must be an integer between + ``1000`` and ``999_999_999``, the default is ``5000``. For + ``METHOD_BLOWFISH`` it must be a power of two between ``16`` (2\ :sup:`4`) + and ``2_147_483_648`` (2\ :sup:`31`), the default is ``4096`` + (2\ :sup:`12`). .. versionadded:: 3.3 .. versionchanged:: 3.7 - Added the *log_rounds* parameter. + Added the *rounds* parameter. Examples diff --git a/Doc/library/development.rst b/Doc/library/development.rst index d2b5fa2aa4f5f8..ab34e1f7ce5f64 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -24,3 +24,6 @@ The list of modules described in this chapter is: unittest.mock-examples.rst 2to3.rst test.rst + +See also the Python development mode: the :option:`-X` ``dev`` option and +:envvar:`PYTHONDEVMODE` environment variable. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 4d011036d7ae74..01d46f88fb5796 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1077,7 +1077,7 @@ All of the following opcodes use their arguments. Pops all function arguments, and the function itself off the stack, and pushes the return value. Note that this opcode pops at most three items from the stack. Var-positional and var-keyword arguments are packed - by :opcode:`BUILD_MAP_UNPACK_WITH_CALL` and + by :opcode:`BUILD_TUPLE_UNPACK_WITH_CALL` and :opcode:`BUILD_MAP_UNPACK_WITH_CALL`. .. versionadded:: 3.6 diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index fa6c340bb7bb3e..a91894059d0007 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -753,15 +753,16 @@ which incur interpreter overhead. def roundrobin(*iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" # Recipe credited to George Sakkis - pending = len(iterables) + num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) - while pending: + while num_active: try: for next in nexts: yield next() except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) + # Remove the iterator we just exhausted from the cycle. + num_active -= 1 + nexts = cycle(islice(nexts, num_active)) def partition(pred, iterable): 'Use a predicate to partition entries into false entries and true entries' diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst index a66e52ca57691d..83b3d4973bf0d9 100644 --- a/Doc/library/msilib.rst +++ b/Doc/library/msilib.rst @@ -124,9 +124,9 @@ structures. .. seealso:: - `FCICreateFile `_ - `UuidCreate `_ - `UuidToString `_ + `FCICreate `_ + `UuidCreate `_ + `UuidToString `_ .. _database-objects: @@ -160,10 +160,10 @@ Database Objects .. seealso:: - `MSIDatabaseOpenView `_ - `MSIDatabaseCommit `_ - `MSIGetSummaryInformation `_ - `MsiCloseHandle `_ + `MSIDatabaseOpenView `_ + `MSIDatabaseCommit `_ + `MSIGetSummaryInformation `_ + `MsiCloseHandle `_ .. _view-objects: @@ -209,11 +209,11 @@ View Objects .. seealso:: - `MsiViewExecute `_ - `MSIViewGetColumnInfo `_ - `MsiViewFetch `_ - `MsiViewModify `_ - `MsiViewClose `_ + `MsiViewExecute `_ + `MSIViewGetColumnInfo `_ + `MsiViewFetch `_ + `MsiViewModify `_ + `MsiViewClose `_ .. _summary-objects: @@ -253,10 +253,10 @@ Summary Information Objects .. seealso:: - `MsiSummaryInfoGetProperty `_ - `MsiSummaryInfoGetPropertyCount `_ - `MsiSummaryInfoSetProperty `_ - `MsiSummaryInfoPersist `_ + `MsiSummaryInfoGetProperty `_ + `MsiSummaryInfoGetPropertyCount `_ + `MsiSummaryInfoSetProperty `_ + `MsiSummaryInfoPersist `_ .. _record-objects: @@ -307,11 +307,11 @@ Record Objects .. seealso:: - `MsiRecordGetFieldCount `_ - `MsiRecordSetString `_ - `MsiRecordSetStream `_ - `MsiRecordSetInteger `_ - `MsiRecordClear `_ + `MsiRecordGetFieldCount `_ + `MsiRecordSetString `_ + `MsiRecordSetStream `_ + `MsiRecordSetInteger `_ + `MsiRecordClearData `_ .. _msi-errors: @@ -398,15 +398,15 @@ Directory Objects .. method:: remove_pyc() - Remove ``.pyc``/``.pyo`` files on uninstall. + Remove ``.pyc`` files on uninstall. .. seealso:: - `Directory Table `_ - `File Table `_ - `Component Table `_ - `FeatureComponents Table `_ + `Directory Table `_ + `File Table `_ + `Component Table `_ + `FeatureComponents Table `_ .. _features: @@ -431,7 +431,7 @@ Features .. seealso:: - `Feature Table `_ + `Feature Table `_ .. _msi-gui: @@ -526,13 +526,13 @@ for installing Python packages. .. seealso:: - `Dialog Table `_ - `Control Table `_ - `Control Types `_ - `ControlCondition Table `_ - `ControlEvent Table `_ - `EventMapping Table `_ - `RadioButton Table `_ + `Dialog Table `_ + `Control Table `_ + `Control Types `_ + `ControlCondition Table `_ + `ControlEvent Table `_ + `EventMapping Table `_ + `RadioButton Table `_ .. _msi-tables: diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 3619cccfe63578..8b3081cb80c972 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1837,8 +1837,8 @@ Running the following commands creates a server for a single shared queue which remote clients can access:: >>> from multiprocessing.managers import BaseManager - >>> import queue - >>> queue = queue.Queue() + >>> from queue import Queue + >>> queue = Queue() >>> class QueueManager(BaseManager): pass >>> QueueManager.register('get_queue', callable=lambda:queue) >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 64aa3ac7c8aefd..3d29ac49b9191a 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -20,8 +20,10 @@ the Unix :program:`ftp` program and other FTP clients. A :class:`~netrc.netrc` instance or subclass instance encapsulates data from a netrc file. The initialization argument, if present, specifies the file to parse. If - no argument is given, the file :file:`.netrc` in the user's home directory will - be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic + no argument is given, the file :file:`.netrc` in the user's home directory -- + as determined by :func:`os.path.expanduser` -- will be read. Otherwise, + a :exc:`FileNotFoundError` exception will be raised. + Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. If no argument is specified on a POSIX system, the presence of passwords in the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file @@ -32,6 +34,10 @@ the Unix :program:`ftp` program and other FTP clients. .. versionchanged:: 3.4 Added the POSIX permission check. + .. versionchanged:: 3.7 + :func:`os.path.expanduser` is used to find the location of the + :file:`.netrc` file when *file* is not passed as argument. + .. exception:: NetrcParseError @@ -82,4 +88,3 @@ Instances of :class:`~netrc.netrc` have public instance variables: punctuation is allowed in passwords, however, note that whitespace and non-printable characters are not allowed in passwords. This is a limitation of the way the .netrc file is parsed and may be removed in the future. - diff --git a/Doc/library/re.rst b/Doc/library/re.rst index e0cb626305d893..dae1d7ea10a031 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -200,6 +200,20 @@ The special characters are: place it at the beginning of the set. For example, both ``[()[\]{}]`` and ``[]()[{}]`` will both match a parenthesis. + * Support of nested sets and set operations as in `Unicode Technical + Standard #18`_ might be added in the future. This would change the + syntax, so to facilitate this change a :exc:`FutureWarning` will be raised + in ambiguous cases for the time being. + That include sets starting with a literal ``'['`` or containing literal + character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To + avoid a warning escape them with a backslash. + + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + + .. versionchanged:: 3.7 + :exc:`FutureWarning` is raised if a character set contains constructs + that will change semantically in the future. + ``|`` ``A|B``, where *A* and *B* can be arbitrary REs, creates a regular expression that will match either *A* or *B*. An arbitrary number of REs can be separated by the @@ -617,7 +631,8 @@ form. This flag allows you to write regular expressions that look nicer and are more readable by allowing you to visually separate logical sections of the pattern and add comments. Whitespace within the pattern is ignored, except - when in a character class or when preceded by an unescaped backslash. + when in a character class, or when preceded by an unescaped backslash, + or within tokens like ``*?``, ``(?:`` or ``(?P<...>``. When a line contains a ``#`` that is not in a character class and is not preceded by an unescaped backslash, all characters from the leftmost such ``#`` through the end of the line are ignored. @@ -674,11 +689,11 @@ form. splits occur, and the remainder of the string is returned as the final element of the list. :: - >>> re.split('\W+', 'Words, words, words.') + >>> re.split(r'\W+', 'Words, words, words.') ['Words', 'words', 'words', ''] - >>> re.split('(\W+)', 'Words, words, words.') + >>> re.split(r'(\W+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split('\W+', 'Words, words, words.', 1) + >>> re.split(r'\W+', 'Words, words, words.', 1) ['Words', 'words, words.'] >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) ['0', '3', '9'] @@ -687,43 +702,25 @@ form. the string, the result will start with an empty string. The same holds for the end of the string:: - >>> re.split('(\W+)', '...words, words...') + >>> re.split(r'(\W+)', '...words, words...') ['', '...', 'words', ', ', 'words', '...', ''] That way, separator components are always found at the same relative indices within the result list. - .. note:: - - :func:`split` doesn't currently split a string on an empty pattern match. - For example:: - - >>> re.split('x*', 'axbc') - ['a', 'bc'] - - Even though ``'x*'`` also matches 0 'x' before 'a', between 'b' and 'c', - and after 'c', currently these matches are ignored. The correct behavior - (i.e. splitting on empty matches too and returning ``['', 'a', 'b', 'c', - '']``) will be implemented in future versions of Python, but since this - is a backward incompatible change, a :exc:`FutureWarning` will be raised - in the meanwhile. + The pattern can match empty strings. :: - Patterns that can only match empty strings currently never split the - string. Since this doesn't match the expected behavior, a - :exc:`ValueError` will be raised starting from Python 3.5:: - - >>> re.split("^$", "foo\n\nbar\n", flags=re.M) - Traceback (most recent call last): - File "", line 1, in - ... - ValueError: split() requires a non-empty pattern match. + >>> re.split(r'\b', 'Words, words, words.') + ['', 'Words', ', ', 'words', ', ', 'words', '.'] + >>> re.split(r'(\W*)', '...words...') + ['', '...', 'w', '', 'o', '', 'r', '', 'd', '', 's', '...', ''] .. versionchanged:: 3.1 Added the optional flags argument. - .. versionchanged:: 3.5 - Splitting on a pattern that could match an empty string now raises - a warning. Patterns that can only match empty strings are now rejected. + .. versionchanged:: 3.7 + Added support of splitting on a pattern that could match an empty string. + .. function:: findall(pattern, string, flags=0) @@ -731,8 +728,10 @@ form. strings. The *string* is scanned left-to-right, and matches are returned in the order found. If one or more groups are present in the pattern, return a list of groups; this will be a list of tuples if the pattern has more than - one group. Empty matches are included in the result unless they touch the - beginning of another match. + one group. Empty matches are included in the result. + + .. versionchanged:: 3.7 + Non-empty matches can now start just after a previous empty match. .. function:: finditer(pattern, string, flags=0) @@ -740,8 +739,10 @@ form. Return an :term:`iterator` yielding :ref:`match objects ` over all non-overlapping matches for the RE *pattern* in *string*. The *string* is scanned left-to-right, and matches are returned in the order found. Empty - matches are included in the result unless they touch the beginning of another - match. + matches are included in the result. + + .. versionchanged:: 3.7 + Non-empty matches can now start just after a previous empty match. .. function:: sub(pattern, repl, string, count=0, flags=0) @@ -828,7 +829,7 @@ form. >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" >>> print('[%s]+' % re.escape(legal_chars)) - [abcdefghijklmnopqrstuvwxyz0123456789!\#\$%&'\*\+\-\.\^_`\|~:]+ + [abcdefghijklmnopqrstuvwxyz0123456789!\#\$%\&'\*\+\-\.\^_`\|\~:]+ >>> operators = ['+', '-', '*', '/', '**'] >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4d4a6161057cc5..6094a7b871454e 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -69,7 +69,7 @@ Scheduler Objects Schedule a new event. The *time* argument should be a numeric type compatible with the return value of the *timefunc* function passed to the constructor. Events scheduled for the same *time* will be executed in the order of their - *priority*. + *priority*. A lower number represents a higher priority. Executing the event means executing ``action(*argument, **kwargs)``. *argument* is a sequence holding the positional arguments for *action*. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 218a31c848aa1b..e12c8c97844fa7 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -289,7 +289,7 @@ Server Objects .. XXX should the default implementations of these be documented, or should it be assumed that the user will look at socketserver.py? - .. method:: finish_request() + .. method:: finish_request(request, client_address) Actually processes the request by instantiating :attr:`RequestHandlerClass` and calling its :meth:`~BaseRequestHandler.handle` method. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9b3bdd5489d4a9..45bb65ff0715e3 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -429,6 +429,10 @@ Certificate handling Matching of IP addresses, when present in the subjectAltName field of the certificate, is now supported. + .. versionchanged:: 3.7 + Allow wildcard when it is the leftmost and the only character + in that segment. + .. function:: cert_time_to_seconds(cert_time) Return the time in seconds since the Epoch, given the ``cert_time`` diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 5b25428525766a..e9606783ef796a 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -755,13 +755,15 @@ attributes: * *idpattern* -- This is the regular expression describing the pattern for non-braced placeholders. The default value is the regular expression - ``(?-i:[_a-zA-Z][_a-zA-Z0-9]*)``. If this is given and *braceidpattern* is + ``(?a:[_a-zA-Z][_a-zA-Z0-9]*)``. If this is given and *braceidpattern* is ``None`` this pattern will also apply to braced placeholders. .. note:: Since default *flags* is ``re.IGNORECASE``, pattern ``[a-z]`` can match - with some non-ASCII characters. That's why we use local ``-i`` flag here. + with some non-ASCII characters. That's why we use the local ``a`` flag + here. Further, with the default *flags* value, including ``A-Z`` in the + ranges is redundant, but required for backward compatibility. While *flags* is kept to ``re.IGNORECASE`` for backward compatibility, you can override it to ``0`` or ``re.IGNORECASE | re.ASCII`` when diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index faf540c4ea43b2..9e47681804b5eb 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -334,6 +334,7 @@ always available. :const:`bytes_warning` :option:`-b` :const:`quiet` :option:`-q` :const:`hash_randomization` :option:`-R` + :const:`dev_mode` :option:`-X` ``dev`` ============================= ============================= .. versionchanged:: 3.2 @@ -345,6 +346,9 @@ always available. .. versionchanged:: 3.3 Removed obsolete ``division_warning`` attribute. + .. versionchanged:: 3.7 + Added ``dev_mode`` attribute for the new :option:`-X` ``dev`` flag. + .. data:: float_info diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index ad6a9f7c972dae..7693ecd02789cc 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -12,7 +12,7 @@ -------------- This module provides an interface to the POSIX calls for tty I/O control. For a -complete description of these calls, see :manpage:`termios(2)` Unix manual +complete description of these calls, see :manpage:`termios(3)` Unix manual page. It is only available for those Unix versions that support POSIX *termios* style tty I/O control configured during installation. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 4ffb4d22cd72ca..ccbb3f37877b2c 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -241,6 +241,7 @@ Functions * ``'monotonic'``: :func:`time.monotonic` * ``'perf_counter'``: :func:`time.perf_counter` * ``'process_time'``: :func:`time.process_time` + * ``'thread_time'``: :func:`time.thread_time` * ``'time'``: :func:`time.time` The result has the following attributes: @@ -603,6 +604,32 @@ Functions of the calendar date may be accessed as attributes. +.. function:: thread_time() -> float + + .. index:: + single: CPU time + single: processor time + single: benchmarking + + Return the value (in fractional seconds) of the sum of the system and user + CPU time of the current thread. It does not include time elapsed during + sleep. It is thread-specific by definition. The reference point of the + returned value is undefined, so that only the difference between the results + of consecutive calls in the same thread is valid. + + Availability: Windows, Linux, Unix systems supporting + ``CLOCK_THREAD_CPUTIME_ID``. + + .. versionadded:: 3.7 + + +.. function:: thread_time_ns() -> int + + Similar to :func:`thread_time` but return time as nanoseconds. + + .. versionadded:: 3.7 + + .. function:: time_ns() -> int Similar to :func:`time` but returns time as an integer number of nanoseconds diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 3e1faed8c7b8d5..f51add2b41fe46 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -154,7 +154,7 @@ background material, while the second half can be taken to the keyboard as a handy reference. When trying to answer questions of the form "how do I do blah", it is often best -to find out how to do"blah" in straight Tk, and then convert this back into the +to find out how to do "blah" in straight Tk, and then convert this back into the corresponding :mod:`tkinter` call. Python programmers can often guess at the correct Python command by looking at the Tk documentation. This means that in order to use Tkinter, you will have to know a little bit about Tk. This document diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 048ee64aac9851..2d327c02540995 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -650,8 +650,8 @@ Traceback .. class:: Traceback - Sequence of :class:`Frame` instances sorted from the most recent frame to - the oldest frame. + Sequence of :class:`Frame` instances sorted from the oldest frame to the + most recent frame. A traceback contains at least ``1`` frame. If the ``tracemalloc`` module failed to get a frame, the filename ``""`` at line number ``0`` is @@ -663,11 +663,17 @@ Traceback The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` instance. - .. method:: format(limit=None) + .. versionchanged:: 3.7 + Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. - Format the traceback as a list of lines with newlines. Use the - :mod:`linecache` module to retrieve lines from the source code. If - *limit* is set, only format the *limit* most recent frames. + .. method:: format(limit=None, most_recent_first=False) + + Format the traceback as a list of lines with newlines. Use the + :mod:`linecache` module to retrieve lines from the source code. + If *limit* is set, format the *limit* most recent frames if *limit* + is positive. Otherwise, format the ``abs(limit)`` oldest frames. + If *most_recent_first* is ``True``, the order of the formatted frames + is reversed, returning the most recent frame first instead of last. Similar to the :func:`traceback.format_tb` function, except that :meth:`.format` does not include newlines. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 9883d8bbe86596..d28a5d548431ef 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -146,6 +146,8 @@ See :pep:`484` for more details. ``Derived`` is expected. This is useful when you want to prevent logic errors with minimal runtime cost. +.. versionadded:: 3.5.2 + Callable -------- @@ -494,6 +496,8 @@ The module defines the following classes, functions and decorators: ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. + .. versionadded:: 3.5.2 + .. class:: Iterable(Generic[T_co]) A generic version of :class:`collections.abc.Iterable`. @@ -674,6 +678,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.defaultdict`. + .. versionadded:: 3.5.2 + .. class:: Counter(collections.Counter, Dict[T, int]) A generic version of :class:`collections.Counter`. @@ -762,6 +768,8 @@ The module defines the following classes, functions and decorators: def add_unicode_checkmark(text: Text) -> Text: return text + u' \u2713' + .. versionadded:: 3.5.2 + .. class:: io Wrapper namespace for I/O stream types. @@ -847,6 +855,8 @@ The module defines the following classes, functions and decorators: UserId = NewType('UserId', int) first_user = UserId(1) + .. versionadded:: 3.5.2 + .. function:: cast(typ, val) Cast a value to a type. @@ -1054,3 +1064,5 @@ The module defines the following classes, functions and decorators: "forward reference", to hide the ``expensive_mod`` reference from the interpreter runtime. Type annotations for local variables are not evaluated, so the second annotation does not need to be enclosed in quotes. + + .. versionadded:: 3.5.2 diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e52f140029c57a..4755488d91db06 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -219,6 +219,22 @@ Command-line options Stop the test run on the first error or failure. +.. cmdoption:: -k + + Only run test methods and classes that match the pattern or substring. + This option may be used multiple times, in which case all test cases that + match of the given patterns are included. + + Patterns that contain a wildcard character (``*``) are matched against the + test name using :meth:`fnmatch.fnmatchcase`; otherwise simple case-sensitive + substring matching is used. + + Patterns are matched against the fully qualified test method name as + imported by the test loader. + + For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, + ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. + .. cmdoption:: --locals Show local variables in tracebacks. @@ -229,6 +245,9 @@ Command-line options .. versionadded:: 3.5 The command-line option ``--locals``. +.. versionadded:: 3.7 + The command-line option ``-k``. + The command line can also be used for test discovery, for running all of the tests in a project or just a subset. @@ -1745,6 +1764,21 @@ Loading and running tests This affects all the :meth:`loadTestsFrom\*` methods. + .. attribute:: testNamePatterns + + List of Unix shell-style wildcard test name patterns that test methods + have to match to be included in test suites (see ``-v`` option). + + If this attribute is not ``None`` (the default), all test methods to be + included in test suites must match one of the patterns in this list. + Note that matches are always performed using :meth:`fnmatch.fnmatchcase`, + so unlike patterns passed to the ``-v`` option, simple substring patterns + will have to be converted using ``*`` wildcards. + + This affects all the :meth:`loadTestsFrom\*` methods. + + .. versionadded:: 3.7 + .. class:: TestResult diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 7d31932f9656e4..e3b90e673caaf0 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -69,10 +69,10 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. .. method:: request_rate(useragent) Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` in the form of a :func:`~collections.namedtuple` - ``(requests, seconds)``. If there is no such parameter or it doesn't - apply to the *useragent* specified or the ``robots.txt`` entry for this - parameter has invalid syntax, return ``None``. + ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. + If there is no such parameter or it doesn't apply to the *useragent* + specified or the ``robots.txt`` entry for this parameter has invalid + syntax, return ``None``. .. versionadded:: 3.6 diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index ea9ea7dc7d9b82..8ec75a79acfa71 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -156,10 +156,18 @@ The :mod:`uuid` module defines the following functions: Get the hardware address as a 48-bit positive integer. The first time this runs, it may launch a separate program, which could be quite slow. If all - attempts to obtain the hardware address fail, we choose a random 48-bit number - with its eighth bit set to 1 as recommended in RFC 4122. "Hardware address" - means the MAC address of a network interface, and on a machine with multiple - network interfaces the MAC address of any one of them may be returned. + attempts to obtain the hardware address fail, we choose a random 48-bit + number with the multicast bit (least significant bit of the first octet) + set to 1 as recommended in RFC 4122. "Hardware address" means the MAC + address of a network interface. On a machine with multiple network + interfaces, universally administered MAC addresses (i.e. where the second + least significant bit of the first octet is *unset*) will be preferred over + locally administered MAC addresses, but with no other ordering guarantees. + + .. versionchanged:: 3.7 + Universally administered MAC addresses are preferred over locally + administered MAC addresses, since the former are guaranteed to be + globally unique, while the latter are not. .. index:: single: getnode diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 4b425a482096b1..dca9362400455c 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -773,7 +773,6 @@ Is semantically equivalent to:: mgr = (EXPR) aexit = type(mgr).__aexit__ aenter = type(mgr).__aenter__(mgr) - exc = True VAR = await aenter try: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1cff8a52df959d..fb92ad0f07c21e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -183,8 +183,21 @@ by considering each of the :keyword:`for` or :keyword:`if` clauses a block, nesting from left to right, and evaluating the expression to produce an element each time the innermost block is reached. -Note that the comprehension is executed in a separate scope, so names assigned -to in the target list don't "leak" into the enclosing scope. +However, aside from the iterable expression in the leftmost :keyword:`for` clause, +the comprehension is executed in a separate implicitly nested scope. This ensures +that names assigned to in the target list don't "leak" into the enclosing scope. + +The iterable expression in the leftmost :keyword:`for` clause is evaluated +directly in the enclosing scope and then passed as an argument to the implictly +nested scope. Subsequent :keyword:`for` clauses and any filter condition in the +leftmost :keyword:`for` clause cannot be evaluated in the enclosing scope as +they may depend on the values obtained from the leftmost iterable. For example: +``[x*y for x in range(10) for y in range(x, x+10)]``. + +To ensure the comprehension always results in a container of the appropriate +type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly +nested scope (in Python 3.7, such expressions emit :exc:`DeprecationWarning` +when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`). Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. @@ -198,6 +211,13 @@ or :keyword:`await` expressions it is called an suspend the execution of the coroutine function in which it appears. See also :pep:`530`. +.. versionadded:: 3.6 + Asynchronous comprehensions were introduced. + +.. deprecated:: 3.7 + ``yield`` and ``yield from`` deprecated in the implicitly nested scope. + + .. _lists: List displays @@ -316,27 +336,42 @@ brackets or curly braces. Variables used in the generator expression are evaluated lazily when the :meth:`~generator.__next__` method is called for the generator object (in the same -fashion as normal generators). However, the leftmost :keyword:`for` clause is -immediately evaluated, so that an error produced by it can be seen before any -other possible error in the code that handles the generator expression. -Subsequent :keyword:`for` clauses cannot be evaluated immediately since they -may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in -range(10) for y in bar(x))``. +fashion as normal generators). However, the iterable expression in the +leftmost :keyword:`for` clause is immediately evaluated, so that an error +produced by it will be emitted at the point where the generator expression +is defined, rather than at the point where the first value is retrieved. +Subsequent :keyword:`for` clauses and any filter condition in the leftmost +:keyword:`for` clause cannot be evaluated in the enclosing scope as they may +depend on the values obtained from the leftmost iterable. For example: +``(x*y for x in range(10) for y in range(x, x+10))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. +To avoid interfering with the expected operation of the generator expression +itself, ``yield`` and ``yield from`` expressions are prohibited in the +implicitly defined generator (in Python 3.7, such expressions emit +:exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit +:exc:`SyntaxError`). + If a generator expression contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an :dfn:`asynchronous generator expression`. An asynchronous generator expression returns a new asynchronous generator object, which is an asynchronous iterator (see :ref:`async-iterators`). +.. versionadded:: 3.6 + Asynchronous generator expressions were introduced. + .. versionchanged:: 3.7 Prior to Python 3.7, asynchronous generator expressions could only appear in :keyword:`async def` coroutines. Starting with 3.7, any function can use asynchronous generator expressions. +.. deprecated:: 3.7 + ``yield`` and ``yield from`` deprecated in the implicitly nested scope. + + .. _yieldexpr: Yield expressions @@ -364,6 +399,16 @@ coroutine function to be an asynchronous generator. For example:: async def agen(): # defines an asynchronous generator function (PEP 525) yield 123 +Due to their side effects on the containing scope, ``yield`` expressions +are not permitted as part of the implicitly defined scopes used to +implement comprehensions and generator expressions (in Python 3.7, such +expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+ +they will emit :exc:`SyntaxError`).. + +.. deprecated:: 3.7 + Yield expressions deprecated in the implicitly nested scopes used to + implement comprehensions and generator expressions. + Generator functions are described below, while asynchronous generator functions are described separately in section :ref:`asynchronous-generator-functions`. diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 2b3ccf3ac60700..d52f81b76b52f0 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -300,7 +300,7 @@ whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') a whatsnew/3.2,,:location,zope9-location = ${zope9:location} whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf library/re,,`,!#$%&'*+-.^_`|~: -library/re,,`,!\#\$%&'\*\+\-\.\^_`\|~: +library/re,,`,!\#\$%\&'\*\+\-\.\^_`\|\~: library/tarfile,,:xz,'x:xz' library/xml.etree.elementtree,,:sometag,prefix:sometag library/xml.etree.elementtree,,:fictional,">> 'Py' 'thon' 'Python' +This feature is particularly useful when you want to break long strings:: + + >>> text = ('Put several strings within parentheses ' + ... 'to have them joined together.') + >>> text + 'Put several strings within parentheses to have them joined together.' + This only works with two literals though, not with variables or expressions:: >>> prefix = 'Py' @@ -227,13 +234,6 @@ If you want to concatenate variables or a variable and a literal, use ``+``:: >>> prefix + 'thon' 'Python' -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - Strings can be *indexed* (subscripted), with the first character having index 0. There is no separate character type; a character is simply a string of size one:: diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9100494fa1ad5d..d110ae3baf2800 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -212,7 +212,7 @@ Miscellaneous options .. cmdoption:: -d - Turn on parser debugging output (for wizards only, depending on compilation + Turn on parser debugging output (for expert only, depending on compilation options). See also :envvar:`PYTHONDEBUG`. @@ -388,8 +388,6 @@ Miscellaneous options Skip the first line of the source, allowing use of non-Unix forms of ``#!cmd``. This is intended for a DOS specific hack only. - .. note:: The line numbers in error messages will be off by one. - .. cmdoption:: -X @@ -413,6 +411,23 @@ Miscellaneous options nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. + * ``-X dev``: enable CPython's "development mode", introducing additional + runtime checks which are too expensive to be enabled by default. It should + not be more verbose than the default if the code is correct: new warnings + are only emitted when an issue is detected. Effect of the developer mode: + + * Warning filters: add a filter to display all warnings (``"default"`` + action), except of :exc:`BytesWarning` which still depends on the + :option:`-b` option, and use ``"always"`` action for + :exc:`ResourceWarning` warnings. For example, display + :exc:`DeprecationWarning` warnings. + * Install debug hooks on memory allocators: see the + :c:func:`PyMem_SetupDebugHooks` C function. + * Enable the :mod:`faulthandler` module to dump the Python traceback + on a crash. + * Enable :ref:`asyncio debug mode `. + * Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to + ``True`` It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -430,7 +445,7 @@ Miscellaneous options The ``-X showalloccount`` option. .. versionadded:: 3.7 - The ``-X importtime`` and :envvar:`PYTHONPROFILEIMPORTTIME` options. + The ``-X importtime`` and ``-X dev`` options. Options you shouldn't use @@ -674,6 +689,8 @@ conflict. Set the family of memory allocators used by Python: + * ``default``: use the :ref:`default memory allocators + `. * ``malloc``: use the :c:func:`malloc` function of the C library for all domains (:c:data:`PYMEM_DOMAIN_RAW`, :c:data:`PYMEM_DOMAIN_MEM`, :c:data:`PYMEM_DOMAIN_OBJ`). @@ -683,20 +700,17 @@ conflict. Install debug hooks: - * ``debug``: install debug hooks on top of the default memory allocator + * ``debug``: install debug hooks on top of the :ref:`default memory + allocators `. * ``malloc_debug``: same as ``malloc`` but also install debug hooks * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks - When Python is compiled in release mode, the default is ``pymalloc``. When - compiled in debug mode, the default is ``pymalloc_debug`` and the debug hooks - are used automatically. - - If Python is configured without ``pymalloc`` support, ``pymalloc`` and - ``pymalloc_debug`` are not available, the default is ``malloc`` in release - mode and ``malloc_debug`` in debug mode. + See the :ref:`default memory allocators ` and the + :c:func:`PyMem_SetupDebugHooks` function (install debug hooks on Python + memory allocators). - See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python - memory allocators. + .. versionchanged:: 3.7 + Added the ``"default"`` allocator. .. versionadded:: 3.6 @@ -784,6 +798,14 @@ conflict. .. versionadded:: 3.7 See :pep:`538` for more details. + +.. envvar:: PYTHONDEVMODE + + If this environment variable is set to a non-empty string, enable the + CPython "development mode". See the :option:`-X` ``dev`` option. + + .. versionadded:: 3.7 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index f6d051de2eb25e..9363730bf1fd64 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -185,6 +185,19 @@ resolution on Linux and Windows. PEP written and implemented by Victor Stinner +New Development Mode: -X dev +---------------------------- + +Add a new "development mode": ``-X dev`` command line option to enable debug +checks at runtime. + +In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W +default -X faulthandler ...``, except that the PYTHONMALLOC environment +variable is not set in practice. + +See :option:`-X` ``dev`` for the details. + + Other Language Changes ====================== @@ -267,6 +280,9 @@ crypt Added support for the Blowfish method. (Contributed by Serhiy Storchaka in :issue:`31664`.) +The :func:`~crypt.mksalt` function now allows to specify the number of rounds +for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) + dis --- @@ -282,6 +298,10 @@ README.rst is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) +:class:`distutils.core.setup` now warns if the ``classifiers``, ``keywords`` +and ``platforms`` fields are not specified as a list or a string. +(Contributed by Berker Peksag in :issue:`19610`.) + http.client ----------- @@ -342,6 +362,10 @@ The flags :const:`re.ASCII`, :const:`re.LOCALE` and :const:`re.UNICODE` can be set within the scope of a group. (Contributed by Serhiy Storchaka in :issue:`31690`.) +:func:`re.split` now supports splitting on a pattern like ``r'\b'``, +``'^$'`` or ``(?=-)`` that matches an empty string. +(Contributed by Serhiy Storchaka in :issue:`25054`.) + string ------ @@ -349,6 +373,11 @@ string expression pattern for braced placeholders and non-braced placeholders separately. (Contributed by Barry Warsaw in :issue:`1198569`.) +sys +--- + +Added :attr:`sys.flags.dev_mode` flag for the new development mode. + time ---- @@ -372,6 +401,19 @@ Add new clock identifiers: the time the system has been running and not suspended, providing accurate uptime measurement, both absolute and interval. +Added functions :func:`time.thread_time` and :func:`time.thread_time_ns` +to get per-thread CPU time measurements. +(Contributed by Antoine Pitrou in :issue:`32025`.) + + +unittest +-------- +Added new command-line option ``-k`` to filter tests to run with a substring or +Unix shell-like pattern. For example, ``python -m unittest -k foo`` runs the +tests ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, +but not ``bar_tests.FooTest.test_something``. + + unittest.mock ------------- @@ -530,6 +572,18 @@ Other CPython Implementation Changes Deprecated ========== +* Yield expressions (both ``yield`` and ``yield from`` clauses) are now deprecated + in comprehensions and generator expressions (aside from the iterable expression + in the leftmost :keyword:`for` clause). This ensures that comprehensions + always immediately return a container of the appropriate type (rather than + potentially returning a :term:`generator iterator` object), while generator + expressions won't attempt to interleave their implicit output with the output + from any explicit yield expressions. + + In Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, + in Python 3.8+ they will emit :exc:`SyntaxError`. (Contributed by Serhiy + Storchaka in :issue:`10544`.) + - Function :c:func:`PySlice_GetIndicesEx` is deprecated and replaced with a macro if ``Py_LIMITED_API`` is not set or set to the value between ``0x03050400`` and ``0x03060000`` (not including) or ``0x03060100`` or @@ -630,9 +684,38 @@ This section lists previously described changes and other bugfixes that may require changes to your code. +Changes in Python behavior +-------------------------- + +* Due to an oversight, earlier Python versions erroneously accepted the + following syntax:: + + f(1 for x in [1],) + + class C(1 for x in [1]): + pass + + Python 3.7 now correctly raises a :exc:`SyntaxError`, as a generator + expression always needs to be directly inside a set of parentheses + and cannot have a comma on either side, and the duplication of the + parentheses can be omitted only on calls. + (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.) + + Changes in the Python API ------------------------- +* The ``asyncio.windows_utils.socketpair()`` function has been + removed: use directly :func:`socket.socketpair` which is available on all + platforms since Python 3.5 (before, it wasn't available on Windows). + ``asyncio.windows_utils.socketpair()`` was just an alias to + ``socket.socketpair`` on Python 3.5 and newer. + +* :mod:`asyncio`: The module doesn't export :mod:`selectors` and + :mod:`_overlapped` modules as ``asyncio.selectors`` and + ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with + ``import selectors`` for example. + * :meth:`pkgutil.walk_packages` now raises ValueError if *path* is a string. Previously an empty list was returned. (Contributed by Sanyam Khurana in :issue:`24744`.) @@ -678,6 +761,38 @@ Changes in the Python API argument ``os.scandir`` instead of ``os.listdir`` when listing the direcory is failed. +* Support of nested sets and set operations in regular expressions as in + `Unicode Technical Standard #18`_ might be added in the future. This would + change the syntax, so to facilitate this change a :exc:`FutureWarning` will + be raised in ambiguous cases for the time being. + That include sets starting with a literal ``'['`` or containing literal + character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To + avoid a warning escape them with a backslash. + (Contributed by Serhiy Storchaka in :issue:`30349`.) + +* The result of splitting a string on a :mod:`regular expression ` + that could match an empty string has been changed. For example + splitting on ``r'\s*'`` will now split not only on whitespaces as it + did previously, but also between any pair of non-whitespace + characters. The previous behavior can be restored by changing the pattern + to ``r'\s+'``. A :exc:`FutureWarning` was emitted for such patterns since + Python 3.5. + + For patterns that match both empty and non-empty strings, the result of + searching for all matches may also be changed in other cases. For example + in the string ``'a\n\n'``, the pattern ``r'(?m)^\s*?$'`` will not only + match empty strings at positions 2 and 3, but also the string ``'\n'`` at + positions 2--3. To match only blank lines, the pattern should be rewritten + as ``r'(?m)^[^\S\n]*$'``. + + (Contributed by Serhiy Storchaka in :issue:`25054`.) + +* :class:`tracemalloc.Traceback` frames are now sorted from oldest to most + recent to be more consistent with :mod:`traceback`. + (Contributed by Jesse Bakker in :issue:`32121`.) + +.. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + Changes in the C API -------------------- diff --git a/Include/bitset.h b/Include/bitset.h index faeb41913df31f..b22fa77815cfb8 100644 --- a/Include/bitset.h +++ b/Include/bitset.h @@ -7,7 +7,7 @@ extern "C" { /* Bitset interface */ -#define BYTE char +#define BYTE char typedef BYTE *bitset; @@ -18,13 +18,13 @@ int addbit(bitset bs, int ibit); /* Returns 0 if already set */ int samebitset(bitset bs1, bitset bs2, int nbits); void mergebitset(bitset bs1, bitset bs2, int nbits); -#define BITSPERBYTE (8*sizeof(BYTE)) -#define NBYTES(nbits) (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE) +#define BITSPERBYTE (8*sizeof(BYTE)) +#define NBYTES(nbits) (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE) -#define BIT2BYTE(ibit) ((ibit) / BITSPERBYTE) -#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE) -#define BIT2MASK(ibit) (1 << BIT2SHIFT(ibit)) -#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE) +#define BIT2BYTE(ibit) ((ibit) / BITSPERBYTE) +#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE) +#define BIT2MASK(ibit) (1 << BIT2SHIFT(ibit)) +#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE) #ifdef __cplusplus } diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 0f0bf9f369df4f..3fde4a221fdb18 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -52,9 +52,9 @@ PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) - Py_GCC_ATTRIBUTE((format(printf, 1, 0))); + Py_GCC_ATTRIBUTE((format(printf, 1, 0))); PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); @@ -72,8 +72,8 @@ PyAPI_FUNC(PyObject*) _PyBytes_FromHex( int use_bytearray); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, Py_ssize_t, - const char *); + const char *, Py_ssize_t, + const char *); #ifndef Py_LIMITED_API /* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, @@ -132,10 +132,10 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer, /* Flags used by string formatting */ #define F_LJUST (1<<0) -#define F_SIGN (1<<1) +#define F_SIGN (1<<1) #define F_BLANK (1<<2) -#define F_ALT (1<<3) -#define F_ZERO (1<<4) +#define F_ALT (1<<3) +#define F_ZERO (1<<4) #ifndef Py_LIMITED_API /* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". diff --git a/Include/cellobject.h b/Include/cellobject.h index a0aa4d947c2211..2f9b5b75d998ae 100644 --- a/Include/cellobject.h +++ b/Include/cellobject.h @@ -7,8 +7,8 @@ extern "C" { #endif typedef struct { - PyObject_HEAD - PyObject *ob_ref; /* Content of the cell or NULL when empty */ + PyObject_HEAD + PyObject *ob_ref; /* Content of the cell or NULL when empty */ } PyCellObject; PyAPI_DATA(PyTypeObject) PyCell_Type; diff --git a/Include/classobject.h b/Include/classobject.h index eeeb3e95a87e71..209f0f4a284301 100644 --- a/Include/classobject.h +++ b/Include/classobject.h @@ -30,13 +30,13 @@ PyAPI_FUNC(PyObject *) PyMethod_Self(PyObject *); #define PyMethod_GET_FUNCTION(meth) \ (((PyMethodObject *)meth) -> im_func) #define PyMethod_GET_SELF(meth) \ - (((PyMethodObject *)meth) -> im_self) + (((PyMethodObject *)meth) -> im_self) PyAPI_FUNC(int) PyMethod_ClearFreeList(void); typedef struct { - PyObject_HEAD - PyObject *func; + PyObject_HEAD + PyObject *func; } PyInstanceMethodObject; PyAPI_DATA(PyTypeObject) PyInstanceMethod_Type; diff --git a/Include/code.h b/Include/code.h index 385258f93ce491..8b0f84042372d9 100644 --- a/Include/code.h +++ b/Include/code.h @@ -20,17 +20,17 @@ typedef uint16_t _Py_CODEUNIT; /* Bytecode object */ typedef struct { PyObject_HEAD - int co_argcount; /* #arguments, except *args */ - int co_kwonlyargcount; /* #keyword only arguments */ - int co_nlocals; /* #local variables */ - int co_stacksize; /* #entries needed for evaluation stack */ - int co_flags; /* CO_..., see below */ - int co_firstlineno; /* first source line number */ - PyObject *co_code; /* instruction opcodes */ - PyObject *co_consts; /* list (constants used) */ - PyObject *co_names; /* list of strings (names used) */ - PyObject *co_varnames; /* tuple of strings (local variable names) */ - PyObject *co_freevars; /* tuple of strings (free variable names) */ + int co_argcount; /* #arguments, except *args */ + int co_kwonlyargcount; /* #keyword only arguments */ + int co_nlocals; /* #local variables */ + int co_stacksize; /* #entries needed for evaluation stack */ + int co_flags; /* CO_..., see below */ + int co_firstlineno; /* first source line number */ + PyObject *co_code; /* instruction opcodes */ + PyObject *co_consts; /* list (constants used) */ + PyObject *co_names; /* list of strings (names used) */ + PyObject *co_varnames; /* tuple of strings (local variable names) */ + PyObject *co_freevars; /* tuple of strings (free variable names) */ PyObject *co_cellvars; /* tuple of strings (cell variable names) */ /* The rest aren't used in either hash or comparisons, except for co_name, used in both. This is done to preserve the name and line number @@ -38,11 +38,11 @@ typedef struct { would collapse identical functions/lambdas defined on different lines. */ Py_ssize_t *co_cell2arg; /* Maps cell vars which are arguments. */ - PyObject *co_filename; /* unicode (where it was loaded from) */ - PyObject *co_name; /* unicode (name, for reference) */ - PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) See - Objects/lnotab_notes.txt for details. */ - void *co_zombieframe; /* for optimization only (see frameobject.c) */ + PyObject *co_filename; /* unicode (where it was loaded from) */ + PyObject *co_name; /* unicode (name, for reference) */ + PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) See + Objects/lnotab_notes.txt for details. */ + void *co_zombieframe; /* for optimization only (see frameobject.c) */ PyObject *co_weakreflist; /* to support weakrefs to code objects */ /* Scratch space for extra data relating to the code object. Type is a void* to keep the format private in codeobject.c to force @@ -51,10 +51,10 @@ typedef struct { } PyCodeObject; /* Masks for co_flags above */ -#define CO_OPTIMIZED 0x0001 -#define CO_NEWLOCALS 0x0002 -#define CO_VARARGS 0x0004 -#define CO_VARKEYWORDS 0x0008 +#define CO_OPTIMIZED 0x0001 +#define CO_NEWLOCALS 0x0002 +#define CO_VARARGS 0x0004 +#define CO_VARKEYWORDS 0x0008 #define CO_NESTED 0x0010 #define CO_GENERATOR 0x0020 /* The CO_NOFREE flag is set if there are no free or cell variables. @@ -74,7 +74,7 @@ typedef struct { #if 0 #define CO_GENERATOR_ALLOWED 0x1000 #endif -#define CO_FUTURE_DIVISION 0x2000 +#define CO_FUTURE_DIVISION 0x2000 #define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */ #define CO_FUTURE_WITH_STATEMENT 0x8000 #define CO_FUTURE_PRINT_FUNCTION 0x10000 @@ -101,9 +101,9 @@ PyAPI_DATA(PyTypeObject) PyCode_Type; /* Public interface */ PyAPI_FUNC(PyCodeObject *) PyCode_New( - int, int, int, int, int, PyObject *, PyObject *, - PyObject *, PyObject *, PyObject *, PyObject *, - PyObject *, PyObject *, int, PyObject *); + int, int, int, int, int, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, PyObject *, + PyObject *, PyObject *, int, PyObject *); /* same as struct above */ /* Creates a new empty code object with the specified source location. */ diff --git a/Include/errcode.h b/Include/errcode.h index 5946686c659b54..b37cd261d5ec4d 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -13,24 +13,24 @@ extern "C" { the parser only returns E_EOF when it hits EOF immediately, and it never returns E_OK. */ -#define E_OK 10 /* No error */ -#define E_EOF 11 /* End Of File */ -#define E_INTR 12 /* Interrupted */ -#define E_TOKEN 13 /* Bad token */ -#define E_SYNTAX 14 /* Syntax error */ -#define E_NOMEM 15 /* Ran out of memory */ -#define E_DONE 16 /* Parsing complete */ -#define E_ERROR 17 /* Execution error */ -#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ -#define E_OVERFLOW 19 /* Node had too many children */ -#define E_TOODEEP 20 /* Too many indentation levels */ -#define E_DEDENT 21 /* No matching outer block for dedent */ -#define E_DECODE 22 /* Error in decoding into Unicode */ -#define E_EOFS 23 /* EOF in triple-quoted string */ -#define E_EOLS 24 /* EOL in single-quoted string */ -#define E_LINECONT 25 /* Unexpected characters after a line continuation */ +#define E_OK 10 /* No error */ +#define E_EOF 11 /* End Of File */ +#define E_INTR 12 /* Interrupted */ +#define E_TOKEN 13 /* Bad token */ +#define E_SYNTAX 14 /* Syntax error */ +#define E_NOMEM 15 /* Ran out of memory */ +#define E_DONE 16 /* Parsing complete */ +#define E_ERROR 17 /* Execution error */ +#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ +#define E_OVERFLOW 19 /* Node had too many children */ +#define E_TOODEEP 20 /* Too many indentation levels */ +#define E_DEDENT 21 /* No matching outer block for dedent */ +#define E_DECODE 22 /* Error in decoding into Unicode */ +#define E_EOFS 23 /* EOF in triple-quoted string */ +#define E_EOLS 24 /* EOL in single-quoted string */ +#define E_LINECONT 25 /* Unexpected characters after a line continuation */ #define E_IDENTIFIER 26 /* Invalid characters in identifier */ -#define E_BADSINGLE 27 /* Ill-formed single statement input */ +#define E_BADSINGLE 27 /* Ill-formed single statement input */ #ifdef __cplusplus } diff --git a/Include/funcobject.h b/Include/funcobject.h index 77bb8c39aeb750..89305dcf768687 100644 --- a/Include/funcobject.h +++ b/Include/funcobject.h @@ -20,17 +20,17 @@ extern "C" { typedef struct { PyObject_HEAD - PyObject *func_code; /* A code object, the __code__ attribute */ - PyObject *func_globals; /* A dictionary (other mappings won't do) */ - PyObject *func_defaults; /* NULL or a tuple */ - PyObject *func_kwdefaults; /* NULL or a dict */ - PyObject *func_closure; /* NULL or a tuple of cell objects */ - PyObject *func_doc; /* The __doc__ attribute, can be anything */ - PyObject *func_name; /* The __name__ attribute, a string object */ - PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ - PyObject *func_weakreflist; /* List of weak references */ - PyObject *func_module; /* The __module__ attribute, can be anything */ - PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_code; /* A code object, the __code__ attribute */ + PyObject *func_globals; /* A dictionary (other mappings won't do) */ + PyObject *func_defaults; /* NULL or a tuple */ + PyObject *func_kwdefaults; /* NULL or a dict */ + PyObject *func_closure; /* NULL or a tuple of cell objects */ + PyObject *func_doc; /* The __doc__ attribute, can be anything */ + PyObject *func_name; /* The __name__ attribute, a string object */ + PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ + PyObject *func_weakreflist; /* List of weak references */ + PyObject *func_module; /* The __module__ attribute, can be anything */ + PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_qualname; /* The qualified name */ /* Invariant: @@ -77,17 +77,17 @@ PyAPI_FUNC(PyObject *) _PyFunction_FastCallKeywords( #define PyFunction_GET_CODE(func) \ (((PyFunctionObject *)func) -> func_code) #define PyFunction_GET_GLOBALS(func) \ - (((PyFunctionObject *)func) -> func_globals) + (((PyFunctionObject *)func) -> func_globals) #define PyFunction_GET_MODULE(func) \ - (((PyFunctionObject *)func) -> func_module) + (((PyFunctionObject *)func) -> func_module) #define PyFunction_GET_DEFAULTS(func) \ - (((PyFunctionObject *)func) -> func_defaults) + (((PyFunctionObject *)func) -> func_defaults) #define PyFunction_GET_KW_DEFAULTS(func) \ - (((PyFunctionObject *)func) -> func_kwdefaults) + (((PyFunctionObject *)func) -> func_kwdefaults) #define PyFunction_GET_CLOSURE(func) \ - (((PyFunctionObject *)func) -> func_closure) + (((PyFunctionObject *)func) -> func_closure) #define PyFunction_GET_ANNOTATIONS(func) \ - (((PyFunctionObject *)func) -> func_annotations) + (((PyFunctionObject *)func) -> func_annotations) /* The classmethod and staticmethod types lives here, too */ PyAPI_DATA(PyTypeObject) PyClassMethod_Type; diff --git a/Include/grammar.h b/Include/grammar.h index f775f9638172b3..e1703f4b360344 100644 --- a/Include/grammar.h +++ b/Include/grammar.h @@ -12,58 +12,58 @@ extern "C" { /* A label of an arc */ typedef struct { - int lb_type; - char *lb_str; + int lb_type; + char *lb_str; } label; -#define EMPTY 0 /* Label number 0 is by definition the empty label */ +#define EMPTY 0 /* Label number 0 is by definition the empty label */ /* A list of labels */ typedef struct { - int ll_nlabels; - label *ll_label; + int ll_nlabels; + label *ll_label; } labellist; /* An arc from one state to another */ typedef struct { - short a_lbl; /* Label of this arc */ - short a_arrow; /* State where this arc goes to */ + short a_lbl; /* Label of this arc */ + short a_arrow; /* State where this arc goes to */ } arc; /* A state in a DFA */ typedef struct { - int s_narcs; - arc *s_arc; /* Array of arcs */ + int s_narcs; + arc *s_arc; /* Array of arcs */ /* Optional accelerators */ - int s_lower; /* Lowest label index */ - int s_upper; /* Highest label index */ - int *s_accel; /* Accelerator */ - int s_accept; /* Nonzero for accepting state */ + int s_lower; /* Lowest label index */ + int s_upper; /* Highest label index */ + int *s_accel; /* Accelerator */ + int s_accept; /* Nonzero for accepting state */ } state; /* A DFA */ typedef struct { - int d_type; /* Non-terminal this represents */ - char *d_name; /* For printing */ - int d_initial; /* Initial state */ - int d_nstates; - state *d_state; /* Array of states */ - bitset d_first; + int d_type; /* Non-terminal this represents */ + char *d_name; /* For printing */ + int d_initial; /* Initial state */ + int d_nstates; + state *d_state; /* Array of states */ + bitset d_first; } dfa; /* A grammar */ typedef struct { - int g_ndfas; - dfa *g_dfa; /* Array of DFAs */ - labellist g_ll; - int g_start; /* Start symbol of the grammar */ - int g_accel; /* Set if accelerators present */ + int g_ndfas; + dfa *g_dfa; /* Array of DFAs */ + labellist g_ll; + int g_start; /* Start symbol of the grammar */ + int g_accel; /* Set if accelerators present */ } grammar; /* FUNCTIONS */ diff --git a/Include/import.h b/Include/import.h index c30f3ea33942fa..26c4b1f18ef2c1 100644 --- a/Include/import.h +++ b/Include/import.h @@ -8,7 +8,7 @@ extern "C" { #endif #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyImportZip_Init(void); +PyAPI_FUNC(_PyInitError) _PyImportZip_Init(void); PyMODINIT_FUNC PyInit_imp(void); #endif /* !Py_LIMITED_API */ diff --git a/Include/internal/mem.h b/Include/internal/mem.h index 471cdf45df2766..a731e30e6af7d2 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -7,54 +7,6 @@ extern "C" { #include "objimpl.h" #include "pymem.h" -#ifdef WITH_PYMALLOC -#include "internal/pymalloc.h" -#endif - -/* Low-level memory runtime state */ - -struct _pymem_runtime_state { - struct _allocator_runtime_state { - PyMemAllocatorEx mem; - PyMemAllocatorEx obj; - PyMemAllocatorEx raw; - } allocators; -#ifdef WITH_PYMALLOC - /* Array of objects used to track chunks of memory (arenas). */ - struct arena_object* arenas; - /* The head of the singly-linked, NULL-terminated list of available - arena_objects. */ - struct arena_object* unused_arena_objects; - /* The head of the doubly-linked, NULL-terminated at each end, - list of arena_objects associated with arenas that have pools - available. */ - struct arena_object* usable_arenas; - /* Number of slots currently allocated in the `arenas` vector. */ - unsigned int maxarenas; - /* Number of arenas allocated that haven't been free()'d. */ - size_t narenas_currently_allocated; - /* High water mark (max value ever seen) for - * narenas_currently_allocated. */ - size_t narenas_highwater; - /* Total number of times malloc() called to allocate an arena. */ - size_t ntimes_arena_allocated; - poolp usedpools[MAX_POOLS]; - Py_ssize_t num_allocated_blocks; -#endif /* WITH_PYMALLOC */ - size_t serialno; /* incremented on each debug {m,re}alloc */ -}; - -PyAPI_FUNC(void) _PyMem_Initialize(struct _pymem_runtime_state *); - - -/* High-level memory runtime state */ - -struct _pyobj_runtime_state { - PyObjectArenaAllocator allocator_arenas; -}; - -PyAPI_FUNC(void) _PyObject_Initialize(struct _pyobj_runtime_state *); - /* GC runtime state */ diff --git a/Include/internal/pymalloc.h b/Include/internal/pymalloc.h deleted file mode 100644 index 723d9e7e671a8e..00000000000000 --- a/Include/internal/pymalloc.h +++ /dev/null @@ -1,443 +0,0 @@ - -/* An object allocator for Python. - - Here is an introduction to the layers of the Python memory architecture, - showing where the object allocator is actually used (layer +2), It is - called for every object allocation and deallocation (PyObject_New/Del), - unless the object-specific allocators implement a proprietary allocation - scheme (ex.: ints use a simple free list). This is also the place where - the cyclic garbage collector operates selectively on container objects. - - - Object-specific allocators - _____ ______ ______ ________ - [ int ] [ dict ] [ list ] ... [ string ] Python core | -+3 | <----- Object-specific memory -----> | <-- Non-object memory --> | - _______________________________ | | - [ Python's object allocator ] | | -+2 | ####### Object memory ####### | <------ Internal buffers ------> | - ______________________________________________________________ | - [ Python's raw memory allocator (PyMem_ API) ] | -+1 | <----- Python memory (under PyMem manager's control) ------> | | - __________________________________________________________________ - [ Underlying general-purpose allocator (ex: C library malloc) ] - 0 | <------ Virtual memory allocated for the python process -------> | - - ========================================================================= - _______________________________________________________________________ - [ OS-specific Virtual Memory Manager (VMM) ] --1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | - __________________________________ __________________________________ - [ ] [ ] --2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | - -*/ -/*==========================================================================*/ - -/* A fast, special-purpose memory allocator for small blocks, to be used - on top of a general-purpose malloc -- heavily based on previous art. */ - -/* Vladimir Marangozov -- August 2000 */ - -/* - * "Memory management is where the rubber meets the road -- if we do the wrong - * thing at any level, the results will not be good. And if we don't make the - * levels work well together, we are in serious trouble." (1) - * - * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, - * "Dynamic Storage Allocation: A Survey and Critical Review", - * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. - */ - -#ifndef Py_INTERNAL_PYMALLOC_H -#define Py_INTERNAL_PYMALLOC_H - -/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ - -/*==========================================================================*/ - -/* - * Allocation strategy abstract: - * - * For small requests, the allocator sub-allocates blocks of memory. - * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the - * system's allocator. - * - * Small requests are grouped in size classes spaced 8 bytes apart, due - * to the required valid alignment of the returned address. Requests of - * a particular size are serviced from memory pools of 4K (one VMM page). - * Pools are fragmented on demand and contain free lists of blocks of one - * particular size class. In other words, there is a fixed-size allocator - * for each size class. Free pools are shared by the different allocators - * thus minimizing the space reserved for a particular size class. - * - * This allocation strategy is a variant of what is known as "simple - * segregated storage based on array of free lists". The main drawback of - * simple segregated storage is that we might end up with lot of reserved - * memory for the different free lists, which degenerate in time. To avoid - * this, we partition each free list in pools and we share dynamically the - * reserved space between all free lists. This technique is quite efficient - * for memory intensive programs which allocate mainly small-sized blocks. - * - * For small requests we have the following table: - * - * Request in bytes Size of allocated block Size class idx - * ---------------------------------------------------------------- - * 1-8 8 0 - * 9-16 16 1 - * 17-24 24 2 - * 25-32 32 3 - * 33-40 40 4 - * 41-48 48 5 - * 49-56 56 6 - * 57-64 64 7 - * 65-72 72 8 - * ... ... ... - * 497-504 504 62 - * 505-512 512 63 - * - * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying - * allocator. - */ - -/*==========================================================================*/ - -/* - * -- Main tunable settings section -- - */ - -/* - * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address busses). - * The alignment value is also used for grouping small requests in size - * classes spaced ALIGNMENT bytes apart. - * - * You shouldn't change this unless you know what you are doing. - */ -#define ALIGNMENT 8 /* must be 2^N */ -#define ALIGNMENT_SHIFT 3 - -/* Return the number of bytes in size class I, as a uint. */ -#define INDEX2SIZE(I) (((unsigned int)(I) + 1) << ALIGNMENT_SHIFT) - -/* - * Max size threshold below which malloc requests are considered to be - * small enough in order to use preallocated memory pools. You can tune - * this value according to your application behaviour and memory needs. - * - * Note: a size threshold of 512 guarantees that newly created dictionaries - * will be allocated from preallocated memory pools on 64-bit. - * - * The following invariants must hold: - * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 - * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT - * - * Although not required, for better performance and space efficiency, - * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. - */ -#define SMALL_REQUEST_THRESHOLD 512 -#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) - -#if NB_SMALL_SIZE_CLASSES > 64 -#error "NB_SMALL_SIZE_CLASSES should be less than 64" -#endif /* NB_SMALL_SIZE_CLASSES > 64 */ - -/* - * The system's VMM page size can be obtained on most unices with a - * getpagesize() call or deduced from various header files. To make - * things simpler, we assume that it is 4K, which is OK for most systems. - * It is probably better if this is the native page size, but it doesn't - * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page - * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation - * violation fault. 4K is apparently OK for all the platforms that python - * currently targets. - */ -#define SYSTEM_PAGE_SIZE (4 * 1024) -#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) - -/* - * Maximum amount of memory managed by the allocator for small requests. - */ -#ifdef WITH_MEMORY_LIMITS -#ifndef SMALL_MEMORY_LIMIT -#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MiB -- more? */ -#endif -#endif - -/* - * The allocator sub-allocates blocks of memory (called arenas) aligned - * on a page boundary. This is a reserved virtual address space for the - * current process (obtained through a malloc()/mmap() call). In no way this - * means that the memory arenas will be used entirely. A malloc() is - * usually an address range reservation for bytes, unless all pages within - * this space are referenced subsequently. So malloc'ing big blocks and not - * using them does not mean "wasting memory". It's an addressable range - * wastage... - * - * Arenas are allocated with mmap() on systems supporting anonymous memory - * mappings to reduce heap fragmentation. - */ -#define ARENA_SIZE (256 << 10) /* 256 KiB */ - -#ifdef WITH_MEMORY_LIMITS -#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) -#endif - -/* - * Size of the pools used for small blocks. Should be a power of 2, - * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. - */ -#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ -#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK - -/* - * -- End of tunable settings section -- - */ - -/*==========================================================================*/ - -/* - * Locking - * - * To reduce lock contention, it would probably be better to refine the - * crude function locking with per size class locking. I'm not positive - * however, whether it's worth switching to such locking policy because - * of the performance penalty it might introduce. - * - * The following macros describe the simplest (should also be the fastest) - * lock object on a particular platform and the init/fini/lock/unlock - * operations on it. The locks defined here are not expected to be recursive - * because it is assumed that they will always be called in the order: - * INIT, [LOCK, UNLOCK]*, FINI. - */ - -/* - * Python's threads are serialized, so object malloc locking is disabled. - */ -#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ -#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ -#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ -#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ -#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ - -/* When you say memory, my mind reasons in terms of (pointers to) blocks */ -typedef uint8_t pyblock; - -/* Pool for small blocks. */ -struct pool_header { - union { pyblock *_padding; - unsigned int count; } ref; /* number of allocated blocks */ - pyblock *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ - unsigned int arenaindex; /* index into arenas of base adr */ - unsigned int szidx; /* block size class index */ - unsigned int nextoffset; /* bytes to virgin block */ - unsigned int maxnextoffset; /* largest valid nextoffset */ -}; - -typedef struct pool_header *poolp; - -/* Record keeping for arenas. */ -struct arena_object { - /* The address of the arena, as returned by malloc. Note that 0 - * will never be returned by a successful malloc, and is used - * here to mark an arena_object that doesn't correspond to an - * allocated arena. - */ - uintptr_t address; - - /* Pool-aligned pointer to the next pool to be carved off. */ - pyblock* pool_address; - - /* The number of available pools in the arena: free pools + never- - * allocated pools. - */ - unsigned int nfreepools; - - /* The total number of pools in the arena, whether or not available. */ - unsigned int ntotalpools; - - /* Singly-linked list of available pools. */ - struct pool_header* freepools; - - /* Whenever this arena_object is not associated with an allocated - * arena, the nextarena member is used to link all unassociated - * arena_objects in the singly-linked `unused_arena_objects` list. - * The prevarena member is unused in this case. - * - * When this arena_object is associated with an allocated arena - * with at least one available pool, both members are used in the - * doubly-linked `usable_arenas` list, which is maintained in - * increasing order of `nfreepools` values. - * - * Else this arena_object is associated with an allocated arena - * all of whose pools are in use. `nextarena` and `prevarena` - * are both meaningless in this case. - */ - struct arena_object* nextarena; - struct arena_object* prevarena; -}; - -#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) - -#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ - -/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) - -/* Return total number of blocks in pool of size index I, as a uint. */ -#define NUMBLOCKS(I) \ - ((unsigned int)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) - -/*==========================================================================*/ - -/* - * This malloc lock - */ -SIMPLELOCK_DECL(_malloc_lock) -#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) -#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) -#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) -#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) - -/* - * Pool table -- headed, circular, doubly-linked lists of partially used pools. - -This is involved. For an index i, usedpools[i+i] is the header for a list of -all partially used pools holding small blocks with "size class idx" i. So -usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size -16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to -the start of a singly-linked list of free blocks within the pool. When a -block is freed, it's inserted at the front of its pool's freeblock list. Note -that the available blocks in a pool are *not* linked all together when a pool -is initialized. Instead only "the first two" (lowest addresses) blocks are -set up, returning the first such block, and setting pool->freeblock to a -one-block list holding the second such block. This is consistent with that -pymalloc strives at all levels (arena, pool, and block) never to touch a piece -of memory until it's actually needed. - -So long as a pool is in the used state, we're certain there *is* a block -available for allocating, and pool->freeblock is not NULL. If pool->freeblock -points to the end of the free list before we've carved the entire pool into -blocks, that means we simply haven't yet gotten to one of the higher-address -blocks. The offset from the pool_header to the start of "the next" virgin -block is stored in the pool_header nextoffset member, and the largest value -of nextoffset that makes sense is stored in the maxnextoffset member when a -pool is initialized. All the blocks in a pool have been passed out at least -once when and only when nextoffset > maxnextoffset. - - -Major obscurity: While the usedpools vector is declared to have poolp -entries, it doesn't really. It really contains two pointers per (conceptual) -poolp entry, the nextpool and prevpool members of a pool_header. The -excruciating initialization code below fools C so that - - usedpool[i+i] - -"acts like" a genuine poolp, but only so long as you only reference its -nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is -compensating for that a pool_header's nextpool and prevpool members -immediately follow a pool_header's first two members: - - union { block *_padding; - uint count; } ref; - block *freeblock; - -each of which consume sizeof(block *) bytes. So what usedpools[i+i] really -contains is a fudged-up pointer p such that *if* C believes it's a poolp -pointer, then p->nextpool and p->prevpool are both p (meaning that the headed -circular list is empty). - -It's unclear why the usedpools setup is so convoluted. It could be to -minimize the amount of cache required to hold this heavily-referenced table -(which only *needs* the two interpool pointer members of a pool_header). OTOH, -referencing code has to remember to "double the index" and doing so isn't -free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying -on that C doesn't insert any padding anywhere in a pool_header at or before -the prevpool member. -**************************************************************************** */ - -#define MAX_POOLS (2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8) - -/*========================================================================== -Arena management. - -`arenas` is a vector of arena_objects. It contains maxarenas entries, some of -which may not be currently used (== they're arena_objects that aren't -currently associated with an allocated arena). Note that arenas proper are -separately malloc'ed. - -Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, -we do try to free() arenas, and use some mild heuristic strategies to increase -the likelihood that arenas eventually can be freed. - -unused_arena_objects - - This is a singly-linked list of the arena_objects that are currently not - being used (no arena is associated with them). Objects are taken off the - head of the list in new_arena(), and are pushed on the head of the list in - PyObject_Free() when the arena is empty. Key invariant: an arena_object - is on this list if and only if its .address member is 0. - -usable_arenas - - This is a doubly-linked list of the arena_objects associated with arenas - that have pools available. These pools are either waiting to be reused, - or have not been used before. The list is sorted to have the most- - allocated arenas first (ascending order based on the nfreepools member). - This means that the next allocation will come from a heavily used arena, - which gives the nearly empty arenas a chance to be returned to the system. - In my unscientific tests this dramatically improved the number of arenas - that could be freed. - -Note that an arena_object associated with an arena all of whose pools are -currently in use isn't on either list. -*/ - -/* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4 MiB before growing the - * `arenas` vector. - */ -#define INITIAL_ARENA_OBJECTS 16 - -#endif /* Py_INTERNAL_PYMALLOC_H */ diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 210917bb325949..6b527fbe2144ee 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -37,6 +37,34 @@ struct _gilstate_runtime_state { #define _PyGILState_check_enabled _PyRuntime.gilstate.check_enabled +typedef struct { + /* Full path to the Python program */ + wchar_t *program_full_path; + wchar_t *prefix; +#ifdef MS_WINDOWS + wchar_t *dll_path; +#else + wchar_t *exec_prefix; +#endif + /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */ + wchar_t *module_search_path; + /* Python program name */ + wchar_t *program_name; + /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ + wchar_t *home; +} _PyPathConfig; + +#define _PyPathConfig_INIT {.module_search_path = NULL} +/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ + +PyAPI_DATA(_PyPathConfig) _Py_path_config; + +PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( + _PyPathConfig *config, + const _PyMainInterpreterConfig *main_config); +PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); + + /* Full Python runtime state */ typedef struct pyruntimestate { @@ -64,9 +92,7 @@ typedef struct pyruntimestate { int nexitfuncs; void (*pyexitfunc)(void); - struct _pyobj_runtime_state obj; struct _gc_runtime_state gc; - struct _pymem_runtime_state mem; struct _warnings_runtime_state warnings; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; @@ -74,17 +100,24 @@ typedef struct pyruntimestate { // XXX Consolidate globals found via the check-c-globals script. } _PyRuntimeState; +#define _PyRuntimeState_INIT {.initialized = 0, .core_initialized = 0} +/* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */ + PyAPI_DATA(_PyRuntimeState) _PyRuntime; -PyAPI_FUNC(void) _PyRuntimeState_Init(_PyRuntimeState *); +PyAPI_FUNC(_PyInitError) _PyRuntimeState_Init(_PyRuntimeState *); PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *); +/* Initialize _PyRuntimeState. + Return NULL on success, or return an error message on failure. */ +PyAPI_FUNC(_PyInitError) _PyRuntime_Initialize(void); + #define _Py_CURRENTLY_FINALIZING(tstate) \ (_PyRuntime.finalizing == tstate) /* Other */ -PyAPI_FUNC(void) _PyInterpreterState_Enable(_PyRuntimeState *); +PyAPI_FUNC(_PyInitError) _PyInterpreterState_Enable(_PyRuntimeState *); #ifdef __cplusplus } diff --git a/Include/longintrepr.h b/Include/longintrepr.h index a3b74b4f6dd214..ff4155f9656def 100644 --- a/Include/longintrepr.h +++ b/Include/longintrepr.h @@ -46,22 +46,22 @@ typedef uint32_t digit; typedef int32_t sdigit; /* signed variant of digit */ typedef uint64_t twodigits; typedef int64_t stwodigits; /* signed variant of twodigits */ -#define PyLong_SHIFT 30 -#define _PyLong_DECIMAL_SHIFT 9 /* max(e such that 10**e fits in a digit) */ -#define _PyLong_DECIMAL_BASE ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */ +#define PyLong_SHIFT 30 +#define _PyLong_DECIMAL_SHIFT 9 /* max(e such that 10**e fits in a digit) */ +#define _PyLong_DECIMAL_BASE ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */ #elif PYLONG_BITS_IN_DIGIT == 15 typedef unsigned short digit; typedef short sdigit; /* signed variant of digit */ typedef unsigned long twodigits; typedef long stwodigits; /* signed variant of twodigits */ -#define PyLong_SHIFT 15 -#define _PyLong_DECIMAL_SHIFT 4 /* max(e such that 10**e fits in a digit) */ -#define _PyLong_DECIMAL_BASE ((digit)10000) /* 10 ** DECIMAL_SHIFT */ +#define PyLong_SHIFT 15 +#define _PyLong_DECIMAL_SHIFT 4 /* max(e such that 10**e fits in a digit) */ +#define _PyLong_DECIMAL_BASE ((digit)10000) /* 10 ** DECIMAL_SHIFT */ #else #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" #endif -#define PyLong_BASE ((digit)1 << PyLong_SHIFT) -#define PyLong_MASK ((digit)(PyLong_BASE - 1)) +#define PyLong_BASE ((digit)1 << PyLong_SHIFT) +#define PyLong_MASK ((digit)(PyLong_BASE - 1)) #if PyLong_SHIFT % 5 != 0 #error "longobject.c requires that PyLong_SHIFT be divisible by 5" @@ -69,12 +69,12 @@ typedef long stwodigits; /* signed variant of twodigits */ /* Long integer representation. The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) + SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) Negative numbers are represented with ob_size < 0; zero is represented by ob_size == 0. In a normalized number, ob_digit[abs(ob_size)-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= MASK. The allocation function takes care of allocating extra memory so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. @@ -83,8 +83,8 @@ typedef long stwodigits; /* signed variant of twodigits */ */ struct _longobject { - PyObject_VAR_HEAD - digit ob_digit[1]; + PyObject_VAR_HEAD + digit ob_digit[1]; }; PyAPI_FUNC(PyLongObject *) _PyLong_New(Py_ssize_t); diff --git a/Include/node.h b/Include/node.h index 654ad858230145..40596dfecf1807 100644 --- a/Include/node.h +++ b/Include/node.h @@ -8,12 +8,12 @@ extern "C" { #endif typedef struct _node { - short n_type; - char *n_str; - int n_lineno; - int n_col_offset; - int n_nchildren; - struct _node *n_child; + short n_type; + char *n_str; + int n_lineno; + int n_col_offset; + int n_nchildren; + struct _node *n_child; } node; PyAPI_FUNC(node *) PyNode_New(int type); @@ -25,12 +25,12 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); #endif /* Node access functions */ -#define NCH(n) ((n)->n_nchildren) +#define NCH(n) ((n)->n_nchildren) -#define CHILD(n, i) (&(n)->n_child[i]) -#define RCHILD(n, i) (CHILD(n, NCH(n) + i)) -#define TYPE(n) ((n)->n_type) -#define STR(n) ((n)->n_str) +#define CHILD(n, i) (&(n)->n_child[i]) +#define RCHILD(n, i) (CHILD(n, NCH(n) + i)) +#define TYPE(n) ((n)->n_type) +#define STR(n) ((n)->n_str) #define LINENO(n) ((n)->n_lineno) /* Assert that the type of a node is what we expect */ diff --git a/Include/object.h b/Include/object.h index c65f948709fd15..7db5bfea615e98 100644 --- a/Include/object.h +++ b/Include/object.h @@ -728,7 +728,6 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void); /* Py_REF_DEBUG also controls the display of refcounts and memory block * allocations at the interactive prompt and at interpreter shutdown */ -PyAPI_FUNC(PyObject *) _PyDebug_XOptionShowRefCount(void); PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void); #else #define _Py_INC_REFTOTAL diff --git a/Include/parsetok.h b/Include/parsetok.h index 2acb854672774d..c9407a3f7020fe 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -21,13 +21,13 @@ typedef struct { } perrdetail; #if 0 -#define PyPARSE_YIELD_IS_KEYWORD 0x0001 +#define PyPARSE_YIELD_IS_KEYWORD 0x0001 #endif -#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 +#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 #if 0 -#define PyPARSE_WITH_IS_KEYWORD 0x0003 +#define PyPARSE_WITH_IS_KEYWORD 0x0003 #define PyPARSE_PRINT_IS_FUNCTION 0x0004 #define PyPARSE_UNICODE_LITERALS 0x0008 #endif diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 23e224d8c3d4c4..64c518e97da7da 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -8,28 +8,28 @@ */ /* Values for PY_RELEASE_LEVEL */ -#define PY_RELEASE_LEVEL_ALPHA 0xA -#define PY_RELEASE_LEVEL_BETA 0xB -#define PY_RELEASE_LEVEL_GAMMA 0xC /* For release candidates */ -#define PY_RELEASE_LEVEL_FINAL 0xF /* Serial should be 0 here */ - /* Higher for patch releases */ +#define PY_RELEASE_LEVEL_ALPHA 0xA +#define PY_RELEASE_LEVEL_BETA 0xB +#define PY_RELEASE_LEVEL_GAMMA 0xC /* For release candidates */ +#define PY_RELEASE_LEVEL_FINAL 0xF /* Serial should be 0 here */ + /* Higher for patch releases */ /* Version parsed out into numeric values */ /*--start constants--*/ -#define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 2 +#define PY_MAJOR_VERSION 3 +#define PY_MINOR_VERSION 7 +#define PY_MICRO_VERSION 0 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.7.0a2+" +#define PY_VERSION "3.7.0a2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */ #define PY_VERSION_HEX ((PY_MAJOR_VERSION << 24) | \ - (PY_MINOR_VERSION << 16) | \ - (PY_MICRO_VERSION << 8) | \ - (PY_RELEASE_LEVEL << 4) | \ - (PY_RELEASE_SERIAL << 0)) + (PY_MINOR_VERSION << 16) | \ + (PY_MICRO_VERSION << 8) | \ + (PY_RELEASE_LEVEL << 4) | \ + (PY_RELEASE_SERIAL << 0)) diff --git a/Include/pgenheaders.h b/Include/pgenheaders.h index 4843de6c023e48..dbc5e0a5f13940 100644 --- a/Include/pgenheaders.h +++ b/Include/pgenheaders.h @@ -10,9 +10,9 @@ extern "C" { #include "Python.h" PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); #define addarc _Py_addarc #define addbit _Py_addbit diff --git a/Include/pydebug.h b/Include/pydebug.h index 6e23a896c3d1b1..bd4aafe3b49f83 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -15,7 +15,6 @@ PyAPI_DATA(int) Py_InspectFlag; PyAPI_DATA(int) Py_OptimizeFlag; PyAPI_DATA(int) Py_NoSiteFlag; PyAPI_DATA(int) Py_BytesWarningFlag; -PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; PyAPI_DATA(int) Py_DontWriteBytecodeFlag; @@ -25,6 +24,7 @@ PyAPI_DATA(int) Py_HashRandomizationFlag; PyAPI_DATA(int) Py_IsolatedFlag; #ifdef MS_WINDOWS +PyAPI_DATA(int) Py_LegacyWindowsFSEncodingFlag; PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; #endif diff --git a/Include/pyfpe.h b/Include/pyfpe.h index f9a15e622bab6b..d86cb7486ce33e 100644 --- a/Include/pyfpe.h +++ b/Include/pyfpe.h @@ -135,9 +135,9 @@ extern double PyFPE_dummy(void *); #define PyFPE_START_PROTECT(err_string, leave_stmt) \ if (!PyFPE_counter++ && setjmp(PyFPE_jbuf)) { \ - PyErr_SetString(PyExc_FloatingPointError, err_string); \ - PyFPE_counter = 0; \ - leave_stmt; \ + PyErr_SetString(PyExc_FloatingPointError, err_string); \ + PyFPE_counter = 0; \ + leave_stmt; \ } /* diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 8bbce7fd61dc7c..fa751692a6670a 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -7,6 +7,36 @@ extern "C" { #endif +#ifndef Py_LIMITED_API +typedef struct { + const char *prefix; + const char *msg; + int user_err; +} _PyInitError; + +/* Almost all errors causing Python initialization to fail */ +#ifdef _MSC_VER + /* Visual Studio 2015 doesn't implement C99 __func__ in C */ +# define _Py_INIT_GET_FUNC() __FUNCTION__ +#else +# define _Py_INIT_GET_FUNC() __func__ +#endif + +#define _Py_INIT_OK() \ + (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0} +#define _Py_INIT_ERR(MSG) \ + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0} +/* Error that can be fixed by the user like invalid input parameter. + Don't abort() the process on such error. */ +#define _Py_INIT_USER_ERR(MSG) \ + (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} +#define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") +#define _Py_INIT_FAILED(err) \ + (err.msg != NULL) + +#endif + + PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); @@ -21,17 +51,22 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, const char *errors); /* PEP 432 Multi-phase initialization API (Private while provisional!) */ -PyAPI_FUNC(void) _Py_InitializeCore(const _PyCoreConfig *); +PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *); PyAPI_FUNC(int) _Py_IsCoreInitialized(void); -PyAPI_FUNC(int) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *); -PyAPI_FUNC(int) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); + +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *); +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *); +PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *); + +PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); #endif /* Initialization and finalization */ PyAPI_FUNC(void) Py_Initialize(void); PyAPI_FUNC(void) Py_InitializeEx(int); #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int); +PyAPI_FUNC(_PyInitError) _Py_InitializeEx_Private(int, int); +PyAPI_FUNC(void) _Py_FatalInitError(_PyInitError err) _Py_NO_RETURN; #endif PyAPI_FUNC(void) Py_Finalize(void); PyAPI_FUNC(int) Py_FinalizeEx(void); @@ -50,7 +85,7 @@ PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void)); #endif PyAPI_FUNC(int) Py_AtExit(void (*func)(void)); -PyAPI_FUNC(void) Py_Exit(int); +PyAPI_FUNC(void) Py_Exit(int) _Py_NO_RETURN; /* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ #ifndef Py_LIMITED_API @@ -67,9 +102,13 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(_PyInitError) _PyPathConfig_Init( + const _PyMainInterpreterConfig *main_config); +#endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS -int _Py_CheckPython3(); +int _Py_CheckPython3(void); #endif /* In their own files */ @@ -86,15 +125,15 @@ PyAPI_FUNC(const char *) _Py_gitversion(void); /* Internal -- various one-time initializations */ #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void); -PyAPI_FUNC(PyObject *) _PySys_BeginInit(void); +PyAPI_FUNC(_PyInitError) _PySys_BeginInit(PyObject **sysmod); PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict); -PyAPI_FUNC(void) _PyImport_Init(void); +PyAPI_FUNC(_PyInitError) _PyImport_Init(void); PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod); -PyAPI_FUNC(void) _PyImportHooks_Init(void); +PyAPI_FUNC(_PyInitError) _PyImportHooks_Init(void); PyAPI_FUNC(int) _PyFrame_Init(void); PyAPI_FUNC(int) _PyFloat_Init(void); PyAPI_FUNC(int) PyByteArray_Init(void); -PyAPI_FUNC(void) _Py_HashRandomization_Init(_PyCoreConfig *core_config); +PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(_PyCoreConfig *core_config); #endif /* Various internal finalizers */ @@ -137,6 +176,7 @@ PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); #ifndef Py_LIMITED_API PyAPI_FUNC(void) _Py_CoerceLegacyLocale(void); PyAPI_FUNC(int) _Py_LegacyLocaleDetected(void); +PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); #endif #ifdef __cplusplus diff --git a/Include/pymem.h b/Include/pymem.h index 2670dcdfa92639..09d15020e0e4a6 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -21,6 +21,9 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr); allocators. */ PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); +/* Try to get the allocators name set by _PyMem_SetupAllocators(). */ +PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void); + #ifdef WITH_PYMALLOC PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); #endif @@ -105,8 +108,14 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifndef Py_LIMITED_API +/* strdup() using PyMem_RawMalloc() */ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); + +/* strdup() using PyMem_Malloc() */ PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); + +/* wcsdup() using PyMem_RawMalloc() */ +PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); #endif /* Macros. */ @@ -223,6 +232,15 @@ PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain, PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); #endif +#ifdef Py_BUILD_CORE +/* Set the memory allocator of the specified domain to the default. + Save the old allocator into *old_alloc if it's non-NULL. + Return on success, or return -1 if the domain is unknown. */ +PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( + PyMemAllocatorDomain domain, + PyMemAllocatorEx *old_alloc); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/pyport.h b/Include/pyport.h index 0e82543ac78355..f2e247a374e063 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -784,7 +784,9 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif /* Py_BUILD_CORE */ #ifdef __ANDROID__ -#include +/* The Android langinfo.h header is not used. */ +#undef HAVE_LANGINFO_H +#undef CODESET #endif /* Maximum value of the Windows DWORD type */ diff --git a/Include/pystate.h b/Include/pystate.h index 93815850f3bea1..cf45b059977a5b 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -25,14 +25,21 @@ typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int); typedef struct { - int ignore_environment; - int use_hash_seed; + int ignore_environment; /* -E */ + int use_hash_seed; /* PYTHONHASHSEED=x */ unsigned long hash_seed; int _disable_importlib; /* Needed by freeze_importlib */ - char *allocator; + const char *allocator; /* Memory allocator: _PyMem_SetupAllocators() */ + int dev_mode; /* -X dev */ + int faulthandler; /* -X faulthandler */ + int tracemalloc; /* -X tracemalloc=N */ + int import_time; /* -X importtime */ + int show_ref_count; /* -X showrefcount */ + int show_alloc_count; /* -X showalloccount */ } _PyCoreConfig; -#define _PyCoreConfig_INIT {0, -1, 0, 0, NULL} +#define _PyCoreConfig_INIT (_PyCoreConfig){.use_hash_seed = -1} +/* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ /* Placeholders while working on the new configuration API * @@ -42,9 +49,17 @@ typedef struct { */ typedef struct { int install_signal_handlers; + /* PYTHONPATH environment variable */ + wchar_t *module_search_path_env; + /* PYTHONHOME environment variable, see also Py_SetPythonHome(). */ + wchar_t *home; + /* Program name, see also Py_GetProgramName() */ + wchar_t *program_name; } _PyMainInterpreterConfig; -#define _PyMainInterpreterConfig_INIT {-1} +#define _PyMainInterpreterConfig_INIT \ + (_PyMainInterpreterConfig){.install_signal_handlers = -1} +/* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */ typedef struct _is { diff --git a/Include/sliceobject.h b/Include/sliceobject.h index b24f2926f13d14..c238b099ea8151 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -21,7 +21,7 @@ let these be any arbitrary python type. Py_None stands for omitted values. #ifndef Py_LIMITED_API typedef struct { PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ + PyObject *start, *stop, *step; /* not NULL */ } PySliceObject; #endif diff --git a/Include/sysmodule.h b/Include/sysmodule.h index c5547ff6742e06..719ecfcf61f2e7 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -37,6 +37,11 @@ PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *); #endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(int) _PySys_AddXOptionWithError(const wchar_t *s); +PyAPI_FUNC(int) _PySys_AddWarnOptionWithError(PyObject *option); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index d2a8ec281f1a64..61e713be072cd1 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1723,6 +1723,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage( #endif /* MS_WINDOWS */ +#ifndef Py_LIMITED_API /* --- Decimal Encoder ---------------------------------------------------- */ /* Takes a Unicode string holding a decimal value and writes it into @@ -1747,14 +1748,12 @@ PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage( */ -#ifndef Py_LIMITED_API PyAPI_FUNC(int) PyUnicode_EncodeDecimal( Py_UNICODE *s, /* Unicode buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ char *output, /* Output buffer; must have size >= length */ const char *errors /* error handling */ ) /* Py_DEPRECATED(3.3) */; -#endif /* Transforms code points that have decimal digit property to the corresponding ASCII digit code points. @@ -1762,19 +1761,18 @@ PyAPI_FUNC(int) PyUnicode_EncodeDecimal( Returns a new Unicode string on success, NULL on failure. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII( Py_UNICODE *s, /* Unicode buffer */ Py_ssize_t length /* Number of Py_UNICODE chars to transform */ ) /* Py_DEPRECATED(3.3) */; -#endif -/* Similar to PyUnicode_TransformDecimalToASCII(), but takes a PyObject - as argument instead of a raw buffer and length. This function additionally - transforms spaces to ASCII because this is what the callers in longobject, - floatobject, and complexobject did anyways. */ +/* Coverts a Unicode object holding a decimal value to an ASCII string + for using in int, float and complex parsers. + Transforms code points that have decimal digit property to the + corresponding ASCII digit code points. Transforms spaces to ASCII. + Transforms code points starting from the first non-ASCII code point that + is neither a decimal digit nor a space to the end into '?'. */ -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII( PyObject *unicode /* Unicode object */ ); diff --git a/Include/warnings.h b/Include/warnings.h index a3f83ff6967e40..25f715e3a8bb05 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -7,6 +7,9 @@ extern "C" { #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); #endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject*) _PyWarnings_InitWithConfig(const _PyCoreConfig *config); +#endif PyAPI_FUNC(int) PyErr_WarnEx( PyObject *category, diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 011466b3e0dc8f..1ee1b2516d459c 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -2,21 +2,6 @@ import sys -# The selectors module is in the stdlib in Python 3.4 but not in 3.3. -# Do this first, so the other submodules can use "from . import selectors". -# Prefer asyncio/selectors.py over the stdlib one, as ours may be newer. -try: - from . import selectors -except ImportError: - import selectors # Will also be exported. - -if sys.platform == 'win32': - # Similar thing for _overlapped. - try: - from . import _overlapped - except ImportError: - import _overlapped # Will also be exported. - # This relies on each of the submodules having an __all__ variable. from .base_events import * from .coroutines import * diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index b94856c9393618..ffdb50f4beea3e 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -244,8 +244,7 @@ def __init__(self): self._thread_id = None self._clock_resolution = time.get_clock_info('monotonic').resolution self._exception_handler = None - self.set_debug((not sys.flags.ignore_environment - and bool(os.environ.get('PYTHONASYNCIODEBUG')))) + self.set_debug(coroutines._is_debug_mode()) # In debug mode, if the execution of a callback or a step of a task # exceed this duration in seconds, the slow callback/task is logged. self.slow_callback_duration = 0.1 @@ -861,7 +860,7 @@ def create_datagram_endpoint(self, protocol_factory, addr_pairs_info = (((family, proto), (None, None)),) elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: for addr in (local_addr, remote_addr): - if addr is not None and not isistance(addr, str): + if addr is not None and not isinstance(addr, str): raise TypeError('string is expected') addr_pairs_info = (((family, proto), (local_addr, remote_addr)), ) diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py deleted file mode 100644 index 520ec6870c8882..00000000000000 --- a/Lib/asyncio/compat.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compatibility helpers for the different Python versions.""" - -import sys - -PY35 = sys.version_info >= (3, 5) -PY352 = sys.version_info >= (3, 5, 2) diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index 520a309f846049..7d2ca05d2a0856 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -9,72 +9,30 @@ import traceback import types -from . import compat +from collections.abc import Awaitable, Coroutine + from . import constants from . import events from . import base_futures from .log import logger -# Opcode of "yield from" instruction -_YIELD_FROM = opcode.opmap['YIELD_FROM'] - -# If you set _DEBUG to true, @coroutine will wrap the resulting -# generator objects in a CoroWrapper instance (defined below). That -# instance will log a message when the generator is never iterated -# over, which may happen when you forget to use "yield from" with a -# coroutine call. Note that the value of the _DEBUG flag is taken -# when the decorator is used, so to be of any use it must be set -# before you define your coroutines. A downside of using this feature -# is that tracebacks show entries for the CoroWrapper.__next__ method -# when _DEBUG is true. -_DEBUG = (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG'))) - - -try: - _types_coroutine = types.coroutine - _types_CoroutineType = types.CoroutineType -except AttributeError: - # Python 3.4 - _types_coroutine = None - _types_CoroutineType = None - -try: - _inspect_iscoroutinefunction = inspect.iscoroutinefunction -except AttributeError: - # Python 3.4 - _inspect_iscoroutinefunction = lambda func: False - -try: - from collections.abc import Coroutine as _CoroutineABC, \ - Awaitable as _AwaitableABC -except ImportError: - _CoroutineABC = _AwaitableABC = None - - -# Check for CPython issue #21209 -def has_yield_from_bug(): - class MyGen: - def __init__(self): - self.send_args = None - def __iter__(self): - return self - def __next__(self): - return 42 - def send(self, *what): - self.send_args = what - return None - def yield_from_gen(gen): - yield from gen - value = (1, 2, 3) - gen = MyGen() - coro = yield_from_gen(gen) - next(coro) - coro.send(value) - return gen.send_args != (value,) -_YIELD_FROM_BUG = has_yield_from_bug() -del has_yield_from_bug +def _is_debug_mode(): + # If you set _DEBUG to true, @coroutine will wrap the resulting + # generator objects in a CoroWrapper instance (defined below). That + # instance will log a message when the generator is never iterated + # over, which may happen when you forget to use "yield from" with a + # coroutine call. Note that the value of the _DEBUG flag is taken + # when the decorator is used, so to be of any use it must be set + # before you define your coroutines. A downside of using this feature + # is that tracebacks show entries for the CoroWrapper.__next__ method + # when _DEBUG is true. + return (sys.flags.dev_mode + or (not sys.flags.ignore_environment + and bool(os.environ.get('PYTHONASYNCIODEBUG')))) + + +_DEBUG = _is_debug_mode() def debug_wrapper(gen): @@ -109,21 +67,8 @@ def __iter__(self): def __next__(self): return self.gen.send(None) - if _YIELD_FROM_BUG: - # For for CPython issue #21209: using "yield from" and a custom - # generator, generator.send(tuple) unpacks the tuple instead of passing - # the tuple unchanged. Check if the caller is a generator using "yield - # from" to decide if the parameter should be unpacked or not. - def send(self, *value): - frame = sys._getframe() - caller = frame.f_back - assert caller.f_lasti >= 0 - if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM: - value = value[0] - return self.gen.send(value) - else: - def send(self, value): - return self.gen.send(value) + def send(self, value): + return self.gen.send(value) def throw(self, type, value=None, traceback=None): return self.gen.throw(type, value, traceback) @@ -143,35 +88,33 @@ def gi_running(self): def gi_code(self): return self.gen.gi_code - if compat.PY35: - - def __await__(self): - cr_await = getattr(self.gen, 'cr_await', None) - if cr_await is not None: - raise RuntimeError( - "Cannot await on coroutine {!r} while it's " - "awaiting for {!r}".format(self.gen, cr_await)) - return self + def __await__(self): + cr_await = getattr(self.gen, 'cr_await', None) + if cr_await is not None: + raise RuntimeError( + "Cannot await on coroutine {!r} while it's " + "awaiting for {!r}".format(self.gen, cr_await)) + return self - @property - def gi_yieldfrom(self): - return self.gen.gi_yieldfrom + @property + def gi_yieldfrom(self): + return self.gen.gi_yieldfrom - @property - def cr_await(self): - return self.gen.cr_await + @property + def cr_await(self): + return self.gen.cr_await - @property - def cr_running(self): - return self.gen.cr_running + @property + def cr_running(self): + return self.gen.cr_running - @property - def cr_code(self): - return self.gen.cr_code + @property + def cr_code(self): + return self.gen.cr_code - @property - def cr_frame(self): - return self.gen.cr_frame + @property + def cr_frame(self): + return self.gen.cr_frame def __del__(self): # Be careful accessing self.gen.frame -- self.gen might not exist. @@ -197,7 +140,7 @@ def coroutine(func): If the coroutine is not yielded from before it is destroyed, an error message is logged. """ - if _inspect_iscoroutinefunction(func): + if inspect.iscoroutinefunction(func): # In Python 3.5 that's all we need to do for coroutines # defined with "async def". # Wrapping in CoroWrapper will happen via @@ -213,7 +156,7 @@ def coro(*args, **kw): if (base_futures.isfuture(res) or inspect.isgenerator(res) or isinstance(res, CoroWrapper)): res = yield from res - elif _AwaitableABC is not None: + else: # If 'func' returns an Awaitable (new in 3.5) we # want to run it. try: @@ -221,15 +164,12 @@ def coro(*args, **kw): except AttributeError: pass else: - if isinstance(res, _AwaitableABC): + if isinstance(res, Awaitable): res = yield from await_meth() return res if not _DEBUG: - if _types_coroutine is None: - wrapper = coro - else: - wrapper = _types_coroutine(coro) + wrapper = types.coroutine(coro) else: @functools.wraps(func) def wrapper(*args, **kwds): @@ -254,17 +194,14 @@ def wrapper(*args, **kwds): def iscoroutinefunction(func): """Return True if func is a decorated coroutine function.""" - return (getattr(func, '_is_coroutine', None) is _is_coroutine or - _inspect_iscoroutinefunction(func)) + return (inspect.iscoroutinefunction(func) or + getattr(func, '_is_coroutine', None) is _is_coroutine) -_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper) -if _CoroutineABC is not None: - _COROUTINE_TYPES += (_CoroutineABC,) -if _types_CoroutineType is not None: - # Prioritize native coroutine check to speed-up - # asyncio.iscoroutine. - _COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES +# Prioritize native coroutine check to speed-up +# asyncio.iscoroutine. +_COROUTINE_TYPES = (types.CoroutineType, types.GeneratorType, + Coroutine, CoroWrapper) def iscoroutine(obj): diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index f2f2e286342b9a..e59d3d2760e840 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -362,12 +362,12 @@ def create_server(self, protocol_factory, host=None, port=None, *, """ raise NotImplementedError - def create_unix_connection(self, protocol_factory, path, *, + def create_unix_connection(self, protocol_factory, path=None, *, ssl=None, sock=None, server_hostname=None): raise NotImplementedError - def create_unix_server(self, protocol_factory, path, *, + def create_unix_server(self, protocol_factory, path=None, *, sock=None, backlog=100, ssl=None): """A coroutine which creates a UNIX Domain Socket server. diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 472f2a8c74e860..7b6204a626f864 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -9,7 +9,6 @@ import traceback from . import base_futures -from . import compat from . import events @@ -63,8 +62,7 @@ class Future: # `yield Future()` (incorrect). _asyncio_future_blocking = False - _log_traceback = False # Used for Python 3.4 and later - _tb_logger = None # Used for Python 3.3 only + _log_traceback = False def __init__(self, *, loop=None): """Initialize the future. @@ -156,9 +154,6 @@ def result(self): if self._state != _FINISHED: raise InvalidStateError('Result is not ready.') self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None if self._exception is not None: raise self._exception return self._result @@ -176,9 +171,6 @@ def exception(self): if self._state != _FINISHED: raise InvalidStateError('Exception is not set.') self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None return self._exception def add_done_callback(self, fn): @@ -245,8 +237,7 @@ def __iter__(self): assert self.done(), "yield from wasn't used with future" return self.result() # May raise too. - if compat.PY35: - __await__ = __iter__ # make compatible with 'await' expression + __await__ = __iter__ # make compatible with 'await' expression # Needed for testing purposes. diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 92661830a06228..750c43591791e8 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -4,7 +4,6 @@ import collections -from . import compat from . import events from . import futures from .coroutines import coroutine @@ -67,23 +66,21 @@ def __iter__(self): yield from self.acquire() return _ContextManager(self) - if compat.PY35: - - def __await__(self): - # To make "with await lock" work. - yield from self.acquire() - return _ContextManager(self) + def __await__(self): + # To make "with await lock" work. + yield from self.acquire() + return _ContextManager(self) - @coroutine - def __aenter__(self): - yield from self.acquire() - # We have no use for the "as ..." clause in the with - # statement for locks. - return None + @coroutine + def __aenter__(self): + yield from self.acquire() + # We have no use for the "as ..." clause in the with + # statement for locks. + return None - @coroutine - def __aexit__(self, exc_type, exc, tb): - self.release() + @coroutine + def __aexit__(self, exc_type, exc, tb): + self.release() class Lock(_ContextManagerMixin): diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 5e7a39727cce2e..d7aa5ff3017568 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -392,11 +392,6 @@ def _make_socket_transport(self, sock, protocol, waiter=None, def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, *, server_side=False, server_hostname=None, extra=None, server=None): - if not sslproto._is_sslproto_available(): - raise NotImplementedError("Proactor event loop requires Python 3.5" - " or newer (ssl.MemoryBIO) to support " - "SSL") - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, server_side, server_hostname) _ProactorSocketTransport(self, rawsock, ssl_protocol, @@ -451,9 +446,6 @@ def sock_connect(self, sock, address): def sock_accept(self, sock): return self._proactor.accept(sock) - def _socketpair(self): - raise NotImplementedError - def _close_self_pipe(self): if self._self_reading_future is not None: self._self_reading_future.cancel() @@ -466,7 +458,7 @@ def _close_self_pipe(self): def _make_self_pipe(self): # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() + self._ssock, self._csock = socket.socketpair() self._ssock.setblocking(False) self._csock.setblocking(False) self._internal_fds += 1 diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 1c66d67b041304..4fc681dde97532 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -5,7 +5,6 @@ import collections import heapq -from . import compat from . import events from . import locks from .coroutines import coroutine @@ -251,9 +250,3 @@ def _put(self, item): def _get(self): return self._queue.pop() - - -if not compat.PY35: - JoinableQueue = Queue - """Deprecated alias for Queue.""" - __all__.append('JoinableQueue') diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 7143ca2660afaa..3639466f6c2bb9 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -9,6 +9,7 @@ import collections import errno import functools +import selectors import socket import warnings import weakref @@ -21,7 +22,6 @@ from . import constants from . import events from . import futures -from . import selectors from . import transports from . import sslproto from .coroutines import coroutine @@ -74,28 +74,12 @@ def _make_socket_transport(self, sock, protocol, waiter=None, *, def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, *, server_side=False, server_hostname=None, extra=None, server=None): - if not sslproto._is_sslproto_available(): - return self._make_legacy_ssl_transport( - rawsock, protocol, sslcontext, waiter, - server_side=server_side, server_hostname=server_hostname, - extra=extra, server=server) - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, server_side, server_hostname) _SelectorSocketTransport(self, rawsock, ssl_protocol, extra=extra, server=server) return ssl_protocol._app_transport - def _make_legacy_ssl_transport(self, rawsock, protocol, sslcontext, - waiter, *, - server_side=False, server_hostname=None, - extra=None, server=None): - # Use the legacy API: SSL_write, SSL_read, etc. The legacy API is used - # on Python 3.4 and older, when ssl.MemoryBIO is not available. - return _SelectorSslTransport( - self, rawsock, protocol, sslcontext, waiter, - server_side, server_hostname, extra, server) - def _make_datagram_transport(self, sock, protocol, address=None, waiter=None, extra=None): return _SelectorDatagramTransport(self, sock, protocol, @@ -112,9 +96,6 @@ def close(self): self._selector.close() self._selector = None - def _socketpair(self): - raise NotImplementedError - def _close_self_pipe(self): self._remove_reader(self._ssock.fileno()) self._ssock.close() @@ -125,7 +106,7 @@ def _close_self_pipe(self): def _make_self_pipe(self): # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() + self._ssock, self._csock = socket.socketpair() self._ssock.setblocking(False) self._csock.setblocking(False) self._internal_fds += 1 @@ -246,8 +227,16 @@ def _accept_connection2(self, protocol_factory, conn, extra, self.call_exception_handler(context) def _ensure_fd_no_transport(self, fd): + fileno = fd + if not isinstance(fileno, int): + try: + fileno = int(fileno.fileno()) + except (AttributeError, TypeError, ValueError): + # This code matches selectors._fileobj_to_fd function. + raise ValueError("Invalid file object: " + "{!r}".format(fd)) from None try: - transport = self._transports[fd] + transport = self._transports[fileno] except KeyError: pass else: @@ -362,25 +351,25 @@ def sock_recv(self, sock, n): if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() - self._sock_recv(fut, False, sock, n) + self._sock_recv(fut, None, sock, n) return fut - def _sock_recv(self, fut, registered, sock, n): + def _sock_recv(self, fut, registered_fd, sock, n): # _sock_recv() can add itself as an I/O callback if the operation can't # be done immediately. Don't use it directly, call sock_recv(). - fd = sock.fileno() - if registered: + if registered_fd is not None: # Remove the callback early. It should be rare that the # selector says the fd is ready but the call still returns # EAGAIN, and I am willing to take a hit in that case in # order to simplify the common case. - self.remove_reader(fd) + self.remove_reader(registered_fd) if fut.cancelled(): return try: data = sock.recv(n) except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_recv, fut, True, sock, n) + fd = sock.fileno() + self.add_reader(fd, self._sock_recv, fut, fd, sock, n) except Exception as exc: fut.set_exception(exc) else: @@ -397,25 +386,25 @@ def sock_recv_into(self, sock, buf): if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() - self._sock_recv_into(fut, False, sock, buf) + self._sock_recv_into(fut, None, sock, buf) return fut - def _sock_recv_into(self, fut, registered, sock, buf): + def _sock_recv_into(self, fut, registered_fd, sock, buf): # _sock_recv_into() can add itself as an I/O callback if the operation # can't be done immediately. Don't use it directly, call sock_recv_into(). - fd = sock.fileno() - if registered: + if registered_fd is not None: # Remove the callback early. It should be rare that the # selector says the fd is ready but the call still returns # EAGAIN, and I am willing to take a hit in that case in # order to simplify the common case. - self.remove_reader(fd) + self.remove_reader(registered_fd) if fut.cancelled(): return try: nbytes = sock.recv_into(buf) except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_recv_into, fut, True, sock, buf) + fd = sock.fileno() + self.add_reader(fd, self._sock_recv_into, fut, fd, sock, buf) except Exception as exc: fut.set_exception(exc) else: @@ -436,16 +425,14 @@ def sock_sendall(self, sock, data): raise ValueError("the socket must be non-blocking") fut = self.create_future() if data: - self._sock_sendall(fut, False, sock, data) + self._sock_sendall(fut, None, sock, data) else: fut.set_result(None) return fut - def _sock_sendall(self, fut, registered, sock, data): - fd = sock.fileno() - - if registered: - self.remove_writer(fd) + def _sock_sendall(self, fut, registered_fd, sock, data): + if registered_fd is not None: + self.remove_writer(registered_fd) if fut.cancelled(): return @@ -462,7 +449,8 @@ def _sock_sendall(self, fut, registered, sock, data): else: if n: data = data[n:] - self.add_writer(fd, self._sock_sendall, fut, True, sock, data) + fd = sock.fileno() + self.add_writer(fd, self._sock_sendall, fut, fd, sock, data) @coroutine def sock_connect(self, sock, address): @@ -841,238 +829,6 @@ def can_write_eof(self): return True -class _SelectorSslTransport(_SelectorTransport): - - _buffer_factory = bytearray - - def __init__(self, loop, rawsock, protocol, sslcontext, waiter=None, - server_side=False, server_hostname=None, - extra=None, server=None): - if ssl is None: - raise RuntimeError('stdlib ssl module not available') - - if not sslcontext: - sslcontext = sslproto._create_transport_context(server_side, server_hostname) - - wrap_kwargs = { - 'server_side': server_side, - 'do_handshake_on_connect': False, - } - if server_hostname and not server_side: - wrap_kwargs['server_hostname'] = server_hostname - sslsock = sslcontext.wrap_socket(rawsock, **wrap_kwargs) - - super().__init__(loop, sslsock, protocol, extra, server) - # the protocol connection is only made after the SSL handshake - self._protocol_connected = False - - self._server_hostname = server_hostname - self._waiter = waiter - self._sslcontext = sslcontext - self._paused = False - - # SSL-specific extra info. (peercert is set later) - self._extra.update(sslcontext=sslcontext) - - if self._loop.get_debug(): - logger.debug("%r starts SSL handshake", self) - start_time = self._loop.time() - else: - start_time = None - self._on_handshake(start_time) - - def _wakeup_waiter(self, exc=None): - if self._waiter is None: - return - if not self._waiter.cancelled(): - if exc is not None: - self._waiter.set_exception(exc) - else: - self._waiter.set_result(None) - self._waiter = None - - def _on_handshake(self, start_time): - try: - self._sock.do_handshake() - except ssl.SSLWantReadError: - self._loop._add_reader(self._sock_fd, - self._on_handshake, start_time) - return - except ssl.SSLWantWriteError: - self._loop._add_writer(self._sock_fd, - self._on_handshake, start_time) - return - except BaseException as exc: - if self._loop.get_debug(): - logger.warning("%r: SSL handshake failed", - self, exc_info=True) - self._loop._remove_reader(self._sock_fd) - self._loop._remove_writer(self._sock_fd) - self._sock.close() - self._wakeup_waiter(exc) - if isinstance(exc, Exception): - return - else: - raise - - self._loop._remove_reader(self._sock_fd) - self._loop._remove_writer(self._sock_fd) - - peercert = self._sock.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname and - self._sslcontext.verify_mode != ssl.CERT_NONE): - try: - ssl.match_hostname(peercert, self._server_hostname) - except Exception as exc: - if self._loop.get_debug(): - logger.warning("%r: SSL handshake failed " - "on matching the hostname", - self, exc_info=True) - self._sock.close() - self._wakeup_waiter(exc) - return - - # Add extra info that becomes available after handshake. - self._extra.update(peercert=peercert, - cipher=self._sock.cipher(), - compression=self._sock.compression(), - ssl_object=self._sock, - ) - - self._read_wants_write = False - self._write_wants_read = False - self._loop._add_reader(self._sock_fd, self._read_ready) - self._protocol_connected = True - self._loop.call_soon(self._protocol.connection_made, self) - # only wake up the waiter when connection_made() has been called - self._loop.call_soon(self._wakeup_waiter) - - if self._loop.get_debug(): - dt = self._loop.time() - start_time - logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3) - - def pause_reading(self): - # XXX This is a bit icky, given the comment at the top of - # _read_ready(). Is it possible to evoke a deadlock? I don't - # know, although it doesn't look like it; write() will still - # accept more data for the buffer and eventually the app will - # call resume_reading() again, and things will flow again. - - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') - self._paused = True - self._loop._remove_reader(self._sock_fd) - if self._loop.get_debug(): - logger.debug("%r pauses reading", self) - - def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: - return - self._loop._add_reader(self._sock_fd, self._read_ready) - if self._loop.get_debug(): - logger.debug("%r resumes reading", self) - - def _read_ready(self): - if self._conn_lost: - return - if self._write_wants_read: - self._write_wants_read = False - self._write_ready() - - if self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) - - try: - data = self._sock.recv(self.max_size) - except (BlockingIOError, InterruptedError, ssl.SSLWantReadError): - pass - except ssl.SSLWantWriteError: - self._read_wants_write = True - self._loop._remove_reader(self._sock_fd) - self._loop._add_writer(self._sock_fd, self._write_ready) - except Exception as exc: - self._fatal_error(exc, 'Fatal read error on SSL transport') - else: - if data: - self._protocol.data_received(data) - else: - try: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if keep_open: - logger.warning('returning true from eof_received() ' - 'has no effect when using ssl') - finally: - self.close() - - def _write_ready(self): - if self._conn_lost: - return - if self._read_wants_write: - self._read_wants_write = False - self._read_ready() - - if not (self._paused or self._closing): - self._loop._add_reader(self._sock_fd, self._read_ready) - - if self._buffer: - try: - n = self._sock.send(self._buffer) - except (BlockingIOError, InterruptedError, ssl.SSLWantWriteError): - n = 0 - except ssl.SSLWantReadError: - n = 0 - self._loop._remove_writer(self._sock_fd) - self._write_wants_read = True - except Exception as exc: - self._loop._remove_writer(self._sock_fd) - self._buffer.clear() - self._fatal_error(exc, 'Fatal write error on SSL transport') - return - - if n: - del self._buffer[:n] - - self._maybe_resume_protocol() # May append to buffer. - - if not self._buffer: - self._loop._remove_writer(self._sock_fd) - if self._closing: - self._call_connection_lost(None) - - def write(self, data): - if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be a bytes-like object, ' - 'not %r' % type(data).__name__) - if not data: - return - - if self._conn_lost: - if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: - logger.warning('socket.send() raised exception.') - self._conn_lost += 1 - return - - if not self._buffer: - self._loop._add_writer(self._sock_fd, self._write_ready) - - # Add it to the buffer. - self._buffer.extend(data) - self._maybe_pause_protocol() - - def can_write_eof(self): - return False - - class _SelectorDatagramTransport(_SelectorTransport): _buffer_factory = collections.deque diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 53e6d2b667b12c..c231eb58ee590a 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -18,25 +18,13 @@ def _create_transport_context(server_side, server_hostname): # Client side may pass ssl=True to use a default # context; in that case the sslcontext passed is None. # The default is secure for client connections. - if hasattr(ssl, 'create_default_context'): - # Python 3.4+: use up-to-date strong settings. - sslcontext = ssl.create_default_context() - if not server_hostname: - sslcontext.check_hostname = False - else: - # Fallback for Python 3.3. - sslcontext = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - sslcontext.options |= ssl.OP_NO_SSLv2 - sslcontext.options |= ssl.OP_NO_SSLv3 - sslcontext.set_default_verify_paths() - sslcontext.verify_mode = ssl.CERT_REQUIRED + # Python 3.4+: use up-to-date strong settings. + sslcontext = ssl.create_default_context() + if not server_hostname: + sslcontext.check_hostname = False return sslcontext -def _is_sslproto_available(): - return hasattr(ssl, "MemoryBIO") - - # States of an _SSLPipe. _UNWRAPPED = "UNWRAPPED" _DO_HANDSHAKE = "DO_HANDSHAKE" diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 9fda8537686d0a..15c9513527f308 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -12,7 +12,6 @@ __all__.extend(['open_unix_connection', 'start_unix_server']) from . import coroutines -from . import compat from . import events from . import protocols from .coroutines import coroutine @@ -35,6 +34,9 @@ def __init__(self, partial, expected): self.partial = partial self.expected = expected + def __reduce__(self): + return type(self), (self.partial, self.expected) + class LimitOverrunError(Exception): """Reached the buffer limit while looking for a separator. @@ -46,6 +48,9 @@ def __init__(self, message, consumed): super().__init__(message) self.consumed = consumed + def __reduce__(self): + return type(self), (self.args[0], self.consumed) + @coroutine def open_connection(host=None, port=None, *, @@ -149,7 +154,7 @@ class FlowControlMixin(protocols.Protocol): """Reusable flow control logic for StreamWriter.drain(). This implements the protocol methods pause_writing(), - resume_reading() and connection_lost(). If the subclass overrides + resume_writing() and connection_lost(). If the subclass overrides these it must call the super methods. StreamWriter.drain() must wait for _drain_helper() coroutine. diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 52fef181cecc66..5d744c3d30eea8 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -13,7 +13,6 @@ import weakref from . import base_tasks -from . import compat from . import coroutines from . import events from . import futures @@ -525,7 +524,7 @@ def ensure_future(coro_or_future, *, loop=None): if task._source_traceback: del task._source_traceback[-1] return task - elif compat.PY35 and inspect.isawaitable(coro_or_future): + elif inspect.isawaitable(coro_or_future): return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) else: raise TypeError('An asyncio.Future, a coroutine or an awaitable is ' diff --git a/Lib/asyncio/test_utils.py b/Lib/asyncio/test_utils.py index 65805947fda6a4..32d3b0bf630849 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/asyncio/test_utils.py @@ -6,6 +6,7 @@ import logging import os import re +import selectors import socket import socketserver import sys @@ -28,19 +29,12 @@ from . import base_events from . import events from . import futures -from . import selectors from . import tasks from .coroutines import coroutine from .log import logger from test import support -if sys.platform == 'win32': # pragma: no cover - from .windows_utils import socketpair -else: - from socket import socketpair # pragma: no cover - - def dummy_ssl_context(): if ssl is None: return None @@ -361,6 +355,13 @@ def assert_writer(self, fd, callback, *args): handle._args, args) def _ensure_fd_no_transport(self, fd): + if not isinstance(fd, int): + try: + fd = int(fd.fileno()) + except (AttributeError, TypeError, ValueError): + # This code matches selectors._fileobj_to_fd function. + raise ValueError("Invalid file object: " + "{!r}".format(fd)) from None try: transport = self._transports[fd] except KeyError: @@ -501,8 +502,3 @@ def mock_nonblocking_socket(proto=socket.IPPROTO_TCP, type=socket.SOCK_STREAM, sock.family = family sock.gettimeout.return_value = 0.0 return sock - - -def force_legacy_ssl_support(): - return mock.patch('asyncio.sslproto._is_sslproto_available', - return_value=False) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index bf682a1a98a39f..ab818da1dfabfa 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -2,6 +2,7 @@ import errno import os +import selectors import signal import socket import stat @@ -18,7 +19,6 @@ from . import events from . import futures from . import selector_events -from . import selectors from . import transports from .coroutines import coroutine from .log import logger @@ -38,13 +38,6 @@ def _sighandler_noop(signum, frame): pass -try: - _fspath = os.fspath -except AttributeError: - # Python 3.5 or earlier - _fspath = lambda path: path - - class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. @@ -55,9 +48,6 @@ def __init__(self, selector=None): super().__init__(selector) self._signal_handlers = {} - def _socketpair(self): - return socket.socketpair() - def close(self): super().close() for sig in list(self._signal_handlers): @@ -77,7 +67,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) - or coroutines.iscoroutinefunction(callback)): + or coroutines.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -212,7 +202,7 @@ def _child_watcher_callback(self, pid, returncode, transp): self.call_soon_threadsafe(transp._process_exited, returncode) @coroutine - def create_unix_connection(self, protocol_factory, path, *, + def create_unix_connection(self, protocol_factory, path=None, *, ssl=None, sock=None, server_hostname=None): assert server_hostname is None or isinstance(server_hostname, str) @@ -229,6 +219,7 @@ def create_unix_connection(self, protocol_factory, path, *, raise ValueError( 'path and sock can not be specified at the same time') + path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) try: sock.setblocking(False) @@ -262,7 +253,7 @@ def create_unix_server(self, protocol_factory, path=None, *, raise ValueError( 'path and sock can not be specified at the same time') - path = _fspath(path) + path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # Check for abstract socket. `str` and `bytes` paths are supported. @@ -274,7 +265,8 @@ def create_unix_server(self, protocol_factory, path=None, *, pass except OSError as err: # Directory may have permissions only to create socket. - logger.error('Unable to check or remove stale UNIX socket %r: %r', path, err) + logger.error('Unable to check or remove stale UNIX socket ' + '%r: %r', path, err) try: sock.bind(path) @@ -308,18 +300,6 @@ def create_unix_server(self, protocol_factory, path=None, *, return server -if hasattr(os, 'set_blocking'): - def _set_nonblocking(fd): - os.set_blocking(fd, False) -else: - import fcntl - - def _set_nonblocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(fd, fcntl.F_SETFL, flags) - - class _UnixReadPipeTransport(transports.ReadTransport): max_size = 256 * 1024 # max bytes we read in one event loop iteration @@ -342,7 +322,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): self._protocol = None raise ValueError("Pipe transport is for pipes/sockets only.") - _set_nonblocking(self._fileno) + os.set_blocking(self._fileno, False) self._loop.call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called @@ -471,7 +451,7 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): raise ValueError("Pipe transport is only for " "pipes, sockets and character devices") - _set_nonblocking(self._fileno) + os.set_blocking(self._fileno, False) self._loop.call_soon(self._protocol.connection_made, self) # On AIX, the reader trick (to be notified when the read end of the @@ -650,22 +630,6 @@ def _call_connection_lost(self, exc): self._loop = None -if hasattr(os, 'set_inheritable'): - # Python 3.4 and newer - _set_inheritable = os.set_inheritable -else: - import fcntl - - def _set_inheritable(fd, inheritable): - cloexec_flag = getattr(fcntl, 'FD_CLOEXEC', 1) - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if not inheritable: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): @@ -676,13 +640,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): # socket (which we use in order to detect closing of the # other end). Notably this is needed on AIX, and works # just fine on other platforms. - stdin, stdin_w = self._loop._socketpair() - - # Mark the write end of the stdin pipe as non-inheritable, - # needed by close_fds=False on Python 3.3 and older - # (Python 3.4 implements the PEP 446, socketpair returns - # non-inheritable sockets) - _set_inheritable(stdin_w.fileno(), False) + stdin, stdin_w = socket.socketpair() self._proc = subprocess.Popen( args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, universal_newlines=False, bufsize=bufsize, **kwargs) @@ -1037,8 +995,8 @@ def set_event_loop(self, loop): super().set_event_loop(loop) - if self._watcher is not None and \ - isinstance(threading.current_thread(), threading._MainThread): + if (self._watcher is not None and + isinstance(threading.current_thread(), threading._MainThread)): self._watcher.attach_loop(loop) def get_child_watcher(self): diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 6045ba029e577a..de41e645163dcc 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -1,5 +1,6 @@ """Selector and proactor event loops for Windows.""" +import _overlapped import _winapi import errno import math @@ -14,7 +15,6 @@ from . import selector_events from . import tasks from . import windows_utils -from . import _overlapped from .coroutines import coroutine from .log import logger @@ -296,9 +296,6 @@ def close(self): class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Windows version of selector event loop.""" - def _socketpair(self): - return windows_utils.socketpair() - class ProactorEventLoop(proactor_events.BaseProactorEventLoop): """Windows version of proactor event loop using IOCP.""" @@ -308,9 +305,6 @@ def __init__(self, proactor=None): proactor = IocpProactor() super().__init__(proactor) - def _socketpair(self): - return windows_utils.socketpair() - @coroutine def create_pipe_connection(self, protocol_factory, address): f = self._proactor.connect_pipe(address) diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index d65ea1790f078d..3b410976f9ddc8 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -17,7 +17,7 @@ import warnings -__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle'] +__all__ = ['pipe', 'Popen', 'PIPE', 'PipeHandle'] # Constants/globals @@ -29,54 +29,6 @@ _mmap_counter = itertools.count() -if hasattr(socket, 'socketpair'): - # Since Python 3.5, socket.socketpair() is now also available on Windows - socketpair = socket.socketpair -else: - # Replacement for socket.socketpair() - def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): - """A socket pair usable as a self-pipe, for Windows. - - Origin: https://gist.github.com/4325783, by Geert Jansen. - Public domain. - """ - if family == socket.AF_INET: - host = '127.0.0.1' - elif family == socket.AF_INET6: - host = '::1' - else: - raise ValueError("Only AF_INET and AF_INET6 socket address " - "families are supported") - if type != socket.SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with setblocking(0) - # that prevents us from having to create a thread. - lsock = socket.socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen(1) - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket.socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) - - # Replacement for os.pipe() using handles instead of fds diff --git a/Lib/codecs.py b/Lib/codecs.py index 44618cbd2c4a36..a70ed20f2bc794 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -480,15 +480,17 @@ def read(self, size=-1, chars=-1, firstline=False): self.charbuffer = self._empty_charbuffer.join(self.linebuffer) self.linebuffer = None + if chars < 0: + # For compatibility with other read() methods that take a + # single argument + chars = size + # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? if chars >= 0: if len(self.charbuffer) >= chars: break - elif size >= 0: - if len(self.charbuffer) >= size: - break # we need more data if size < 0: newdata = self.stream.read() diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 962cedab490eb2..c1f8a84617fce4 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -5,7 +5,7 @@ from collections import deque from functools import wraps -__all__ = ["asynccontextmanager", "contextmanager", "closing", +__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "ContextDecorator", "ExitStack", "redirect_stdout", "redirect_stderr", "suppress"] @@ -469,3 +469,24 @@ def _fix_exception_context(new_exc, old_exc): exc_details[1].__context__ = fixed_ctx raise return received_exc and suppressed_exc + + +class nullcontext(AbstractContextManager): + """Context manager that does no additional processing. + + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass diff --git a/Lib/crypt.py b/Lib/crypt.py index 4d73202b468796..b0e47f430c3cbc 100644 --- a/Lib/crypt.py +++ b/Lib/crypt.py @@ -19,7 +19,7 @@ def __repr__(self): return ''.format(self.name) -def mksalt(method=None, *, log_rounds=12): +def mksalt(method=None, *, rounds=None): """Generate a salt for the specified method. If not specified, the strongest available method will be used. @@ -27,12 +27,32 @@ def mksalt(method=None, *, log_rounds=12): """ if method is None: method = methods[0] - if not method.ident: + if rounds is not None and not isinstance(rounds, int): + raise TypeError(f'{rounds.__class__.__name__} object cannot be ' + f'interpreted as an integer') + if not method.ident: # traditional s = '' - elif method.ident[0] == '2': - s = f'${method.ident}${log_rounds:02d}$' - else: + else: # modular s = f'${method.ident}$' + + if method.ident and method.ident[0] == '2': # Blowfish variants + if rounds is None: + log_rounds = 12 + else: + log_rounds = int.bit_length(rounds-1) + if rounds != 1 << log_rounds: + raise ValueError('rounds must be a power of 2') + if not 4 <= log_rounds <= 31: + raise ValueError('rounds out of the range 2**4 to 2**31') + s += f'{log_rounds:02d}$' + elif method.ident in ('5', '6'): # SHA-2 + if rounds is not None: + if not 1000 <= rounds <= 999_999_999: + raise ValueError('rounds out of the range 1000 to 999_999_999') + s += f'rounds={rounds}$' + elif rounds is not None: + raise ValueError(f"{method} doesn't support the rounds argument") + s += ''.join(_sr.choice(_saltchars) for char in range(method.salt_chars)) return s @@ -55,10 +75,10 @@ def crypt(word, salt=None): # available salting/crypto methods methods = [] -def _add_method(name, *args): +def _add_method(name, *args, rounds=None): method = _Method(name, *args) globals()['METHOD_' + name] = method - salt = mksalt(method, log_rounds=4) + salt = mksalt(method, rounds=rounds) result = crypt('', salt) if result and len(result) == method.total_size: methods.append(method) @@ -74,7 +94,7 @@ def _add_method(name, *args): # 'y' is the same as 'b', for compatibility # with openwall crypt_blowfish. for _v in 'b', 'y', 'a', '': - if _add_method('BLOWFISH', '2' + _v, 22, 59 + len(_v)): + if _add_method('BLOWFISH', '2' + _v, 22, 59 + len(_v), rounds=1<<4): break _add_method('MD5', '1', 8, 34) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py new file mode 100644 index 00000000000000..7a725dfb5208bb --- /dev/null +++ b/Lib/dataclasses.py @@ -0,0 +1,776 @@ +import sys +import types +from copy import deepcopy +import collections +import inspect + +__all__ = ['dataclass', + 'field', + 'FrozenInstanceError', + 'InitVar', + + # Helper functions. + 'fields', + 'asdict', + 'astuple', + 'make_dataclass', + 'replace', + ] + +# Raised when an attempt is made to modify a frozen class. +class FrozenInstanceError(AttributeError): pass + +# A sentinel object for default values to signal that a +# default-factory will be used. +# This is given a nice repr() which will appear in the function +# signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return '' +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() + +# A sentinel object to detect if a parameter is supplied or not. +class _MISSING_FACTORY: + def __repr__(self): + return '' +_MISSING = _MISSING_FACTORY() + +# Since most per-field metadata will be unused, create an empty +# read-only proxy that can be shared among all fields. +_EMPTY_METADATA = types.MappingProxyType({}) + +# Markers for the various kinds of fields and pseudo-fields. +_FIELD = object() # An actual field. +_FIELD_CLASSVAR = object() # Not a field, but a ClassVar. +_FIELD_INITVAR = object() # Not a field, but an InitVar. + +# The name of an attribute on the class where we store the Field +# objects. Also used to check if a class is a Data Class. +_MARKER = '__dataclass_fields__' + +# The name of the function, that if it exists, is called at the end of +# __init__. +_POST_INIT_NAME = '__post_init__' + + +class _InitVarMeta(type): + def __getitem__(self, params): + return self + +class InitVar(metaclass=_InitVarMeta): + pass + + +# Instances of Field are only ever created from within this module, +# and only from the field() function, although Field instances are +# exposed externally as (conceptually) read-only objects. +# name and type are filled in after the fact, not in __init__. They're +# not known at the time this class is instantiated, but it's +# convenient if they're available later. +# When cls._MARKER is filled in with a list of Field objects, the name +# and type fields will have been populated. +class Field: + __slots__ = ('name', + 'type', + 'default', + 'default_factory', + 'repr', + 'hash', + 'init', + 'compare', + 'metadata', + '_field_type', # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, + metadata): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + self.metadata = (_EMPTY_METADATA + if metadata is None or len(metadata) == 0 else + types.MappingProxyType(metadata)) + self._field_type = None + + def __repr__(self): + return ('Field(' + f'name={self.name!r},' + f'type={self.type},' + f'default={self.default},' + f'default_factory={self.default_factory},' + f'init={self.init},' + f'repr={self.repr},' + f'hash={self.hash},' + f'compare={self.compare},' + f'metadata={self.metadata}' + ')') + + +# This function is used instead of exposing Field creation directly, +# so that a type checker can be told (via overloads) that this is a +# function whose type depends on its parameters. +def field(*, default=_MISSING, default_factory=_MISSING, init=True, repr=True, + hash=None, compare=True, metadata=None): + """Return an object to identify dataclass fields. + + default is the default value of the field. default_factory is a + 0-argument function called to initialize a field's value. If init + is True, the field will be a parameter to the class's __init__() + function. If repr is True, the field will be included in the + object's repr(). If hash is True, the field will be included in + the object's hash(). If compare is True, the field will be used in + comparison functions. metadata, if specified, must be a mapping + which is stored but not otherwise examined by dataclass. + + It is an error to specify both default and default_factory. + """ + + if default is not _MISSING and default_factory is not _MISSING: + raise ValueError('cannot specify both default and default_factory') + return Field(default, default_factory, init, repr, hash, compare, + metadata) + + +def _tuple_str(obj_name, fields): + # Return a string representing each field of obj_name as a tuple + # member. So, if fields is ['x', 'y'] and obj_name is "self", + # return "(self.x,self.y)". + + # Special case for the 0-tuple. + if len(fields) == 0: + return '()' + # Note the trailing comma, needed if this turns out to be a 1-tuple. + return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' + + +def _create_fn(name, args, body, globals=None, locals=None, + return_type=_MISSING): + # Note that we mutate locals when exec() is called. Caller beware! + if locals is None: + locals = {} + return_annotation = '' + if return_type is not _MISSING: + locals['_return_type'] = return_type + return_annotation = '->_return_type' + args = ','.join(args) + body = '\n'.join(f' {b}' for b in body) + + txt = f'def {name}({args}){return_annotation}:\n{body}' + + exec(txt, globals, locals) + return locals[name] + + +def _field_assign(frozen, name, value, self_name): + # If we're a frozen class, then assign to our fields in __init__ + # via object.__setattr__. Otherwise, just use a simple + # assignment. + # self_name is what "self" is called in this function: don't + # hard-code "self", since that might be a field name. + if frozen: + return f'object.__setattr__({self_name},{name!r},{value})' + return f'{self_name}.{name}={value}' + + +def _field_init(f, frozen, globals, self_name): + # Return the text of the line in the body of __init__ that will + # initialize this field. + + default_name = f'_dflt_{f.name}' + if f.default_factory is not _MISSING: + if f.init: + # This field has a default factory. If a parameter is + # given, use it. If not, call the factory. + globals[default_name] = f.default_factory + value = (f'{default_name}() ' + f'if {f.name} is _HAS_DEFAULT_FACTORY ' + f'else {f.name}') + else: + # This is a field that's not in the __init__ params, but + # has a default factory function. It needs to be + # initialized here by calling the factory function, + # because there's no other way to initialize it. + + # For a field initialized with a default=defaultvalue, the + # class dict just has the default value + # (cls.fieldname=defaultvalue). But that won't work for a + # default factory, the factory must be called in __init__ + # and we must assign that to self.fieldname. We can't + # fall back to the class dict's value, both because it's + # not set, and because it might be different per-class + # (which, after all, is why we have a factory function!). + + globals[default_name] = f.default_factory + value = f'{default_name}()' + else: + # No default factory. + if f.init: + if f.default is _MISSING: + # There's no default, just do an assignment. + value = f.name + elif f.default is not _MISSING: + globals[default_name] = f.default + value = f.name + else: + # This field does not need initialization. Signify that to + # the caller by returning None. + return None + + # Only test this now, so that we can create variables for the + # default. However, return None to signify that we're not going + # to actually do the assignment statement for InitVars. + if f._field_type == _FIELD_INITVAR: + return None + + # Now, actually generate the field assignment. + return _field_assign(frozen, f.name, value, self_name) + + +def _init_param(f): + # Return the __init__ parameter string for this field. + # For example, the equivalent of 'x:int=3' (except instead of 'int', + # reference a variable set to int, and instead of '3', reference a + # variable set to 3). + if f.default is _MISSING and f.default_factory is _MISSING: + # There's no default, and no default_factory, just + # output the variable name and type. + default = '' + elif f.default is not _MISSING: + # There's a default, this will be the name that's used to look it up. + default = f'=_dflt_{f.name}' + elif f.default_factory is not _MISSING: + # There's a factory function. Set a marker. + default = '=_HAS_DEFAULT_FACTORY' + return f'{f.name}:_type_{f.name}{default}' + + +def _init_fn(fields, frozen, has_post_init, self_name): + # fields contains both real fields and InitVar pseudo-fields. + + # Make sure we don't have fields without defaults following fields + # with defaults. This actually would be caught when exec-ing the + # function source code, but catching it here gives a better error + # message, and future-proofs us in case we build up the function + # using ast. + seen_default = False + for f in fields: + # Only consider fields in the __init__ call. + if f.init: + if not (f.default is _MISSING and f.default_factory is _MISSING): + seen_default = True + elif seen_default: + raise TypeError(f'non-default argument {f.name!r} ' + 'follows default argument') + + globals = {'_MISSING': _MISSING, + '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY} + + body_lines = [] + for f in fields: + # Do not initialize the pseudo-fields, only the real ones. + line = _field_init(f, frozen, globals, self_name) + if line is not None: + # line is None means that this field doesn't require + # initialization. Just skip it. + body_lines.append(line) + + # Does this class have a post-init function? + if has_post_init: + params_str = ','.join(f.name for f in fields + if f._field_type is _FIELD_INITVAR) + body_lines += [f'{self_name}.{_POST_INIT_NAME}({params_str})'] + + # If no body lines, use 'pass'. + if len(body_lines) == 0: + body_lines = ['pass'] + + locals = {f'_type_{f.name}': f.type for f in fields} + return _create_fn('__init__', + [self_name] +[_init_param(f) for f in fields if f.init], + body_lines, + locals=locals, + globals=globals, + return_type=None) + + +def _repr_fn(fields): + return _create_fn('__repr__', + ['self'], + ['return self.__class__.__qualname__ + f"(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in fields]) + + ')"']) + + +def _frozen_setattr(self, name, value): + raise FrozenInstanceError(f'cannot assign to field {name!r}') + + +def _frozen_delattr(self, name): + raise FrozenInstanceError(f'cannot delete field {name!r}') + + +def _cmp_fn(name, op, self_tuple, other_tuple): + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + + return _create_fn(name, + ['self', 'other'], + [ 'if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + 'return NotImplemented']) + + +def _set_eq_fns(cls, fields): + # Create and set the equality comparison methods on cls. + # Pre-compute self_tuple and other_tuple, then re-use them for + # each function. + self_tuple = _tuple_str('self', fields) + other_tuple = _tuple_str('other', fields) + for name, op in [('__eq__', '=='), + ('__ne__', '!='), + ]: + _set_attribute(cls, name, _cmp_fn(name, op, self_tuple, other_tuple)) + + +def _set_order_fns(cls, fields): + # Create and set the ordering methods on cls. + # Pre-compute self_tuple and other_tuple, then re-use them for + # each function. + self_tuple = _tuple_str('self', fields) + other_tuple = _tuple_str('other', fields) + for name, op in [('__lt__', '<'), + ('__le__', '<='), + ('__gt__', '>'), + ('__ge__', '>='), + ]: + _set_attribute(cls, name, _cmp_fn(name, op, self_tuple, other_tuple)) + + +def _hash_fn(fields): + self_tuple = _tuple_str('self', fields) + return _create_fn('__hash__', + ['self'], + [f'return hash({self_tuple})']) + + +def _get_field(cls, a_name, a_type): + # Return a Field object, for this field name and type. ClassVars + # and InitVars are also returned, but marked as such (see + # f._field_type). + + # If the default value isn't derived from field, then it's + # only a normal default value. Convert it to a Field(). + default = getattr(cls, a_name, _MISSING) + if isinstance(default, Field): + f = default + else: + f = field(default=default) + + # Assume it's a normal field until proven otherwise. + f._field_type = _FIELD + + # Only at this point do we know the name and the type. Set them. + f.name = a_name + f.type = a_type + + # If typing has not been imported, then it's impossible for + # any annotation to be a ClassVar. So, only look for ClassVar + # if typing has been imported. + typing = sys.modules.get('typing') + if typing is not None: + # This test uses a typing internal class, but it's the best + # way to test if this is a ClassVar. + if type(a_type) is typing._ClassVar: + # This field is a ClassVar, so it's not a field. + f._field_type = _FIELD_CLASSVAR + + if f._field_type is _FIELD: + # Check if this is an InitVar. + if a_type is InitVar: + # InitVars are not fields, either. + f._field_type = _FIELD_INITVAR + + # Validations for fields. This is delayed until now, instead of + # in the Field() constructor, since only here do we know the field + # name, which allows better error reporting. + + # Special restrictions for ClassVar and InitVar. + if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): + if f.default_factory is not _MISSING: + raise TypeError(f'field {f.name} cannot have a ' + 'default factory') + # Should I check for other field settings? default_factory + # seems the most serious to check for. Maybe add others. For + # example, how about init=False (or really, + # init=)? It makes no sense for + # ClassVar and InitVar to specify init=. + + # For real fields, disallow mutable defaults for known types. + if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): + raise ValueError(f'mutable default {type(f.default)} for field ' + f'{f.name} is not allowed: use default_factory') + + return f + + +def _find_fields(cls): + # Return a list of Field objects, in order, for this class (and no + # base classes). Fields are found from __annotations__ (which is + # guaranteed to be ordered). Default values are from class + # attributes, if a field has a default. If the default value is + # a Field(), then it contains additional info beyond (and + # possibly including) the actual default value. Pseudo-fields + # ClassVars and InitVars are included, despite the fact that + # they're not real fields. That's deal with later. + + annotations = getattr(cls, '__annotations__', {}) + + return [_get_field(cls, a_name, a_type) + for a_name, a_type in annotations.items()] + + +def _set_attribute(cls, name, value): + # Raise TypeError if an attribute by this name already exists. + if name in cls.__dict__: + raise TypeError(f'Cannot overwrite attribute {name} ' + f'in {cls.__name__}') + setattr(cls, name, value) + + +def _process_class(cls, repr, eq, order, hash, init, frozen): + # Use an OrderedDict because: + # - Order matters! + # - Derived class fields overwrite base class fields, but the + # order is defined by the base class, which is found first. + fields = collections.OrderedDict() + + # Find our base classes in reverse MRO order, and exclude + # ourselves. In reversed order so that more derived classes + # override earlier field definitions in base classes. + for b in cls.__mro__[-1:0:-1]: + # Only process classes that have been processed by our + # decorator. That is, they have a _MARKER attribute. + base_fields = getattr(b, _MARKER, None) + if base_fields: + for f in base_fields.values(): + fields[f.name] = f + + # Now find fields in our class. While doing so, validate some + # things, and set the default values (as class attributes) + # where we can. + for f in _find_fields(cls): + fields[f.name] = f + + # If the class attribute (which is the default value for + # this field) exists and is of type 'Field', replace it + # with the real default. This is so that normal class + # introspection sees a real default value, not a Field. + if isinstance(getattr(cls, f.name, None), Field): + if f.default is _MISSING: + # If there's no default, delete the class attribute. + # This happens if we specify field(repr=False), for + # example (that is, we specified a field object, but + # no default value). Also if we're using a default + # factory. The class attribute should not be set at + # all in the post-processed class. + delattr(cls, f.name) + else: + setattr(cls, f.name, f.default) + + # Remember all of the fields on our class (including bases). This + # marks this class as being a dataclass. + setattr(cls, _MARKER, fields) + + # We also need to check if a parent class is frozen: frozen has to + # be inherited down. + is_frozen = frozen or cls.__setattr__ is _frozen_setattr + + # If we're generating ordering methods, we must be generating + # the eq methods. + if order and not eq: + raise ValueError('eq must be true if order is true') + + if init: + # Does this class have a post-init function? + has_post_init = hasattr(cls, _POST_INIT_NAME) + + # Include InitVars and regular fields (so, not ClassVars). + _set_attribute(cls, '__init__', + _init_fn(list(filter(lambda f: f._field_type + in (_FIELD, _FIELD_INITVAR), + fields.values())), + is_frozen, + has_post_init, + # The name to use for the "self" param + # in __init__. Use "self" if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + )) + + # Get the fields as a list, and include only real fields. This is + # used in all of the following methods. + field_list = list(filter(lambda f: f._field_type is _FIELD, + fields.values())) + + if repr: + _set_attribute(cls, '__repr__', + _repr_fn(list(filter(lambda f: f.repr, field_list)))) + + if is_frozen: + _set_attribute(cls, '__setattr__', _frozen_setattr) + _set_attribute(cls, '__delattr__', _frozen_delattr) + + generate_hash = False + if hash is None: + if eq and frozen: + # Generate a hash function. + generate_hash = True + elif eq and not frozen: + # Not hashable. + _set_attribute(cls, '__hash__', None) + elif not eq: + # Otherwise, use the base class definition of hash(). That is, + # don't set anything on this class. + pass + else: + assert "can't get here" + else: + generate_hash = hash + if generate_hash: + _set_attribute(cls, '__hash__', + _hash_fn(list(filter(lambda f: f.compare + if f.hash is None + else f.hash, + field_list)))) + + if eq: + # Create and __eq__ and __ne__ methods. + _set_eq_fns(cls, list(filter(lambda f: f.compare, field_list))) + + if order: + # Create and __lt__, __le__, __gt__, and __ge__ methods. + # Create and set the comparison functions. + _set_order_fns(cls, list(filter(lambda f: f.compare, field_list))) + + if not getattr(cls, '__doc__'): + # Create a class doc-string. + cls.__doc__ = (cls.__name__ + + str(inspect.signature(cls)).replace(' -> None', '')) + + return cls + + +# _cls should never be specified by keyword, so start it with an +# underscore. The presense of _cls is used to detect if this +# decorator is being called with parameters or not. +def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, + hash=None, frozen=False): + """Returns the same class as was passed in, with dunder methods + added based on the fields defined in the class. + + Examines PEP 526 __annotations__ to determine fields. + + If init is true, an __init__() method is added to the class. If + repr is true, a __repr__() method is added. If order is true, rich + comparison dunder methods are added. If hash is true, a __hash__() + method function is added. If frozen is true, fields may not be + assigned to after instance creation. + """ + + def wrap(cls): + return _process_class(cls, repr, eq, order, hash, init, frozen) + + # See if we're being called as @dataclass or @dataclass(). + if _cls is None: + # We're called with parens. + return wrap + + # We're called as @dataclass without parens. + return wrap(_cls) + + +def fields(class_or_instance): + """Return a tuple describing the fields of this dataclass. + + Accepts a dataclass or an instance of one. Tuple elements are of + type Field. + """ + + # Might it be worth caching this, per class? + try: + fields = getattr(class_or_instance, _MARKER) + except AttributeError: + raise TypeError('must be called with a dataclass type or instance') + + # Exclude pseudo-fields. + return tuple(f for f in fields.values() if f._field_type is _FIELD) + + +def _isdataclass(obj): + """Returns True if obj is an instance of a dataclass.""" + return not isinstance(obj, type) and hasattr(obj, _MARKER) + + +def asdict(obj, *, dict_factory=dict): + """Return the fields of a dataclass instance as a new dictionary mapping + field names to field values. + + Example usage: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert asdict(c) == {'x': 1, 'y': 2} + + If given, 'dict_factory' will be used instead of built-in dict. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + if not _isdataclass(obj): + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + +def _asdict_inner(obj, dict_factory): + if _isdataclass(obj): + result = [] + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name), dict_factory) + result.append((f.name, value)) + return dict_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + else: + return deepcopy(obj) + + +def astuple(obj, *, tuple_factory=tuple): + """Return the fields of a dataclass instance as a new tuple of field values. + + Example usage:: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert asdtuple(c) == (1, 2) + + If given, 'tuple_factory' will be used instead of built-in tuple. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + + if not _isdataclass(obj): + raise TypeError("astuple() should be called on dataclass instances") + return _astuple_inner(obj, tuple_factory) + +def _astuple_inner(obj, tuple_factory): + if _isdataclass(obj): + result = [] + for f in fields(obj): + value = _astuple_inner(getattr(obj, f.name), tuple_factory) + result.append(value) + return tuple_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + for k, v in obj.items()) + else: + return deepcopy(obj) + + +def make_dataclass(cls_name, fields, *, bases=(), namespace=None): + """Return a new dynamically created dataclass. + + The dataclass name will be 'cls_name'. 'fields' is an interable + of either (name, type) or (name, type, Field) objects. Field + objects are created by calling 'field(name, type [, Field])'. + + C = make_class('C', [('a', int', ('b', int, Field(init=False))], bases=Base) + + is equivalent to: + + @dataclass + class C(Base): + a: int + b: int = field(init=False) + + For the bases and namespace paremeters, see the builtin type() function. + """ + + if namespace is None: + namespace = {} + else: + # Copy namespace since we're going to mutate it. + namespace = namespace.copy() + + anns = collections.OrderedDict((name, tp) for name, tp, *_ in fields) + namespace['__annotations__'] = anns + for item in fields: + if len(item) == 3: + name, tp, spec = item + namespace[name] = spec + cls = type(cls_name, bases, namespace) + return dataclass(cls) + + +def replace(obj, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for frozen classes. Example usage: + + @dataclass(frozen=True) + class C: + x: int + y: int + + c = C(1, 2) + c1 = replace(c, x=3) + assert c1.x == 3 and c1.y == 2 + """ + + # We're going to mutate 'changes', but that's okay because it's a new + # dict, even if called with 'replace(obj, **my_changes)'. + + if not _isdataclass(obj): + raise TypeError("replace() should be called on dataclass instances") + + # It's an error to have init=False fields in 'changes'. + # If a field is not in 'changes', read its value from the provided obj. + + for f in getattr(obj, _MARKER).values(): + if not f.init: + # Error if this field is specified in changes. + if f.name in changes: + raise ValueError(f'field {f.name} is declared with ' + 'init=False, it cannot be specified with ' + 'replace()') + continue + + if f.name not in changes: + changes[f.name] = getattr(obj, f.name) + + # Create the new object, which calls __init__() and __post_init__ + # (if defined), using all of the init fields we've added and/or + # left in 'changes'. + # If there are values supplied in changes that aren't fields, this + # will correctly raise a TypeError. + return obj.__class__(**changes) diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py index bf8d8dd2f5a43e..2171abd6969f68 100644 --- a/Lib/distutils/config.py +++ b/Lib/distutils/config.py @@ -51,7 +51,6 @@ def _read_pypirc(self): if os.path.exists(rc): self.announce('Using PyPI login from %s' % rc) repository = self.repository or self.DEFAULT_REPOSITORY - realm = self.realm or self.DEFAULT_REALM config = RawConfigParser() config.read(rc) diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index 62a24516cfafe5..6cf0a0d6632dc7 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -27,6 +27,20 @@ command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') +def _ensure_list(value, fieldname): + if isinstance(value, str): + # a string containing comma separated values is okay. It will + # be converted to a list by Distribution.finalize_options(). + pass + elif not isinstance(value, list): + # passing a tuple or an iterator perhaps, warn and convert + typename = type(value).__name__ + msg = f"Warning: '{fieldname}' should be a list, got type '{typename}'" + log.log(log.WARN, msg) + value = list(value) + return value + + class Distribution: """The core of the Distutils. Most of the work hiding behind 'setup' is really done within a Distribution instance, which farms the work out @@ -257,10 +271,7 @@ def __init__(self, attrs=None): setattr(self, key, val) else: msg = "Unknown distribution option: %s" % repr(key) - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") + warnings.warn(msg) # no-user-cfg is handled before other command line args # because other args override the config files, and this @@ -1188,12 +1199,21 @@ def get_long_description(self): def get_keywords(self): return self.keywords or [] + def set_keywords(self, value): + self.keywords = _ensure_list(value, 'keywords') + def get_platforms(self): return self.platforms or ["UNKNOWN"] + def set_platforms(self, value): + self.platforms = _ensure_list(value, 'platforms') + def get_classifiers(self): return self.classifiers or [] + def set_classifiers(self, value): + self.classifiers = _ensure_list(value, 'classifiers') + def get_download_url(self): return self.download_url or "UNKNOWN" @@ -1205,7 +1225,7 @@ def set_requires(self, value): import distutils.versionpredicate for v in value: distutils.versionpredicate.VersionPredicate(v) - self.requires = value + self.requires = list(value) def get_provides(self): return self.provides or [] @@ -1224,7 +1244,7 @@ def set_obsoletes(self, value): import distutils.versionpredicate for v in value: distutils.versionpredicate.VersionPredicate(v) - self.obsoletes = value + self.obsoletes = list(value) def fix_help_options(options): """Convert a 4-tuple 'help_options' list as found in various command diff --git a/Lib/distutils/tests/test_archive_util.py b/Lib/distutils/tests/test_archive_util.py index 02fa1e27a47f53..14ba4ca34b4ac9 100644 --- a/Lib/distutils/tests/test_archive_util.py +++ b/Lib/distutils/tests/test_archive_util.py @@ -162,7 +162,7 @@ def test_tarfile_vs_tar(self): # now create another tarball using `tar` tarball2 = os.path.join(tmpdir, 'archive2.tar.gz') tar_cmd = ['tar', '-cf', 'archive2.tar', 'dist'] - gzip_cmd = ['gzip', '-f9', 'archive2.tar'] + gzip_cmd = ['gzip', '-f', '-9', 'archive2.tar'] old_dir = os.getcwd() os.chdir(tmpdir) try: diff --git a/Lib/distutils/tests/test_dist.py b/Lib/distutils/tests/test_dist.py index 1f104cef675d9a..0a19f0fb6274cb 100644 --- a/Lib/distutils/tests/test_dist.py +++ b/Lib/distutils/tests/test_dist.py @@ -11,7 +11,9 @@ from distutils.dist import Distribution, fix_help_options, DistributionMetadata from distutils.cmd import Command -from test.support import TESTFN, captured_stdout, run_unittest +from test.support import ( + TESTFN, captured_stdout, captured_stderr, run_unittest +) from distutils.tests import support from distutils import log @@ -195,6 +197,13 @@ def test_finalize_options(self): self.assertEqual(dist.metadata.platforms, ['one', 'two']) self.assertEqual(dist.metadata.keywords, ['one', 'two']) + attrs = {'keywords': 'foo bar', + 'platforms': 'foo bar'} + dist = Distribution(attrs=attrs) + dist.finalize_options() + self.assertEqual(dist.metadata.platforms, ['foo bar']) + self.assertEqual(dist.metadata.keywords, ['foo bar']) + def test_get_command_packages(self): dist = Distribution() self.assertEqual(dist.command_packages, None) @@ -312,6 +321,13 @@ def test_requires_illegal(self): "version": "1.0", "requires": ["my.pkg (splat)"]}) + def test_requires_to_list(self): + attrs = {"name": "package", + "requires": iter(["other"])} + dist = Distribution(attrs) + self.assertIsInstance(dist.metadata.requires, list) + + def test_obsoletes(self): attrs = {"name": "package", "version": "1.0", @@ -334,13 +350,69 @@ def test_obsoletes_illegal(self): "version": "1.0", "obsoletes": ["my.pkg (splat)"]}) + def test_obsoletes_to_list(self): + attrs = {"name": "package", + "obsoletes": iter(["other"])} + dist = Distribution(attrs) + self.assertIsInstance(dist.metadata.obsoletes, list) + def test_classifier(self): attrs = {'name': 'Boa', 'version': '3.0', 'classifiers': ['Programming Language :: Python :: 3']} dist = Distribution(attrs) + self.assertEqual(dist.get_classifiers(), + ['Programming Language :: Python :: 3']) meta = self.format_metadata(dist) self.assertIn('Metadata-Version: 1.1', meta) + def test_classifier_invalid_type(self): + attrs = {'name': 'Boa', 'version': '3.0', + 'classifiers': ('Programming Language :: Python :: 3',)} + with captured_stderr() as error: + d = Distribution(attrs) + # should have warning about passing a non-list + self.assertIn('should be a list', error.getvalue()) + # should be converted to a list + self.assertIsInstance(d.metadata.classifiers, list) + self.assertEqual(d.metadata.classifiers, + list(attrs['classifiers'])) + + def test_keywords(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ['spam', 'eggs', 'life of brian']} + dist = Distribution(attrs) + self.assertEqual(dist.get_keywords(), + ['spam', 'eggs', 'life of brian']) + + def test_keywords_invalid_type(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ('spam', 'eggs', 'life of brian')} + with captured_stderr() as error: + d = Distribution(attrs) + # should have warning about passing a non-list + self.assertIn('should be a list', error.getvalue()) + # should be converted to a list + self.assertIsInstance(d.metadata.keywords, list) + self.assertEqual(d.metadata.keywords, list(attrs['keywords'])) + + def test_platforms(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ['GNU/Linux', 'Some Evil Platform']} + dist = Distribution(attrs) + self.assertEqual(dist.get_platforms(), + ['GNU/Linux', 'Some Evil Platform']) + + def test_platforms_invalid_types(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ('GNU/Linux', 'Some Evil Platform')} + with captured_stderr() as error: + d = Distribution(attrs) + # should have warning about passing a non-list + self.assertIn('should be a list', error.getvalue()) + # should be converted to a list + self.assertIsInstance(d.metadata.platforms, list) + self.assertEqual(d.metadata.platforms, list(attrs['platforms'])) + def test_download_url(self): attrs = {'name': 'Boa', 'version': '3.0', 'download_url': 'http://example.org/boa'} diff --git a/Lib/distutils/tests/test_file_util.py b/Lib/distutils/tests/test_file_util.py index 03040afc7966ef..a4e2d025f96615 100644 --- a/Lib/distutils/tests/test_file_util.py +++ b/Lib/distutils/tests/test_file_util.py @@ -8,7 +8,7 @@ from distutils import log from distutils.tests import support from distutils.errors import DistutilsFileError -from test.support import run_unittest +from test.support import run_unittest, unlink class FileUtilTestCase(support.TempdirManager, unittest.TestCase): @@ -80,6 +80,14 @@ def test_move_file_exception_unpacking_unlink(self): def test_copy_file_hard_link(self): with open(self.source, 'w') as f: f.write('some content') + # Check first that copy_file() will not fall back on copying the file + # instead of creating the hard link. + try: + os.link(self.source, self.target) + except OSError as e: + self.skipTest('os.link: %s' % e) + else: + unlink(self.target) st = os.stat(self.source) copy_file(self.source, self.target, link='hard') st2 = os.stat(self.source) diff --git a/Lib/doctest.py b/Lib/doctest.py index 5e5bc21a038670..c1d8a1db111ddd 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1611,7 +1611,7 @@ def check_output(self, want, got, optionflags): '', want) # If a line in got contains only spaces, then remove the # spaces. - got = re.sub(r'(?m)^\s*?$', '', got) + got = re.sub(r'(?m)^[^\S\n]+$', '', got) if got == want: return True diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9b9697f77346a6..b34c58bf85d5b4 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -96,90 +96,6 @@ def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' -# -# Accumulator for header folding -# - -class _Folded: - - def __init__(self, maxlen, policy): - self.maxlen = maxlen - self.policy = policy - self.lastlen = 0 - self.stickyspace = None - self.firstline = True - self.done = [] - self.current = [] - - def newline(self): - self.done.extend(self.current) - self.done.append(self.policy.linesep) - self.current.clear() - self.lastlen = 0 - - def finalize(self): - if self.current: - self.newline() - - def __str__(self): - return ''.join(self.done) - - def append(self, stoken): - self.current.append(stoken) - - def append_if_fits(self, token, stoken=None): - if stoken is None: - stoken = str(token) - l = len(stoken) - if self.stickyspace is not None: - stickyspace_len = len(self.stickyspace) - if self.lastlen + stickyspace_len + l <= self.maxlen: - self.current.append(self.stickyspace) - self.lastlen += stickyspace_len - self.current.append(stoken) - self.lastlen += l - self.stickyspace = None - self.firstline = False - return True - if token.has_fws: - ws = token.pop_leading_fws() - if ws is not None: - self.stickyspace += str(ws) - stickyspace_len += len(ws) - token._fold(self) - return True - if stickyspace_len and l + 1 <= self.maxlen: - margin = self.maxlen - l - if 0 < margin < stickyspace_len: - trim = stickyspace_len - margin - self.current.append(self.stickyspace[:trim]) - self.stickyspace = self.stickyspace[trim:] - stickyspace_len = trim - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.lastlen = l + stickyspace_len - self.stickyspace = None - self.firstline = False - return True - if not self.firstline: - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.stickyspace = None - self.firstline = False - return True - if self.lastlen + l <= self.maxlen: - self.current.append(stoken) - self.lastlen += l - return True - if l < self.maxlen: - self.newline() - self.current.append(stoken) - self.lastlen = l - return True - return False - # # TokenList and its subclasses # @@ -187,6 +103,8 @@ def append_if_fits(self, token, stoken=None): class TokenList(list): token_type = None + syntactic_break = True + ew_combine_allowed = True def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -207,84 +125,13 @@ def value(self): def all_defects(self): return sum((x.all_defects for x in self), self.defects) - # - # Folding API - # - # parts(): - # - # return a list of objects that constitute the "higher level syntactic - # objects" specified by the RFC as the best places to fold a header line. - # The returned objects must include leading folding white space, even if - # this means mutating the underlying parse tree of the object. Each object - # is only responsible for returning *its* parts, and should not drill down - # to any lower level except as required to meet the leading folding white - # space constraint. - # - # _fold(folded): - # - # folded: the result accumulator. This is an instance of _Folded. - # (XXX: I haven't finished factoring this out yet, the folding code - # pretty much uses this as a state object.) When the folded.current - # contains as much text as will fit, the _fold method should call - # folded.newline. - # folded.lastlen: the current length of the test stored in folded.current. - # folded.maxlen: The maximum number of characters that may appear on a - # folded line. Differs from the policy setting in that "no limit" is - # represented by +inf, which means it can be used in the trivially - # logical fashion in comparisons. - # - # Currently no subclasses implement parts, and I think this will remain - # true. A subclass only needs to implement _fold when the generic version - # isn't sufficient. _fold will need to be implemented primarily when it is - # possible for encoded words to appear in the specialized token-list, since - # there is no generic algorithm that can know where exactly the encoded - # words are allowed. A _fold implementation is responsible for filling - # lines in the same general way that the top level _fold does. It may, and - # should, call the _fold method of sub-objects in a similar fashion to that - # of the top level _fold. - # - # XXX: I'm hoping it will be possible to factor the existing code further - # to reduce redundancy and make the logic clearer. - - @property - def parts(self): - klass = self.__class__ - this = [] - for token in self: - if token.startswith_fws(): - if this: - yield this[0] if len(this)==1 else klass(this) - this.clear() - end_ws = token.pop_trailing_ws() - this.append(token) - if end_ws: - yield klass(this) - this = [end_ws] - if this: - yield this[0] if len(this)==1 else klass(this) - def startswith_fws(self): return self[0].startswith_fws() - def pop_leading_fws(self): - if self[0].token_type == 'fws': - return self.pop(0) - return self[0].pop_leading_fws() - - def pop_trailing_ws(self): - if self[-1].token_type == 'cfws': - return self.pop(-1) - return self[-1].pop_trailing_ws() - @property - def has_fws(self): - for part in self: - if part.has_fws: - return True - return False - - def has_leading_comment(self): - return self[0].has_leading_comment() + def as_ew_allowed(self): + """True if all top level tokens of this part may be RFC2047 encoded.""" + return all(part.as_ew_allowed for part in self) @property def comments(self): @@ -294,69 +141,13 @@ def comments(self): return comments def fold(self, *, policy): - # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") - folded = _Folded(maxlen, policy) - self._fold(folded) - folded.finalize() - return str(folded) - - def as_encoded_word(self, charset): - # This works only for things returned by 'parts', which include - # the leading fws, if any, that should be used. - res = [] - ws = self.pop_leading_fws() - if ws: - res.append(ws) - trailer = self.pop(-1) if self[-1].token_type=='fws' else '' - res.append(_ew.encode(str(self), charset)) - res.append(trailer) - return ''.join(res) - - def cte_encode(self, charset, policy): - res = [] - for part in self: - res.append(part.cte_encode(charset, policy)) - return ''.join(res) - - def _fold(self, folded): - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - # XXX: this should be a policy setting when utf8 is False. - charset = 'utf-8' - tstr = part.cte_encode(charset, folded.policy) - tlen = len(tstr) - if folded.append_if_fits(part, tstr): - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # There are no fold points in this one; it is too long for a single - # line and can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() + return _refold_parse_tree(self, policy=policy) def pprint(self, indent=''): - print('\n'.join(self._pp(indent=''))) + print(self.ppstr(indent=indent)) def ppstr(self, indent=''): - return '\n'.join(self._pp(indent='')) + return '\n'.join(self._pp(indent=indent)) def _pp(self, indent=''): yield '{}{}/{}('.format( @@ -391,173 +182,11 @@ class UnstructuredTokenList(TokenList): token_type = 'unstructured' - def _fold(self, folded): - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - is_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None: - # We've already done an EW, combine this one with it - # if there's room. - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - oldlastlen = sum(len(x) for x in folded.current[:last_ew]) - schunk = str(chunk) - lchunk = len(schunk) - if oldlastlen + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = oldlastlen + lchunk - continue - tstr = part.as_encoded_word(charset) - is_ew = True - if folded.append_if_fits(part, tstr): - if is_ew: - last_ew = len(folded.current) - 1 - continue - if is_ew or last_ew: - # It's too big to fit on the line, but since we've - # got encoded words we can use encoded word folding. - part._fold_as_ew(folded) - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # It can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - last_ew = None - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - if last_ew is None: - res.append(part.cte_encode(charset, policy)) - last_ew = len(res) - else: - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res.append(tl.as_encoded_word(charset)) - return ''.join(res) - class Phrase(TokenList): token_type = 'phrase' - def _fold(self, folded): - # As with Unstructured, we can have pure ASCII with or without - # surrogateescape encoded bytes, or we could have unicode. But this - # case is more complicated, since we have to deal with the various - # sub-token types and how they can be composed in the face of - # unicode-that-needs-CTE-encoding, and the fact that if a token a - # comment that becomes a barrier across which we can't compose encoded - # words. - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - has_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None and not part.has_leading_comment(): - # We've already done an EW, let's see if we can combine - # this one with it. The last_ew logic ensures that all we - # have at this point is atoms, no comments or quoted - # strings. So we can treat the text between the last - # encoded word and the content of this token as - # unstructured text, and things will work correctly. But - # we have to strip off any trailing comment on this token - # first, and if it is a quoted string we have to pull out - # the content (we're encoding it, so it no longer needs to - # be quoted). - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - schunk = str(chunk) - lchunk = len(schunk) - if last_ew + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = sum(len(x) for x in folded.current) - continue - tstr = part.as_encoded_word(charset) - tlen = len(tstr) - has_ew = True - if folded.append_if_fits(part, tstr): - if has_ew and not part.comments: - last_ew = len(folded.current) - 1 - elif part.comments or part.token_type == 'quoted-string': - # If a comment is involved we can't combine EWs. And if a - # quoted string is involved, it's not worth the effort to - # try to combine them. - last_ew = None - continue - part._fold(folded) - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - is_ew = False - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - is_ew = True - if last_ew is None: - if not part.comments: - last_ew = len(res) - res.append(part.cte_encode(charset, policy)) - elif not part.has_leading_comment(): - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res[last_ew:] = [tl.as_encoded_word(charset)] - if part.comments or (not is_ew and part.token_type == 'quoted-string'): - last_ew = None - return ''.join(res) - class Word(TokenList): token_type = 'word' @@ -567,9 +196,6 @@ class CFWSList(WhiteSpaceTokenList): token_type = 'cfws' - def has_leading_comment(self): - return bool(self.comments) - class Atom(TokenList): @@ -579,6 +205,7 @@ class Atom(TokenList): class Token(TokenList): token_type = 'token' + encode_as_ew = False class EncodedWord(TokenList): @@ -588,13 +215,6 @@ class EncodedWord(TokenList): charset = None lang = None - @property - def encoded(self): - if self.cte is not None: - return self.cte - _ew.encode(str(self), self.charset) - - class QuotedString(TokenList): @@ -865,6 +485,7 @@ def display_name(self): class Domain(TokenList): token_type = 'domain' + as_ew_allowed = False @property def domain(self): @@ -879,11 +500,13 @@ class DotAtom(TokenList): class DotAtomText(TokenList): token_type = 'dot-atom-text' + as_ew_allowed = True class AddrSpec(TokenList): token_type = 'addr-spec' + as_ew_allowed = False @property def local_part(self): @@ -916,11 +539,13 @@ def addr_spec(self): class ObsLocalPart(TokenList): token_type = 'obs-local-part' + as_ew_allowed = False class DisplayName(Phrase): token_type = 'display-name' + ew_combine_allowed = False @property def display_name(self): @@ -960,6 +585,7 @@ def value(self): class LocalPart(TokenList): token_type = 'local-part' + as_ew_allowed = False @property def value(self): @@ -995,6 +621,7 @@ def local_part(self): class DomainLiteral(TokenList): token_type = 'domain-literal' + as_ew_allowed = False @property def domain(self): @@ -1081,6 +708,7 @@ def stripped_value(self): class MimeParameters(TokenList): token_type = 'mime-parameters' + syntactic_break = False @property def params(self): @@ -1165,6 +793,10 @@ def __str__(self): class ParameterizedHeaderValue(TokenList): + # Set this false so that the value doesn't wind up on a new line even + # if it and the parameters would fit there but not on the first line. + syntactic_break = False + @property def params(self): for token in reversed(self): @@ -1172,18 +804,11 @@ def params(self): return token.params return {} - @property - def parts(self): - if self and self[-1].token_type == 'mime-parameters': - # We don't want to start a new line if all of the params don't fit - # after the value, so unwrap the parameter list. - return TokenList(self[:-1] + self[-1]) - return TokenList(self).parts - class ContentType(ParameterizedHeaderValue): token_type = 'content-type' + as_ew_allowed = False maintype = 'text' subtype = 'plain' @@ -1191,40 +816,27 @@ class ContentType(ParameterizedHeaderValue): class ContentDisposition(ParameterizedHeaderValue): token_type = 'content-disposition' + as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): token_type = 'content-transfer-encoding' + as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): token_type = 'header-label' + as_ew_allowed = False class Header(TokenList): token_type = 'header' - def _fold(self, folded): - folded.append(str(self.pop(0))) - folded.lastlen = len(folded.current[0]) - # The first line of the header is different from all others: we don't - # want to start a new object on a new line if it has any fold points in - # it that would allow part of it to be on the first header line. - # Further, if the first fold point would fit on the new line, we want - # to do that, but if it doesn't we want to put it on the first line. - # Folded supports this via the stickyspace attribute. If this - # attribute is not None, it does the special handling. - folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else '' - rest = self.pop(0) - if self: - raise ValueError("Malformed Header token list") - rest._fold(folded) - # # Terminal classes and instances @@ -1232,6 +844,10 @@ def _fold(self, folded): class Terminal(str): + as_ew_allowed = True + ew_combine_allowed = True + syntactic_break = True + def __new__(cls, value, token_type): self = super().__new__(cls, value) self.token_type = token_type @@ -1241,6 +857,9 @@ def __new__(cls, value, token_type): def __repr__(self): return "{}({})".format(self.__class__.__name__, super().__repr__()) + def pprint(self): + print(self.__class__.__name__ + '/' + self.token_type) + @property def all_defects(self): return list(self.defects) @@ -1254,29 +873,14 @@ def _pp(self, indent=''): '' if not self.defects else ' {}'.format(self.defects), )] - def cte_encode(self, charset, policy): - value = str(self) - try: - value.encode('us-ascii') - return value - except UnicodeEncodeError: - return _ew.encode(value, charset) - def pop_trailing_ws(self): # This terminates the recursion. return None - def pop_leading_fws(self): - # This terminates the recursion. - return None - @property def comments(self): return [] - def has_leading_comment(self): - return False - def __getnewargs__(self): return(str(self), self.token_type) @@ -1290,8 +894,6 @@ def value(self): def startswith_fws(self): return True - has_fws = True - class ValueTerminal(Terminal): @@ -1302,11 +904,6 @@ def value(self): def startswith_fws(self): return False - has_fws = False - - def as_encoded_word(self, charset): - return _ew.encode(str(self), charset) - class EWWhiteSpaceTerminal(WhiteSpaceTerminal): @@ -1314,15 +911,9 @@ class EWWhiteSpaceTerminal(WhiteSpaceTerminal): def value(self): return '' - @property - def encoded(self): - return self[:] - def __str__(self): return '' - has_fws = True - # XXX these need to become classes and used as instances so # that a program can't change them in a parse tree and screw @@ -1354,15 +945,14 @@ def __str__(self): _wsp_splitter = re.compile(r'([{}]+)'.format(''.join(WSP))).split _non_atom_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATOM_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(ATOM_ENDS)))).match _non_printable_finder = re.compile(r"[\x00-\x20\x7F]").findall _non_token_end_matcher = re.compile(r"[^{}]+".format( - ''.join(TOKEN_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(TOKEN_ENDS)))).match _non_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(ATTRIBUTE_ENDS).replace('\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(ATTRIBUTE_ENDS)))).match _non_extended_attribute_end_matcher = re.compile(r"[^{}]+".format( - ''.join(EXTENDED_ATTRIBUTE_ENDS).replace( - '\\','\\\\').replace(']',r'\]'))).match + re.escape(''.join(EXTENDED_ATTRIBUTE_ENDS)))).match def _validate_xtext(xtext): """If input token contains ASCII non-printables, register a defect.""" @@ -2752,7 +2342,7 @@ def get_parameter(value): if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if value and value[0] != "'": token, value = get_attrtext(value) @@ -2761,7 +2351,7 @@ def get_parameter(value): if not value or value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if remainder is not None: # Treat the rest of value as bare quoted string content. @@ -2966,3 +2556,255 @@ def parse_content_transfer_encoding_header(value): token, value = get_phrase(value) cte_header.append(token) return cte_header + + +# +# Header folding +# +# Header folding is complex, with lots of rules and corner cases. The +# following code does its best to obey the rules and handle the corner +# cases, but you can be sure there are few bugs:) +# +# This folder generally canonicalizes as it goes, preferring the stringified +# version of each token. The tokens contain information that supports the +# folder, including which tokens can be encoded in which ways. +# +# Folded text is accumulated in a simple list of strings ('lines'), each +# one of which should be less than policy.max_line_length ('maxlen'). +# + +def _steal_trailing_WSP_if_exists(lines): + wsp = '' + if lines and lines[-1] and lines[-1][-1] in WSP: + wsp = lines[-1][-1] + lines[-1] = lines[-1][:-1] + return wsp + +def _refold_parse_tree(parse_tree, *, policy): + """Return string of contents of parse_tree folded according to RFC rules. + + """ + # max_line_length 0/None means no limit, ie: infinitely long. + maxlen = policy.max_line_length or float("+inf") + encoding = 'utf-8' if policy.utf8 else 'us-ascii' + lines = [''] + last_ew = None + wrap_as_ew_blocked = 0 + want_encoding = False + end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') + parts = list(parse_tree) + while parts: + part = parts.pop(0) + if part is end_ew_not_allowed: + wrap_as_ew_blocked -= 1 + continue + tstr = str(part) + try: + tstr.encode(encoding) + charset = encoding + except UnicodeEncodeError: + if any(isinstance(x, errors.UndecodableBytesDefect) + for x in part.all_defects): + charset = 'unknown-8bit' + else: + # If policy.utf8 is false this should really be taken from a + # 'charset' property on the policy. + charset = 'utf-8' + want_encoding = True + if part.token_type == 'mime-parameters': + # Mime parameter folding (using RFC2231) is extra special. + _fold_mime_parameters(part, lines, maxlen, encoding) + continue + if want_encoding and not wrap_as_ew_blocked: + if not part.as_ew_allowed: + want_encoding = False + last_ew = None + if part.syntactic_break: + encoded_part = part.fold(policy=policy)[:-1] # strip nl + if policy.linesep not in encoded_part: + # It fits on a single line + if len(encoded_part) > maxlen - len(lines[-1]): + # But not on this one, so start a new one. + newline = _steal_trailing_WSP_if_exists(lines) + # XXX what if encoded_part has no leading FWS? + lines.append(newline) + lines[-1] += encoded_part + continue + # Either this is not a major syntactic break, so we don't + # want it on a line by itself even if it fits, or it + # doesn't fit on a line by itself. Either way, fall through + # to unpacking the subparts and wrapping them. + if not hasattr(part, 'encode'): + # It's not a Terminal, do each piece individually. + parts = list(part) + parts + else: + # It's a terminal, wrap it as an encoded word, possibly + # combining it with previously encoded words if allowed. + last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, + part.ew_combine_allowed, charset) + want_encoding = False + continue + if len(tstr) <= maxlen - len(lines[-1]): + lines[-1] += tstr + continue + # This part is too long to fit. The RFC wants us to break at + # "major syntactic breaks", so unless we don't consider this + # to be one, check if it will fit on the next line by itself. + if (part.syntactic_break and + len(tstr) + 1 <= maxlen): + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + continue + if not hasattr(part, 'encode'): + # It's not a terminal, try folding the subparts. + newparts = list(part) + if not part.as_ew_allowed: + wrap_as_ew_blocked += 1 + newparts.append(end_ew_not_allowed) + parts = newparts + parts + continue + if part.as_ew_allowed and not wrap_as_ew_blocked: + # It doesn't need CTE encoding, but encode it anyway so we can + # wrap it. + parts.insert(0, part) + want_encoding = True + continue + # We can't figure out how to wrap, it, so give up. + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + else: + # We can't fold it onto the next line either... + lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep + +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): + """Fold string to_encode into lines as encoded word, combining if allowed. + Return the new value for last_ew, or None if ew_combine_allowed is False. + + If there is already an encoded word in the last line of lines (indicated by + a non-None value for last_ew) and ew_combine_allowed is true, decode the + existing ew, combine it with to_encode, and re-encode. Otherwise, encode + to_encode. In either case, split to_encode as necessary so that the + encoded segments fit within maxlen. + + """ + if last_ew is not None and ew_combine_allowed: + to_encode = str( + get_unstructured(lines[-1][last_ew:] + to_encode)) + lines[-1] = lines[-1][:last_ew] + if to_encode[0] in WSP: + # We're joining this to non-encoded text, so don't encode + # the leading blank. + leading_wsp = to_encode[0] + to_encode = to_encode[1:] + if (len(lines[-1]) == maxlen): + lines.append(_steal_trailing_WSP_if_exists(lines)) + lines[-1] += leading_wsp + trailing_wsp = '' + if to_encode[-1] in WSP: + # Likewise for the trailing space. + trailing_wsp = to_encode[-1] + to_encode = to_encode[:-1] + new_last_ew = len(lines[-1]) if last_ew is None else last_ew + while to_encode: + remaining_space = maxlen - len(lines[-1]) + # The RFC2047 chrome takes up 7 characters plus the length + # of the charset name. + encode_as = 'utf-8' if charset == 'us-ascii' else charset + text_space = remaining_space - len(encode_as) - 7 + if text_space <= 0: + lines.append(' ') + # XXX We'll get an infinite loop here if maxlen is <= 7 + continue + first_part = to_encode[:text_space] + ew = _ew.encode(first_part, charset=encode_as) + excess = len(ew) - remaining_space + if excess > 0: + # encode always chooses the shortest encoding, so this + # is guaranteed to fit at this point. + first_part = first_part[:-excess] + ew = _ew.encode(first_part) + lines[-1] += ew + to_encode = to_encode[len(first_part):] + if to_encode: + lines.append(' ') + new_last_ew = len(lines[-1]) + lines[-1] += trailing_wsp + return new_last_ew if ew_combine_allowed else None + +def _fold_mime_parameters(part, lines, maxlen, encoding): + """Fold TokenList 'part' into the 'lines' list as mime parameters. + + Using the decoded list of parameters and values, format them according to + the RFC rules, including using RFC2231 encoding if the value cannot be + expressed in 'encoding' and/or the paramter+value is too long to fit within + 'maxlen'. + + """ + # Special case for RFC2231 encoding: start from decoded values and use + # RFC2231 encoding iff needed. + # + # Note that the 1 and 2s being added to the length calculations are + # accounting for the possibly-needed spaces and semicolons we'll be adding. + # + for name, value in part.params: + # XXX What if this ';' puts us over maxlen the first time through the + # loop? We should split the header value onto a newline in that case, + # but to do that we need to recognize the need earlier or reparse the + # header, so I'm going to ignore that bug for now. It'll only put us + # one character over. + if not lines[-1].rstrip().endswith(';'): + lines[-1] += ';' + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + if utils._has_surrogates(value): + charset = 'unknown-8bit' + error_handler = 'surrogateescape' + else: + charset = 'utf-8' + if encoding_required: + encoded_value = urllib.parse.quote( + value, safe='', errors=error_handler) + tstr = "{}*={}''{}".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue + elif len(tstr) + 2 <= maxlen: + lines.append(' ' + tstr) + continue + # We need multiple sections. We are allowed to mix encoded and + # non-encoded sections, but we aren't going to. We'll encode them all. + section = 0 + extra_chrome = charset + "''" + while value: + chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) + if maxlen <= chrome_len + 3: + # We need room for the leading blank, the trailing semicolon, + # and at least one character of the value. If we don't + # have that, we'd be stuck, so in that case fall back to + # the RFC standard width. + maxlen = 78 + splitpoint = maxchars = maxlen - chrome_len - 2 + while True: + partial = value[:splitpoint] + encoded_value = urllib.parse.quote( + partial, safe='', errors=error_handler) + if len(encoded_value) <= maxchars: + break + splitpoint -= 1 + lines.append(" {}*{}*={}{}".format( + name, section, extra_chrome, encoded_value)) + extra_chrome = '' + section += 1 + value = value[splitpoint:] + if value: + lines[-1] += ';' diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 81fee146dcc357..00652049f2fa2f 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -245,13 +245,16 @@ def fold(self, *, policy): the header name and the ': ' separator. """ - # At some point we need to only put fws here if it was in the source. + # At some point we need to put fws here iif it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), parser.ValueTerminal(':', 'header-sep')]), - parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), - self._parse_tree]) + ]) + if self._parse_tree: + header.append( + parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')])) + header.append(self._parse_tree) return header.fold(policy=policy) diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index a7a1e9f61e1afa..be68adf667083b 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -3,6 +3,29 @@ Released on 2018-06-18? ======================== +bpo-32207: Improve tk event exception tracebacks in IDLE. +When tk event handling is driven by IDLE's run loop, a confusing +and distracting queue.EMPTY traceback context is no longer added +to tk event exception tracebacks. The traceback is now the same +as when event handling is driven by user code. Patch based on +a suggestion by Serhiy Storchaka. + +bpo-32164: Delete unused file idlelib/tabbedpages.py. +Use of TabbedPageSet in configdialog was replaced by ttk.Notebook. + +bpo-32100: Fix old and new bugs in pathbrowser; improve tests. +Patch mostly by Cheryl Sabella. + +bpo-31860: The font sample in the settings dialog is now editable. +Edits persist while IDLE remains open. +Patch by Serhiy Storchake and Terry Jan Reedy. + +bpo-31858: Restrict shell prompt manipulaton to the shell. +Editor and output windows only see an empty last prompt line. This +simplifies the code and fixes a minor bug when newline is inserted. +Sys.ps1, if present, is read on Shell start-up, but is not set or changed. +Patch by Terry Jan Reedy. + bpo-28603: Fix a TypeError that caused a shell restart when printing a traceback that includes an exception that is unhashable. Patch by Zane Bitter. diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 79eaeb7eb45bf4..447dafcc515e6c 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -79,9 +79,6 @@ def __init__(self, master, path, *, _htest=False, _utest=False): creating ModuleBrowserTreeItem as the rootnode for the tree and subsequently in the children. """ - global file_open - if not (_htest or _utest): - file_open = pyshell.flist.open self.master = master self.path = path self._htest = _htest @@ -95,9 +92,13 @@ def close(self, event=None): def init(self): "Create browser tkinter widgets, including the tree." + global file_open root = self.master - # reset pyclbr + flist = (pyshell.flist if not (self._htest or self._utest) + else pyshell.PyShellFileList(root)) + file_open = flist.open pyclbr._modules.clear() + # create top self.top = top = ListedToplevel(root) top.protocol("WM_DELETE_WINDOW", self.close) @@ -107,6 +108,7 @@ def init(self): (root.winfo_rootx(), root.winfo_rooty() + 200)) self.settitle() top.focus_set() + # create scrolled canvas theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] @@ -236,8 +238,6 @@ class Nested_in_func(TreeNode): def nested_in_class(): pass def closure(): class Nested_in_closure: pass - global file_open - file_open = pyshell.PyShellFileList(parent).open ModuleBrowser(parent, file, _htest=True) if __name__ == "__main__": diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 099f5262b1e4f5..4e8394be928112 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -11,8 +11,8 @@ """ from tkinter import (Toplevel, Listbox, Text, Scale, Canvas, StringVar, BooleanVar, IntVar, TRUE, FALSE, - TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NORMAL, DISABLED, - NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, CENTER, + TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, + NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) from tkinter.ttk import (Button, Checkbutton, Entry, Frame, Label, LabelFrame, OptionMenu, Notebook, Radiobutton, Scrollbar, Style) @@ -25,7 +25,6 @@ from idlelib.dynoption import DynOptionMenu from idlelib import macosx from idlelib.query import SectionName, HelpSource -from idlelib.tabbedpages import TabbedPageSet from idlelib.textview import view_text from idlelib.autocomplete import AutoComplete from idlelib.codecontext import CodeContext @@ -1443,7 +1442,7 @@ def create_page_keys(self): self.bindingslist['xscrollcommand'] = scroll_target_x.set self.button_new_keys = Button( frame_custom, text='Get New Keys for Selection', - command=self.get_new_keys, state=DISABLED) + command=self.get_new_keys, state='disabled') # frame_key_sets. frames = [Frame(frame_key_sets, padding=2, borderwidth=0) for i in range(2)] diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 68450c921f2fad..b51c45c97e50f2 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -668,7 +668,7 @@ def open_module_browser(self, event=None): def open_path_browser(self, event=None): from idlelib import pathbrowser - pathbrowser.PathBrowser(self.flist) + pathbrowser.PathBrowser(self.root) return "break" def open_turtle_demo(self, event = None): diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 59e03c5aab3c9e..34eb332c1df434 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -4,17 +4,19 @@ (Higher, because should exclude 3 lines that .coveragerc won't exclude.) """ +from collections import deque import os.path -import unittest import pyclbr +from tkinter import Tk -from idlelib import browser, filelist -from idlelib.tree import TreeNode from test.support import requires +import unittest from unittest import mock -from tkinter import Tk from idlelib.idle_test.mock_idle import Func -from collections import deque + +from idlelib import browser +from idlelib import filelist +from idlelib.tree import TreeNode class ModuleBrowserTest(unittest.TestCase): @@ -29,6 +31,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.mb.close() + cls.root.update_idletasks() cls.root.destroy() del cls.root, cls.mb @@ -38,6 +41,7 @@ def test_init(self): eq(mb.path, __file__) eq(pyclbr._modules, {}) self.assertIsInstance(mb.node, TreeNode) + self.assertIsNotNone(browser.file_open) def test_settitle(self): mb = self.mb @@ -151,10 +155,9 @@ def test_getsublist(self): self.assertEqual(sub0.name, 'f0') self.assertEqual(sub1.name, 'C0(base)') - - def test_ondoubleclick(self): + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): mbt = self.mbt - fopen = browser.file_open = mock.Mock() with mock.patch('os.path.exists', return_value=False): mbt.OnDoubleClick() @@ -165,8 +168,6 @@ def test_ondoubleclick(self): fopen.assert_called() fopen.called_with(fname) - del browser.file_open - class ChildBrowserTreeItemTest(unittest.TestCase): @@ -212,14 +213,13 @@ def test_getsublist(self): eq(self.cbt_F1.GetSubList(), []) - def test_ondoubleclick(self): - fopen = browser.file_open = mock.Mock() + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): goto = fopen.return_value.gotoline = mock.Mock() self.cbt_F1.OnDoubleClick() fopen.assert_called() goto.assert_called() goto.assert_called_with(self.cbt_F1.obj.lineno) - del browser.file_open # Failure test would have to raise OSError or AttributeError. diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py index 813cbcc63167cc..74b716a3199327 100644 --- a/Lib/idlelib/idle_test/test_pathbrowser.py +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -1,11 +1,68 @@ +""" Test idlelib.pathbrowser. +""" + + +import os.path +import pyclbr # for _modules +import sys # for sys.path +from tkinter import Tk + +from test.support import requires import unittest -import os -import sys -import idlelib +from idlelib.idle_test.mock_idle import Func + +import idlelib # for __file__ +from idlelib import browser from idlelib import pathbrowser +from idlelib.tree import TreeNode + class PathBrowserTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.pb = pathbrowser.PathBrowser(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.pb.close() + cls.root.update_idletasks() + cls.root.destroy() + del cls.root, cls.pb + + def test_init(self): + pb = self.pb + eq = self.assertEqual + eq(pb.master, self.root) + eq(pyclbr._modules, {}) + self.assertIsInstance(pb.node, TreeNode) + self.assertIsNotNone(browser.file_open) + + def test_settitle(self): + pb = self.pb + self.assertEqual(pb.top.title(), 'Path Browser') + self.assertEqual(pb.top.iconname(), 'Path Browser') + + def test_rootnode(self): + pb = self.pb + rn = pb.rootnode() + self.assertIsInstance(rn, pathbrowser.PathBrowserTreeItem) + + def test_close(self): + pb = self.pb + pb.top.destroy = Func() + pb.node.destroy = Func() + pb.close() + self.assertTrue(pb.top.destroy.called) + self.assertTrue(pb.node.destroy.called) + del pb.top.destroy, pb.node.destroy + + +class DirBrowserTreeItemTest(unittest.TestCase): + def test_DirBrowserTreeItem(self): # Issue16226 - make sure that getting a sublist works d = pathbrowser.DirBrowserTreeItem('') @@ -16,6 +73,9 @@ def test_DirBrowserTreeItem(self): self.assertEqual(d.ispackagedir(dir), True) self.assertEqual(d.ispackagedir(dir + '/Icons'), False) + +class PathBrowserTreeItemTest(unittest.TestCase): + def test_PathBrowserTreeItem(self): p = pathbrowser.PathBrowserTreeItem() self.assertEqual(p.GetText(), 'sys.path') @@ -23,5 +83,6 @@ def test_PathBrowserTreeItem(self): self.assertEqual(len(sub), len(sys.path)) self.assertEqual(type(sub[0]), pathbrowser.DirBrowserTreeItem) + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index c0aa2a1590916c..6de242d0000bed 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -3,19 +3,19 @@ import sys from idlelib.browser import ModuleBrowser, ModuleBrowserTreeItem -from idlelib.pyshell import PyShellFileList from idlelib.tree import TreeItem class PathBrowser(ModuleBrowser): - def __init__(self, flist, *, _htest=False, _utest=False): + def __init__(self, master, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ + self.master = master self._htest = _htest self._utest = _utest - self.init(flist) + self.init() def settitle(self): "Set window titles." @@ -100,8 +100,7 @@ def listmodules(self, allnames): def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) + PathBrowser(parent, _htest=True) parent.mainloop() if __name__ == "__main__": diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 6440e673bf5163..176fe3db743bd4 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -134,13 +134,17 @@ def main(del_exitfunc=False): # exiting but got an extra KBI? Try again! continue try: - seq, request = rpc.request_queue.get(block=True, timeout=0.05) + request = rpc.request_queue.get(block=True, timeout=0.05) except queue.Empty: + request = None + # Issue 32207: calling handle_tk_events here adds spurious + # queue.Empty traceback to event handling exceptions. + if request: + seq, (method, args, kwargs) = request + ret = method(*args, **kwargs) + rpc.response_queue.put((seq, ret)) + else: handle_tk_events() - continue - method, args, kwargs = request - ret = method(*args, **kwargs) - rpc.response_queue.put((seq, ret)) except KeyboardInterrupt: if quitting: exit_now = True diff --git a/Lib/idlelib/tabbedpages.py b/Lib/idlelib/tabbedpages.py deleted file mode 100644 index 4186fa201317ae..00000000000000 --- a/Lib/idlelib/tabbedpages.py +++ /dev/null @@ -1,498 +0,0 @@ -"""An implementation of tabbed pages using only standard Tkinter. - -Originally developed for use in IDLE. Based on tabpage.py. - -Classes exported: -TabbedPageSet -- A Tkinter implementation of a tabbed-page widget. -TabSet -- A widget containing tabs (buttons) in one or more rows. - -""" -from tkinter import * - -class InvalidNameError(Exception): pass -class AlreadyExistsError(Exception): pass - - -class TabSet(Frame): - """A widget containing tabs (buttons) in one or more rows. - - Only one tab may be selected at a time. - - """ - def __init__(self, page_set, select_command, - tabs=None, n_rows=1, max_tabs_per_row=5, - expand_tabs=False, **kw): - """Constructor arguments: - - select_command -- A callable which will be called when a tab is - selected. It is called with the name of the selected tab as an - argument. - - tabs -- A list of strings, the names of the tabs. Should be specified in - the desired tab order. The first tab will be the default and first - active tab. If tabs is None or empty, the TabSet will be initialized - empty. - - n_rows -- Number of rows of tabs to be shown. If n_rows <= 0 or is - None, then the number of rows will be decided by TabSet. See - _arrange_tabs() for details. - - max_tabs_per_row -- Used for deciding how many rows of tabs are needed, - when the number of rows is not constant. See _arrange_tabs() for - details. - - """ - Frame.__init__(self, page_set, **kw) - self.select_command = select_command - self.n_rows = n_rows - self.max_tabs_per_row = max_tabs_per_row - self.expand_tabs = expand_tabs - self.page_set = page_set - - self._tabs = {} - self._tab2row = {} - if tabs: - self._tab_names = list(tabs) - else: - self._tab_names = [] - self._selected_tab = None - self._tab_rows = [] - - self.padding_frame = Frame(self, height=2, - borderwidth=0, relief=FLAT, - background=self.cget('background')) - self.padding_frame.pack(side=TOP, fill=X, expand=False) - - self._arrange_tabs() - - def add_tab(self, tab_name): - """Add a new tab with the name given in tab_name.""" - if not tab_name: - raise InvalidNameError("Invalid Tab name: '%s'" % tab_name) - if tab_name in self._tab_names: - raise AlreadyExistsError("Tab named '%s' already exists" %tab_name) - - self._tab_names.append(tab_name) - self._arrange_tabs() - - def remove_tab(self, tab_name): - """Remove the tab named """ - if not tab_name in self._tab_names: - raise KeyError("No such Tab: '%s" % tab_name) - - self._tab_names.remove(tab_name) - self._arrange_tabs() - - def set_selected_tab(self, tab_name): - """Show the tab named as the selected one""" - if tab_name == self._selected_tab: - return - if tab_name is not None and tab_name not in self._tabs: - raise KeyError("No such Tab: '%s" % tab_name) - - # deselect the current selected tab - if self._selected_tab is not None: - self._tabs[self._selected_tab].set_normal() - self._selected_tab = None - - if tab_name is not None: - # activate the tab named tab_name - self._selected_tab = tab_name - tab = self._tabs[tab_name] - tab.set_selected() - # move the tab row with the selected tab to the bottom - tab_row = self._tab2row[tab] - tab_row.pack_forget() - tab_row.pack(side=TOP, fill=X, expand=0) - - def _add_tab_row(self, tab_names, expand_tabs): - if not tab_names: - return - - tab_row = Frame(self) - tab_row.pack(side=TOP, fill=X, expand=0) - self._tab_rows.append(tab_row) - - for tab_name in tab_names: - tab = TabSet.TabButton(tab_name, self.select_command, - tab_row, self) - if expand_tabs: - tab.pack(side=LEFT, fill=X, expand=True) - else: - tab.pack(side=LEFT) - self._tabs[tab_name] = tab - self._tab2row[tab] = tab_row - - # tab is the last one created in the above loop - tab.is_last_in_row = True - - def _reset_tab_rows(self): - while self._tab_rows: - tab_row = self._tab_rows.pop() - tab_row.destroy() - self._tab2row = {} - - def _arrange_tabs(self): - """ - Arrange the tabs in rows, in the order in which they were added. - - If n_rows >= 1, this will be the number of rows used. Otherwise the - number of rows will be calculated according to the number of tabs and - max_tabs_per_row. In this case, the number of rows may change when - adding/removing tabs. - - """ - # remove all tabs and rows - while self._tabs: - self._tabs.popitem()[1].destroy() - self._reset_tab_rows() - - if not self._tab_names: - return - - if self.n_rows is not None and self.n_rows > 0: - n_rows = self.n_rows - else: - # calculate the required number of rows - n_rows = (len(self._tab_names) - 1) // self.max_tabs_per_row + 1 - - # not expanding the tabs with more than one row is very ugly - expand_tabs = self.expand_tabs or n_rows > 1 - i = 0 # index in self._tab_names - for row_index in range(n_rows): - # calculate required number of tabs in this row - n_tabs = (len(self._tab_names) - i - 1) // (n_rows - row_index) + 1 - tab_names = self._tab_names[i:i + n_tabs] - i += n_tabs - self._add_tab_row(tab_names, expand_tabs) - - # re-select selected tab so it is properly displayed - selected = self._selected_tab - self.set_selected_tab(None) - if selected in self._tab_names: - self.set_selected_tab(selected) - - class TabButton(Frame): - """A simple tab-like widget.""" - - bw = 2 # borderwidth - - def __init__(self, name, select_command, tab_row, tab_set): - """Constructor arguments: - - name -- The tab's name, which will appear in its button. - - select_command -- The command to be called upon selection of the - tab. It is called with the tab's name as an argument. - - """ - Frame.__init__(self, tab_row, borderwidth=self.bw, relief=RAISED) - - self.name = name - self.select_command = select_command - self.tab_set = tab_set - self.is_last_in_row = False - - self.button = Radiobutton( - self, text=name, command=self._select_event, - padx=5, pady=1, takefocus=FALSE, indicatoron=FALSE, - highlightthickness=0, selectcolor='', borderwidth=0) - self.button.pack(side=LEFT, fill=X, expand=True) - - self._init_masks() - self.set_normal() - - def _select_event(self, *args): - """Event handler for tab selection. - - With TabbedPageSet, this calls TabbedPageSet.change_page, so that - selecting a tab changes the page. - - Note that this does -not- call set_selected -- it will be called by - TabSet.set_selected_tab, which should be called when whatever the - tabs are related to changes. - - """ - self.select_command(self.name) - return - - def set_selected(self): - """Assume selected look""" - self._place_masks(selected=True) - - def set_normal(self): - """Assume normal look""" - self._place_masks(selected=False) - - def _init_masks(self): - page_set = self.tab_set.page_set - background = page_set.pages_frame.cget('background') - # mask replaces the middle of the border with the background color - self.mask = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - # mskl replaces the bottom-left corner of the border with a normal - # left border - self.mskl = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - self.mskl.ml = Frame(self.mskl, borderwidth=self.bw, - relief=RAISED) - self.mskl.ml.place(x=0, y=-self.bw, - width=2*self.bw, height=self.bw*4) - # mskr replaces the bottom-right corner of the border with a normal - # right border - self.mskr = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - self.mskr.mr = Frame(self.mskr, borderwidth=self.bw, - relief=RAISED) - - def _place_masks(self, selected=False): - height = self.bw - if selected: - height += self.bw - - self.mask.place(in_=self, - relx=0.0, x=0, - rely=1.0, y=0, - relwidth=1.0, width=0, - relheight=0.0, height=height) - - self.mskl.place(in_=self, - relx=0.0, x=-self.bw, - rely=1.0, y=0, - relwidth=0.0, width=self.bw, - relheight=0.0, height=height) - - page_set = self.tab_set.page_set - if selected and ((not self.is_last_in_row) or - (self.winfo_rootx() + self.winfo_width() < - page_set.winfo_rootx() + page_set.winfo_width()) - ): - # for a selected tab, if its rightmost edge isn't on the - # rightmost edge of the page set, the right mask should be one - # borderwidth shorter (vertically) - height -= self.bw - - self.mskr.place(in_=self, - relx=1.0, x=0, - rely=1.0, y=0, - relwidth=0.0, width=self.bw, - relheight=0.0, height=height) - - self.mskr.mr.place(x=-self.bw, y=-self.bw, - width=2*self.bw, height=height + self.bw*2) - - # finally, lower the tab set so that all of the frames we just - # placed hide it - self.tab_set.lower() - - -class TabbedPageSet(Frame): - """A Tkinter tabbed-pane widget. - - Constains set of 'pages' (or 'panes') with tabs above for selecting which - page is displayed. Only one page will be displayed at a time. - - Pages may be accessed through the 'pages' attribute, which is a dictionary - of pages, using the name given as the key. A page is an instance of a - subclass of Tk's Frame widget. - - The page widgets will be created (and destroyed when required) by the - TabbedPageSet. Do not call the page's pack/place/grid/destroy methods. - - Pages may be added or removed at any time using the add_page() and - remove_page() methods. - - """ - - class Page(object): - """Abstract base class for TabbedPageSet's pages. - - Subclasses must override the _show() and _hide() methods. - - """ - uses_grid = False - - def __init__(self, page_set): - self.frame = Frame(page_set, borderwidth=2, relief=RAISED) - - def _show(self): - raise NotImplementedError - - def _hide(self): - raise NotImplementedError - - class PageRemove(Page): - """Page class using the grid placement manager's "remove" mechanism.""" - uses_grid = True - - def _show(self): - self.frame.grid(row=0, column=0, sticky=NSEW) - - def _hide(self): - self.frame.grid_remove() - - class PageLift(Page): - """Page class using the grid placement manager's "lift" mechanism.""" - uses_grid = True - - def __init__(self, page_set): - super(TabbedPageSet.PageLift, self).__init__(page_set) - self.frame.grid(row=0, column=0, sticky=NSEW) - self.frame.lower() - - def _show(self): - self.frame.lift() - - def _hide(self): - self.frame.lower() - - class PagePackForget(Page): - """Page class using the pack placement manager's "forget" mechanism.""" - def _show(self): - self.frame.pack(fill=BOTH, expand=True) - - def _hide(self): - self.frame.pack_forget() - - def __init__(self, parent, page_names=None, page_class=PageLift, - n_rows=1, max_tabs_per_row=5, expand_tabs=False, - **kw): - """Constructor arguments: - - page_names -- A list of strings, each will be the dictionary key to a - page's widget, and the name displayed on the page's tab. Should be - specified in the desired page order. The first page will be the default - and first active page. If page_names is None or empty, the - TabbedPageSet will be initialized empty. - - n_rows, max_tabs_per_row -- Parameters for the TabSet which will - manage the tabs. See TabSet's docs for details. - - page_class -- Pages can be shown/hidden using three mechanisms: - - * PageLift - All pages will be rendered one on top of the other. When - a page is selected, it will be brought to the top, thus hiding all - other pages. Using this method, the TabbedPageSet will not be resized - when pages are switched. (It may still be resized when pages are - added/removed.) - - * PageRemove - When a page is selected, the currently showing page is - hidden, and the new page shown in its place. Using this method, the - TabbedPageSet may resize when pages are changed. - - * PagePackForget - This mechanism uses the pack placement manager. - When a page is shown it is packed, and when it is hidden it is - unpacked (i.e. pack_forget). This mechanism may also cause the - TabbedPageSet to resize when the page is changed. - - """ - Frame.__init__(self, parent, **kw) - - self.page_class = page_class - self.pages = {} - self._pages_order = [] - self._current_page = None - self._default_page = None - - self.columnconfigure(0, weight=1) - self.rowconfigure(1, weight=1) - - self.pages_frame = Frame(self) - self.pages_frame.grid(row=1, column=0, sticky=NSEW) - if self.page_class.uses_grid: - self.pages_frame.columnconfigure(0, weight=1) - self.pages_frame.rowconfigure(0, weight=1) - - # the order of the following commands is important - self._tab_set = TabSet(self, self.change_page, n_rows=n_rows, - max_tabs_per_row=max_tabs_per_row, - expand_tabs=expand_tabs) - if page_names: - for name in page_names: - self.add_page(name) - self._tab_set.grid(row=0, column=0, sticky=NSEW) - - self.change_page(self._default_page) - - def add_page(self, page_name): - """Add a new page with the name given in page_name.""" - if not page_name: - raise InvalidNameError("Invalid TabPage name: '%s'" % page_name) - if page_name in self.pages: - raise AlreadyExistsError( - "TabPage named '%s' already exists" % page_name) - - self.pages[page_name] = self.page_class(self.pages_frame) - self._pages_order.append(page_name) - self._tab_set.add_tab(page_name) - - if len(self.pages) == 1: # adding first page - self._default_page = page_name - self.change_page(page_name) - - def remove_page(self, page_name): - """Destroy the page whose name is given in page_name.""" - if not page_name in self.pages: - raise KeyError("No such TabPage: '%s" % page_name) - - self._pages_order.remove(page_name) - - # handle removing last remaining, default, or currently shown page - if len(self._pages_order) > 0: - if page_name == self._default_page: - # set a new default page - self._default_page = self._pages_order[0] - else: - self._default_page = None - - if page_name == self._current_page: - self.change_page(self._default_page) - - self._tab_set.remove_tab(page_name) - page = self.pages.pop(page_name) - page.frame.destroy() - - def change_page(self, page_name): - """Show the page whose name is given in page_name.""" - if self._current_page == page_name: - return - if page_name is not None and page_name not in self.pages: - raise KeyError("No such TabPage: '%s'" % page_name) - - if self._current_page is not None: - self.pages[self._current_page]._hide() - self._current_page = None - - if page_name is not None: - self._current_page = page_name - self.pages[page_name]._show() - - self._tab_set.set_selected_tab(page_name) - - -def _tabbed_pages(parent): # htest # - top=Toplevel(parent) - x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("+%d+%d" % (x, y + 175)) - top.title("Test tabbed pages") - tabPage=TabbedPageSet(top, page_names=['Foobar','Baz'], n_rows=0, - expand_tabs=False, - ) - tabPage.pack(side=TOP, expand=TRUE, fill=BOTH) - Label(tabPage.pages['Foobar'].frame, text='Foo', pady=20).pack() - Label(tabPage.pages['Foobar'].frame, text='Bar', pady=20).pack() - Label(tabPage.pages['Baz'].frame, text='Baz').pack() - entryPgName=Entry(top) - buttonAdd=Button(top, text='Add Page', - command=lambda:tabPage.add_page(entryPgName.get())) - buttonRemove=Button(top, text='Remove Page', - command=lambda:tabPage.remove_page(entryPgName.get())) - labelPgName=Label(top, text='name of page to add/remove:') - buttonAdd.pack(padx=5, pady=5) - buttonRemove.pack(padx=5, pady=5) - labelPgName.pack(padx=5) - entryPgName.pack(padx=5) - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tabbed_pages) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 1c0b03bff8a379..e1cece0b283f2b 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1166,7 +1166,7 @@ def _match(self, cre, s): self.mo = cre.match(s) if __debug__: if self.mo is not None and self.debug >= 5: - self._mesg("\tmatched r'%r' => %r" % (cre.pattern, self.mo.groups())) + self._mesg("\tmatched %r => %r" % (cre.pattern, self.mo.groups())) return self.mo is not None diff --git a/Lib/inspect.py b/Lib/inspect.py index 6d6fde9ee40401..8c121ce96c4158 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1381,7 +1381,7 @@ def getclosurevars(func): func = func.__func__ if not isfunction(func): - raise TypeError("'{!r}' is not a Python function".format(func)) + raise TypeError("{!r} is not a Python function".format(func)) code = func.__code__ # Nonlocal references are named in co_freevars and resolved @@ -1624,7 +1624,7 @@ def getgeneratorlocals(generator): bound values.""" if not isgenerator(generator): - raise TypeError("'{!r}' is not a Python generator".format(generator)) + raise TypeError("{!r} is not a Python generator".format(generator)) frame = getattr(generator, "gi_frame", None) if frame is not None: @@ -2521,11 +2521,14 @@ def __str__(self): # Add annotation and default value if self._annotation is not _empty: - formatted = '{}:{}'.format(formatted, + formatted = '{}: {}'.format(formatted, formatannotation(self._annotation)) if self._default is not _empty: - formatted = '{}={}'.format(formatted, repr(self._default)) + if self._annotation is not _empty: + formatted = '{} = {}'.format(formatted, repr(self._default)) + else: + formatted = '{}={}'.format(formatted, repr(self._default)) if kind == _VAR_POSITIONAL: formatted = '*' + formatted diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 6d0511ebfe90cf..a5660099af7514 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -76,7 +76,8 @@ >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... raise TypeError(repr(obj) + " is not JSON serializable") + ... raise TypeError(f'Object of type {obj.__class__.__name__} ' + ... f'is not JSON serializable') ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' @@ -344,8 +345,8 @@ def loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, s, 0) else: if not isinstance(s, (bytes, bytearray)): - raise TypeError('the JSON object must be str, bytes or bytearray, ' - 'not {!r}'.format(s.__class__.__name__)) + raise TypeError(f'the JSON object must be str, bytes or bytearray, ' + f'not {s.__class__.__name__}') s = s.decode(detect_encoding(s), 'surrogatepass') if (cls is None and object_hook is None and diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 41a497c5da0160..fb083ed61bb1f8 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -176,8 +176,8 @@ def default(self, o): return JSONEncoder.default(self, o) """ - raise TypeError("Object of type '%s' is not JSON serializable" % - o.__class__.__name__) + raise TypeError(f'Object of type {o.__class__.__name__} ' + f'is not JSON serializable') def encode(self, o): """Return a JSON string representation of a Python data structure. @@ -373,7 +373,8 @@ def _iterencode_dict(dct, _current_indent_level): elif _skipkeys: continue else: - raise TypeError("key " + repr(key) + " is not a string") + raise TypeError(f'keys must be str, int, float, bool or None, ' + f'not {key.__class__.__name__}') if first: first = False else: diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py index 592444e2580451..d303cd2018befb 100644 --- a/Lib/lib2to3/fixes/fix_operator.py +++ b/Lib/lib2to3/fixes/fix_operator.py @@ -1,9 +1,9 @@ """Fixer for operator functions. -operator.isCallable(obj) -> hasattr(obj, '__call__') +operator.isCallable(obj) -> callable(obj) operator.sequenceIncludes(obj) -> operator.contains(obj) -operator.isSequenceType(obj) -> isinstance(obj, collections.Sequence) -operator.isMappingType(obj) -> isinstance(obj, collections.Mapping) +operator.isSequenceType(obj) -> isinstance(obj, collections.abc.Sequence) +operator.isMappingType(obj) -> isinstance(obj, collections.abc.Mapping) operator.isNumberType(obj) -> isinstance(obj, numbers.Number) operator.repeat(obj, n) -> operator.mul(obj, n) operator.irepeat(obj, n) -> operator.imul(obj, n) @@ -49,11 +49,10 @@ def transform(self, node, results): def _sequenceIncludes(self, node, results): return self._handle_rename(node, results, "contains") - @invocation("hasattr(%s, '__call__')") + @invocation("callable(%s)") def _isCallable(self, node, results): obj = results["obj"] - args = [obj.clone(), String(", "), String("'__call__'")] - return Call(Name("hasattr"), args, prefix=node.prefix) + return Call(Name("callable"), [obj.clone()], prefix=node.prefix) @invocation("operator.mul(%s)") def _repeat(self, node, results): @@ -63,13 +62,13 @@ def _repeat(self, node, results): def _irepeat(self, node, results): return self._handle_rename(node, results, "imul") - @invocation("isinstance(%s, collections.Sequence)") + @invocation("isinstance(%s, collections.abc.Sequence)") def _isSequenceType(self, node, results): - return self._handle_type2abc(node, results, "collections", "Sequence") + return self._handle_type2abc(node, results, "collections.abc", "Sequence") - @invocation("isinstance(%s, collections.Mapping)") + @invocation("isinstance(%s, collections.abc.Mapping)") def _isMappingType(self, node, results): - return self._handle_type2abc(node, results, "collections", "Mapping") + return self._handle_type2abc(node, results, "collections.abc", "Mapping") @invocation("isinstance(%s, numbers.Number)") def _isNumberType(self, node, results): diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 3e1a255737ec4b..bfe7a23e70068a 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -4409,7 +4409,7 @@ class Test_operator(FixerTestCase): def test_operator_isCallable(self): b = "operator.isCallable(x)" - a = "hasattr(x, '__call__')" + a = "callable(x)" self.check(b, a) def test_operator_sequenceIncludes(self): @@ -4427,12 +4427,12 @@ def test_operator_sequenceIncludes(self): def test_operator_isSequenceType(self): b = "operator.isSequenceType(x)" - a = "import collections\nisinstance(x, collections.Sequence)" + a = "import collections.abc\nisinstance(x, collections.abc.Sequence)" self.check(b, a) def test_operator_isMappingType(self): b = "operator.isMappingType(x)" - a = "import collections\nisinstance(x, collections.Mapping)" + a = "import collections.abc\nisinstance(x, collections.abc.Mapping)" self.check(b, a) def test_operator_isNumberType(self): @@ -4468,7 +4468,7 @@ def test_operator_irepeat(self): def test_bare_isCallable(self): s = "isCallable(x)" - t = "You should use 'hasattr(x, '__call__')' here." + t = "You should use 'callable(x)' here." self.warns_unchanged(s, t) def test_bare_sequenceIncludes(self): @@ -4478,12 +4478,12 @@ def test_bare_sequenceIncludes(self): def test_bare_operator_isSequenceType(self): s = "isSequenceType(z)" - t = "You should use 'isinstance(z, collections.Sequence)' here." + t = "You should use 'isinstance(z, collections.abc.Sequence)' here." self.warns_unchanged(s, t) def test_bare_operator_isMappingType(self): s = "isMappingType(x)" - t = "You should use 'isinstance(x, collections.Mapping)' here." + t = "You should use 'isinstance(x, collections.abc.Mapping)' here." self.warns_unchanged(s, t) def test_bare_operator_isNumberType(self): diff --git a/Lib/logging/config.py b/Lib/logging/config.py index b08cba0687563d..7927e7618777e5 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -97,7 +97,7 @@ def _resolve(name): return found def _strip_spaces(alist): - return map(lambda x: x.strip(), alist) + return map(str.strip, alist) def _create_formatters(cp): """Create and return formatters""" @@ -185,7 +185,7 @@ def _install_loggers(cp, handlers, disable_existing): # configure the root first llist = cp["loggers"]["keys"] llist = llist.split(",") - llist = list(map(lambda x: x.strip(), llist)) + llist = list(_strip_spaces(llist)) llist.remove("root") section = cp["logger_root"] root = logging.root diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index d1871acfa4ba59..974c089d40ec34 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1180,7 +1180,9 @@ def emit(self, record): i = host.find(":") if i >= 0: host = host[:i] - h.putheader("Host", host) + # See issue #30904: putrequest call above already adds this header + # on Python 3.x. + # h.putheader("Host", host) if self.method == "POST": h.putheader("Content-type", "application/x-www-form-urlencoded") diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 5919b45a9b4f5f..44112027cb9aae 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -412,8 +412,6 @@ def _default_mime_types(): '.bin' : 'application/octet-stream', '.bmp' : 'image/x-ms-bmp', '.c' : 'text/plain', - # Duplicates :( - '.cdf' : 'application/x-cdf', '.cdf' : 'application/x-netcdf', '.cpio' : 'application/x-cpio', '.csh' : 'application/x-csh', @@ -522,8 +520,6 @@ def _default_mime_types(): '.wsdl' : 'application/xml', '.xbm' : 'image/x-xbitmap', '.xlb' : 'application/vnd.ms-excel', - # Duplicates :( - '.xls' : 'application/excel', '.xls' : 'application/vnd.ms-excel', '.xml' : 'text/xml', '.xpdl' : 'application/xml', diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py index 066cf8f031ddeb..6071707027bea4 100644 --- a/Lib/multiprocessing/sharedctypes.py +++ b/Lib/multiprocessing/sharedctypes.py @@ -78,7 +78,7 @@ def Value(typecode_or_type, *args, lock=True, ctx=None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): - raise AttributeError("'%r' has no method 'acquire'" % lock) + raise AttributeError("%r has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): @@ -92,7 +92,7 @@ def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): - raise AttributeError("'%r' has no method 'acquire'" % lock) + raise AttributeError("%r has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def copy(obj): diff --git a/Lib/netrc.py b/Lib/netrc.py index baf8f1d99d9d30..f0ae48cfed9e67 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -23,10 +23,7 @@ class netrc: def __init__(self, file=None): default_netrc = file is None if file is None: - try: - file = os.path.join(os.environ['HOME'], ".netrc") - except KeyError: - raise OSError("Could not find .netrc: $HOME is not set") from None + file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} with open(file) as fp: diff --git a/Lib/pickle.py b/Lib/pickle.py index faa8fd7e557f9c..350d4a46c06a43 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -674,7 +674,10 @@ def save_long(self, obj): else: self.write(LONG4 + pack(">> dis(pickle.dumps(x, 1)) diff --git a/Lib/platform.py b/Lib/platform.py index cc2db9870d84b6..dc981ec144cc07 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1202,9 +1202,6 @@ def _sys_version(sys_version=None): _, branch, revision = sys._git elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial - elif hasattr(sys, 'subversion'): - # sys.subversion was added in Python 2.5 - _, branch, revision = sys.subversion else: branch = '' revision = '' @@ -1259,7 +1256,7 @@ def python_branch(): """ Returns a string identifying the Python implementation branch. - For CPython this is the Subversion branch from which the + For CPython this is the SCM branch from which the Python binary was built. If not available, an empty string is returned. @@ -1273,7 +1270,7 @@ def python_revision(): """ Returns a string identifying the Python implementation revision. - For CPython this is the Subversion revision from which the + For CPython this is the SCM revision from which the Python binary was built. If not available, an empty string is returned. diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 2113a2dc57b4e1..21ebec3f004590 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -525,6 +525,8 @@ def __init__(self, message="Invalid file"): _BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'} +_undefined = object() + class _BinaryPlistParser: """ Read or write a binary plist file, following the description of the binary @@ -555,7 +557,8 @@ def parse(self, fp): ) = struct.unpack('>6xBBQQQ', trailer) self._fp.seek(offset_table_offset) self._object_offsets = self._read_ints(num_objects, offset_size) - return self._read_object(self._object_offsets[top_object]) + self._objects = [_undefined] * num_objects + return self._read_object(top_object) except (OSError, IndexError, struct.error, OverflowError, UnicodeDecodeError): @@ -584,62 +587,68 @@ def _read_ints(self, n, size): def _read_refs(self, n): return self._read_ints(n, self._ref_size) - def _read_object(self, offset): + def _read_object(self, ref): """ - read the object at offset. + read the object by reference. May recursively read sub-objects (content of an array/dict/set) """ + result = self._objects[ref] + if result is not _undefined: + return result + + offset = self._object_offsets[ref] self._fp.seek(offset) token = self._fp.read(1)[0] tokenH, tokenL = token & 0xF0, token & 0x0F if token == 0x00: - return None + result = None elif token == 0x08: - return False + result = False elif token == 0x09: - return True + result = True # The referenced source code also mentions URL (0x0c, 0x0d) and # UUID (0x0e), but neither can be generated using the Cocoa libraries. elif token == 0x0f: - return b'' + result = b'' elif tokenH == 0x10: # int - return int.from_bytes(self._fp.read(1 << tokenL), - 'big', signed=tokenL >= 3) + result = int.from_bytes(self._fp.read(1 << tokenL), + 'big', signed=tokenL >= 3) elif token == 0x22: # real - return struct.unpack('>f', self._fp.read(4))[0] + result = struct.unpack('>f', self._fp.read(4))[0] elif token == 0x23: # real - return struct.unpack('>d', self._fp.read(8))[0] + result = struct.unpack('>d', self._fp.read(8))[0] elif token == 0x33: # date f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - return datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f) + result = (datetime.datetime(2001, 1, 1) + + datetime.timedelta(seconds=f)) elif tokenH == 0x40: # data s = self._get_size(tokenL) if self._use_builtin_types: - return self._fp.read(s) + result = self._fp.read(s) else: - return Data(self._fp.read(s)) + result = Data(self._fp.read(s)) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) result = self._fp.read(s).decode('ascii') - return result + result = result elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) - return self._fp.read(s * 2).decode('utf-16be') + result = self._fp.read(s * 2).decode('utf-16be') # tokenH == 0x80 is documented as 'UID' and appears to be used for # keyed-archiving, not in plists. @@ -647,8 +656,9 @@ def _read_object(self, offset): elif tokenH == 0xA0: # array s = self._get_size(tokenL) obj_refs = self._read_refs(s) - return [self._read_object(self._object_offsets[x]) - for x in obj_refs] + result = [] + self._objects[ref] = result + result.extend(self._read_object(x) for x in obj_refs) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -661,12 +671,15 @@ def _read_object(self, offset): key_refs = self._read_refs(s) obj_refs = self._read_refs(s) result = self._dict_type() + self._objects[ref] = result for k, o in zip(key_refs, obj_refs): - result[self._read_object(self._object_offsets[k]) - ] = self._read_object(self._object_offsets[o]) - return result + result[self._read_object(k)] = self._read_object(o) - raise InvalidFileException() + else: + raise InvalidFileException() + + self._objects[ref] = result + return result def _count_to_size(count): if count < 1 << 8: @@ -681,6 +694,8 @@ def _count_to_size(count): else: return 8 +_scalars = (str, int, float, datetime.datetime, bytes) + class _BinaryPlistWriter (object): def __init__(self, fp, sort_keys, skipkeys): self._fp = fp @@ -736,8 +751,7 @@ def _flatten(self, value): # First check if the object is in the object table, not used for # containers to ensure that two subcontainers with the same contents # will be serialized as distinct values. - if isinstance(value, ( - str, int, float, datetime.datetime, bytes, bytearray)): + if isinstance(value, _scalars): if (type(value), value) in self._objtable: return @@ -745,15 +759,17 @@ def _flatten(self, value): if (type(value.data), value.data) in self._objtable: return + elif id(value) in self._objidtable: + return + # Add to objectreference map refnum = len(self._objlist) self._objlist.append(value) - try: - if isinstance(value, Data): - self._objtable[(type(value.data), value.data)] = refnum - else: - self._objtable[(type(value), value)] = refnum - except TypeError: + if isinstance(value, _scalars): + self._objtable[(type(value), value)] = refnum + elif isinstance(value, Data): + self._objtable[(type(value.data), value.data)] = refnum + else: self._objidtable[id(value)] = refnum # And finally recurse into containers @@ -780,12 +796,11 @@ def _flatten(self, value): self._flatten(o) def _getrefnum(self, value): - try: - if isinstance(value, Data): - return self._objtable[(type(value.data), value.data)] - else: - return self._objtable[(type(value), value)] - except TypeError: + if isinstance(value, _scalars): + return self._objtable[(type(value), value)] + elif isinstance(value, Data): + return self._objtable[(type(value.data), value.data)] + else: return self._objidtable[id(value)] def _write_size(self, token, size): diff --git a/Lib/re.py b/Lib/re.py index abbf8d6e290e54..a8b6753d390964 100644 --- a/Lib/re.py +++ b/Lib/re.py @@ -251,8 +251,9 @@ def template(pattern, flags=0): # SPECIAL_CHARS # closing ')', '}' and ']' # '-' (a range in character set) +# '&', '~', (extended character set operations) # '#' (comment) and WHITESPACE (ignored) in verbose mode -_special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-|^$\\.# \t\n\r\v\f'} +_special_chars_map = {i: '\\' + chr(i) for i in b'()[]{}?*+-|^$\\.&~# \t\n\r\v\f'} def escape(pattern): """ diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index 85274122938bed..a53735b07ded42 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -517,6 +517,12 @@ def _parse(source, state, verbose, nested, first=False): setappend = set.append ## if sourcematch(":"): ## pass # handle character classes + if source.next == '[': + import warnings + warnings.warn( + 'Possible nested set at position %d' % source.tell(), + FutureWarning, stacklevel=nested + 6 + ) negate = sourcematch("^") # check remaining characters while True: @@ -529,6 +535,17 @@ def _parse(source, state, verbose, nested, first=False): elif this[0] == "\\": code1 = _class_escape(source, this) else: + if set and this in '-&~|' and source.next == this: + import warnings + warnings.warn( + 'Possible set %s at position %d' % ( + 'difference' if this == '-' else + 'intersection' if this == '&' else + 'symmetric difference' if this == '~' else + 'union', + source.tell() - 1), + FutureWarning, stacklevel=nested + 6 + ) code1 = LITERAL, _ord(this) if sourcematch("-"): # potential range @@ -545,6 +562,13 @@ def _parse(source, state, verbose, nested, first=False): if that[0] == "\\": code2 = _class_escape(source, that) else: + if that == '-': + import warnings + warnings.warn( + 'Possible set difference at position %d' % ( + source.tell() - 2), + FutureWarning, stacklevel=nested + 6 + ) code2 = LITERAL, _ord(that) if code1[0] != LITERAL or code2[0] != LITERAL: msg = "bad character range %s-%s" % (this, that) diff --git a/Lib/ssl.py b/Lib/ssl.py index 75caae0c440566..fa83606e7cd5a5 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -221,7 +221,7 @@ class CertificateError(ValueError): pass -def _dnsname_match(dn, hostname, max_wildcards=1): +def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 @@ -233,7 +233,12 @@ def _dnsname_match(dn, hostname, max_wildcards=1): leftmost, *remainder = dn.split(r'.') wildcards = leftmost.count('*') - if wildcards > max_wildcards: + if wildcards == 1 and len(leftmost) > 1: + # Only match wildcard in leftmost segment. + raise CertificateError( + "wildcard can only be present in the leftmost segment: " + repr(dn)) + + if wildcards > 1: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a diff --git a/Lib/string.py b/Lib/string.py index a3e6d91bb4a78c..fd4b1f7a62f10d 100644 --- a/Lib/string.py +++ b/Lib/string.py @@ -79,11 +79,14 @@ class Template(metaclass=_TemplateMetaclass): """A string class for supporting $-substitutions.""" delimiter = '$' - # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, - # but without ASCII flag. We can't add re.ASCII to flags because of - # backward compatibility. So we use local -i flag and [a-zA-Z] pattern. + # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, but + # without the ASCII flag. We can't add re.ASCII to flags because of + # backward compatibility. So we use the ?a local flag and [a-z] pattern. + # We also can't remove the A-Z ranges, because although they are + # technically redundant with the IGNORECASE flag, the value is part of the + # publicly documented API. # See https://bugs.python.org/issue31672 - idpattern = r'(?-i:[_a-zA-Z][_a-zA-Z0-9]*)' + idpattern = r'(?a:[_a-zA-Z][_a-zA-Z0-9]*)' braceidpattern = None flags = _re.IGNORECASE diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 43be1f9bffa387..35bfddde4e92f1 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -260,8 +260,25 @@ def _args_from_interpreter_flags(): v = getattr(sys.flags, flag) if v > 0: args.append('-' + opt * v) + + # -W options for opt in sys.warnoptions: args.append('-W' + opt) + + # -X options + xoptions = getattr(sys, '_xoptions', {}) + if 'dev' in xoptions: + args.extend(('-X', 'dev')) + for opt in ('faulthandler', 'tracemalloc', 'importtime', + 'showalloccount', 'showrefcount'): + if opt in xoptions: + value = xoptions[opt] + if value is True: + arg = opt + else: + arg = '%s=%s' % (opt, value) + args.extend(('-X', arg)) + return args diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5b58e99fbcd357..d0886c47baec17 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -3247,8 +3247,6 @@ def test_extract(self): self.assertEqual(dt.timetz(), time(18, 45, 3, 1234, tzinfo=met)) def test_tz_aware_arithmetic(self): - import random - now = self.theclass.now() tz55 = FixedOffset(-330, "west 5:30") timeaware = now.time().replace(tzinfo=tz55) diff --git a/Lib/test/eintrdata/eintr_tester.py b/Lib/test/eintrdata/eintr_tester.py index 1dbe88efe70ffd..bc308fe107b311 100644 --- a/Lib/test/eintrdata/eintr_tester.py +++ b/Lib/test/eintrdata/eintr_tester.py @@ -20,7 +20,6 @@ import unittest from test import support -android_not_root = support.android_not_root @contextlib.contextmanager def kill_on_error(proc): @@ -312,14 +311,16 @@ def test_accept(self): # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=203162 @support.requires_freebsd_version(10, 3) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'needs mkfifo()') - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def _test_open(self, do_open_close_reader, do_open_close_writer): filename = support.TESTFN # Use a fifo: until the child opens it for reading, the parent will # block when trying to open it for writing. support.unlink(filename) - os.mkfifo(filename) + try: + os.mkfifo(filename) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) self.addCleanup(support.unlink, filename) code = '\n'.join(( diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9871a28dbf2d2b..ce01c8ce586d65 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -257,12 +257,12 @@ def _list_cases(self, suite): if isinstance(test, unittest.TestSuite): self._list_cases(test) elif isinstance(test, unittest.TestCase): - if support._match_test(test): + if support.match_test(test): print(test.id()) def list_cases(self): support.verbose = False - support.match_tests = self.ns.match_tests + support.set_match_tests(self.ns.match_tests) for test in self.selected: abstest = get_abs_module(self.ns, test) diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dbd463435c781b..12bf422c902dc1 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -102,7 +102,7 @@ def runtest(ns, test): if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) try: - support.match_tests = ns.match_tests + support.set_match_tests(ns.match_tests) # reset the environment_altered flag to detect if a test altered # the environment support.environment_altered = False diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index ce9db9a1b8bec9..ed63fda20c1b7a 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -53,10 +53,11 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") - l0 = [] + def test_repr_deep(self): + a = self.type2test([]) for i in range(sys.getrecursionlimit() + 100): - l0 = [l0] - self.assertRaises(RecursionError, repr, l0) + a = self.type2test([a]) + self.assertRaises(RecursionError, repr, a) def test_print(self): d = self.type2test(range(200)) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index ff82f4eb7d8988..53f29f605386ba 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,6 +1,7 @@ # tests common to dict and UserDict import unittest import collections +import sys class BasicTestMappingProtocol(unittest.TestCase): @@ -619,6 +620,14 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + def test_repr_deep(self): + d = self._empty_mapping() + for i in range(sys.getrecursionlimit() + 100): + d0 = d + d = self._empty_mapping() + d[1] = d0 + self.assertRaises(RecursionError, repr, d) + def test_eq(self): self.assertEqual(self._empty_mapping(), self._empty_mapping()) self.assertEqual(self._full_mapping({1: 2}), diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 296faf0a76b02f..bf6116b2dfb084 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1821,7 +1821,7 @@ def test_simple_newobj(self): with self.subTest(proto=proto): s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nL64206', s) # LONG + self.assertIn(b'\nI64206', s) # INT else: self.assertIn(b'M\xce\xfa', s) # BININT2 self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), @@ -1837,7 +1837,7 @@ def test_complex_newobj(self): with self.subTest(proto=proto): s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nL64206', s) # LONG + self.assertIn(b'\nI64206', s) # INT elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: @@ -1857,7 +1857,7 @@ def test_complex_newobj_ex(self): with self.subTest(proto=proto): s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nL64206', s) # LONG + self.assertIn(b'\nI64206', s) # INT elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: @@ -2534,7 +2534,7 @@ def test_dump_closed_file(self): f = open(TESTFN, "wb") try: f.close() - self.assertRaises(ValueError, pickle.dump, 123, f) + self.assertRaises(ValueError, self.dump, 123, f) finally: os.remove(TESTFN) @@ -2543,16 +2543,16 @@ def test_load_closed_file(self): f = open(TESTFN, "wb") try: f.close() - self.assertRaises(ValueError, pickle.dump, 123, f) + self.assertRaises(ValueError, self.dump, 123, f) finally: os.remove(TESTFN) def test_load_from_and_dump_to_file(self): stream = io.BytesIO() data = [123, {}, 124] - pickle.dump(data, stream) + self.dump(data, stream) stream.seek(0) - unpickled = pickle.load(stream) + unpickled = self.load(stream) self.assertEqual(unpickled, data) def test_highest_protocol(self): @@ -2562,20 +2562,20 @@ def test_highest_protocol(self): def test_callapi(self): f = io.BytesIO() # With and without keyword arguments - pickle.dump(123, f, -1) - pickle.dump(123, file=f, protocol=-1) - pickle.dumps(123, -1) - pickle.dumps(123, protocol=-1) - pickle.Pickler(f, -1) - pickle.Pickler(f, protocol=-1) + self.dump(123, f, -1) + self.dump(123, file=f, protocol=-1) + self.dumps(123, -1) + self.dumps(123, protocol=-1) + self.Pickler(f, -1) + self.Pickler(f, protocol=-1) def test_bad_init(self): # Test issue3664 (pickle can segfault from a badly initialized Pickler). # Override initialization without calling __init__() of the superclass. - class BadPickler(pickle.Pickler): + class BadPickler(self.Pickler): def __init__(self): pass - class BadUnpickler(pickle.Unpickler): + class BadUnpickler(self.Unpickler): def __init__(self): pass self.assertRaises(pickle.PicklingError, BadPickler().dump, 0) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 85e32a9ae7d008..7ad076ddbca528 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -56,6 +56,14 @@ def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None): info_add(name, value) +def copy_attr(info_add, name, mod, attr_name): + try: + value = getattr(mod, attr_name) + except AttributeError: + return + info_add(name, value) + + def call_func(info_add, name, mod, func_name, *, formatter=None): try: func = getattr(mod, func_name) @@ -168,11 +176,10 @@ def format_attr(attr, value): call_func(info_add, 'os.gid', os, 'getgid') call_func(info_add, 'os.uname', os, 'uname') - if hasattr(os, 'getgroups'): - groups = os.getgroups() - groups = map(str, groups) - groups = ', '.join(groups) - info_add("os.groups", groups) + def format_groups(groups): + return ', '.join(map(str, groups)) + + call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups) if hasattr(os, 'getlogin'): try: @@ -184,11 +191,7 @@ def format_attr(attr, value): else: info_add("os.login", login) - if hasattr(os, 'cpu_count'): - cpu_count = os.cpu_count() - if cpu_count: - info_add('os.cpu_count', cpu_count) - + call_func(info_add, 'os.cpu_count', os, 'cpu_count') call_func(info_add, 'os.loadavg', os, 'getloadavg') # Get environment variables: filter to list @@ -219,7 +222,9 @@ def format_attr(attr, value): ) for name, value in os.environ.items(): uname = name.upper() - if (uname in ENV_VARS or uname.startswith(("PYTHON", "LC_")) + if (uname in ENV_VARS + # Copy PYTHON* and LC_* variables + or uname.startswith(("PYTHON", "LC_")) # Visual Studio: VS140COMNTOOLS or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): info_add('os.environ[%s]' % name, value) @@ -313,12 +318,10 @@ def collect_time(info_add): ) copy_attributes(info_add, time, 'time.%s', attributes) - if not hasattr(time, 'get_clock_info'): - return - - for clock in ('time', 'perf_counter'): - tinfo = time.get_clock_info(clock) - info_add('time.%s' % clock, tinfo) + if hasattr(time, 'get_clock_info'): + for clock in ('time', 'perf_counter'): + tinfo = time.get_clock_info(clock) + info_add('time.%s' % clock, tinfo) def collect_sysconfig(info_add): @@ -331,7 +334,6 @@ def collect_sysconfig(info_add): 'CCSHARED', 'CFLAGS', 'CFLAGSFORSHARED', - 'PY_LDFLAGS', 'CONFIG_ARGS', 'HOST_GNU_TYPE', 'MACHDEP', @@ -339,6 +341,7 @@ def collect_sysconfig(info_add): 'OPT', 'PY_CFLAGS', 'PY_CFLAGS_NODIST', + 'PY_LDFLAGS', 'Py_DEBUG', 'Py_ENABLE_SHARED', 'SHELL', @@ -422,6 +425,16 @@ def collect_decimal(info_add): copy_attributes(info_add, _decimal, '_decimal.%s', attributes) +def collect_testcapi(info_add): + try: + import _testcapi + except ImportError: + return + + call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname') + copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC') + + def collect_info(info): error = False info_add = info.add @@ -444,6 +457,7 @@ def collect_info(info): collect_zlib, collect_expat, collect_decimal, + collect_testcapi, ): try: collect_func(info_add) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d051f961e92357..22868d4ba1a389 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -87,10 +87,10 @@ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", - "check__all__", "requires_android_level", "requires_multiprocessing_queue", + "check__all__", "skip_unless_bind_unix_socket", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", - "setswitchinterval", "android_not_root", + "setswitchinterval", # network "HOST", "IPV6_ENABLED", "find_unused_port", "bind_port", "open_urlresource", "bind_unix_socket", @@ -278,7 +278,6 @@ def get_attribute(obj, name): # small sizes, to make sure they work.) real_max_memuse = 0 failfast = False -match_tests = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -773,14 +772,7 @@ def dec(*args, **kwargs): is_jython = sys.platform.startswith('java') -try: - # constant used by requires_android_level() - _ANDROID_API_LEVEL = sys.getandroidapilevel() - is_android = True -except AttributeError: - # sys.getandroidapilevel() is only available on Android - is_android = False -android_not_root = (is_android and os.geteuid() != 0) +is_android = hasattr(sys, 'getandroidapilevel') if sys.platform != 'win32': unix_shell = '/system/bin/sh' if is_android else '/bin/sh' @@ -1779,13 +1771,6 @@ def requires_resource(resource): else: return unittest.skip("resource {0!r} is not enabled".format(resource)) -def requires_android_level(level, reason): - if is_android and _ANDROID_API_LEVEL < level: - return unittest.skip('%s at Android API level %d' % - (reason, _ANDROID_API_LEVEL)) - else: - return _id - def cpython_only(test): """ Decorator for tests only applicable on CPython. @@ -1805,22 +1790,6 @@ def impl_detail(msg=None, **guards): msg = msg.format(' or '.join(guardnames)) return unittest.skip(msg) -_have_mp_queue = None -def requires_multiprocessing_queue(test): - """Skip decorator for tests that use multiprocessing.Queue.""" - global _have_mp_queue - if _have_mp_queue is None: - import multiprocessing - # Without a functioning shared semaphore implementation attempts to - # instantiate a Queue will result in an ImportError (issue #3770). - try: - multiprocessing.Queue() - _have_mp_queue = True - except ImportError: - _have_mp_queue = False - msg = "requires a functioning shared semaphore implementation" - return test if _have_mp_queue else unittest.skip(msg)(test) - def _parse_guards(guards): # Returns a tuple ({platform_name: run_me}, default_value) if not guards: @@ -1901,21 +1870,67 @@ def _run_suite(suite): raise TestFailed(err) -def _match_test(test): - global match_tests +# By default, don't filter tests +_match_test_func = None +_match_test_patterns = None + - if match_tests is None: +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: return True - test_id = test.id() + else: + return _match_test_func(test.id()) - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, reject 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def set_match_tests(patterns): + global _match_test_func, _match_test_patterns + + if patterns == _match_test_patterns: + # No change: no need to recompile patterns. + return + + if not patterns: + func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matchs the whole identifier like + # 'test.test_os.FileTests.test_access' return True - return False + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + # Create a copy since patterns can be mutable and so modified later + _match_test_patterns = tuple(patterns) + _match_test_func = func + def run_unittest(*classes): @@ -1932,7 +1947,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - _filter_suite(suite, _match_test) + _filter_suite(suite, match_test) _run_suite(suite) #======================================================================= @@ -2388,6 +2403,28 @@ def skip_unless_xattr(test): msg = "no non-broken extended attribute support" return test if ok else unittest.skip(msg)(test) +_bind_nix_socket_error = None +def skip_unless_bind_unix_socket(test): + """Decorator for tests requiring a functional bind() for unix sockets.""" + if not hasattr(socket, 'AF_UNIX'): + return unittest.skip('No UNIX Sockets')(test) + global _bind_nix_socket_error + if _bind_nix_socket_error is None: + path = TESTFN + "can_bind_unix_socket" + with socket.socket(socket.AF_UNIX) as sock: + try: + sock.bind(path) + _bind_nix_socket_error = False + except OSError as e: + _bind_nix_socket_error = e + finally: + unlink(path) + if _bind_nix_socket_error: + msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error + return unittest.skip(msg)(test) + else: + return test + def fs_is_case_insensitive(directory): """Detects if the file system for the specified directory is case-insensitive.""" @@ -2794,3 +2831,8 @@ def save(self): def restore(self): for signum, handler in self.handlers.items(): self.signal.signal(signum, handler) + + +def with_pymalloc(): + import _testcapi + return _testcapi.WITH_PYMALLOC diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index a25069efb5cdcb..98f2aef5627662 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -822,6 +822,10 @@ def test_env_var_debug(self): PYTHONASYNCIODEBUG='1') self.assertEqual(stdout.rstrip(), b'False') + sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', + '-c', code) + self.assertEqual(stdout.rstrip(), b'True') + def test_create_task(self): class MyTask(asyncio.Task): pass diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 5394ddfce24b56..a6941aa4a60e37 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -363,7 +363,7 @@ def run(arg): self.assertNotEqual(thread_id, threading.get_ident()) def test_reader_callback(self): - r, w = test_utils.socketpair() + r, w = socket.socketpair() r.setblocking(False) bytes_read = bytearray() @@ -391,7 +391,7 @@ def reader(): self.assertEqual(bytes_read, b'abcdef') def test_writer_callback(self): - r, w = test_utils.socketpair() + r, w = socket.socketpair() w.setblocking(False) def writer(data): @@ -470,7 +470,7 @@ def test_sock_client_ops(self): sock = socket.socket() self._basetest_sock_recv_into(httpd, sock) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_unix_sock_client_ops(self): with test_utils.run_test_unix_server() as httpd: sock = socket.socket(socket.AF_UNIX) @@ -606,7 +606,7 @@ def test_create_connection(self): lambda: MyProto(loop=self.loop), *httpd.address) self._basetest_create_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -736,12 +736,8 @@ def test_create_ssl_connection(self): self._test_create_ssl_connection(httpd, create_connection, peername=httpd.address) - def test_legacy_create_ssl_connection(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_ssl_connection() - + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_ssl_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -757,10 +753,6 @@ def test_create_ssl_unix_connection(self): check_sockname, peername=httpd.address) - def test_legacy_create_ssl_unix_connection(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_ssl_unix_connection() - def test_create_connection_local_addr(self): with test_utils.run_test_server() as httpd: port = support.find_unused_port() @@ -969,7 +961,7 @@ def _make_unix_server(self, factory, **kwargs): return server, path - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_server(self): proto = MyProto(loop=self.loop) server, path = self._make_unix_server(lambda: proto) @@ -1061,12 +1053,8 @@ def test_create_server_ssl(self): # stop serving server.close() - def test_legacy_create_server_ssl(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_server_ssl() - + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1096,10 +1084,6 @@ def test_create_unix_server_ssl(self): # stop serving server.close() - def test_legacy_create_unix_server_ssl(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_unix_server_ssl() - @unittest.skipIf(ssl is None, 'No ssl module') def test_create_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) @@ -1129,12 +1113,8 @@ def test_create_server_ssl_verify_failed(self): self.assertIsNone(proto.transport) server.close() - def test_legacy_create_server_ssl_verify_failed(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_server_ssl_verify_failed() - + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1163,11 +1143,6 @@ def test_create_unix_server_ssl_verify_failed(self): self.assertIsNone(proto.transport) server.close() - - def test_legacy_create_unix_server_ssl_verify_failed(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_unix_server_ssl_verify_failed() - @unittest.skipIf(ssl is None, 'No ssl module') def test_create_server_ssl_match_failed(self): proto = MyProto(loop=self.loop) @@ -1196,12 +1171,8 @@ def test_create_server_ssl_match_failed(self): proto.transport.close() server.close() - def test_legacy_create_server_ssl_match_failed(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_server_ssl_match_failed() - + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( @@ -1226,10 +1197,6 @@ def test_create_unix_server_ssl_verified(self): server.close() self.loop.run_until_complete(proto.done) - def test_legacy_create_unix_server_ssl_verified(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_unix_server_ssl_verified() - @unittest.skipIf(ssl is None, 'No ssl module') def test_create_server_ssl_verified(self): proto = MyProto(loop=self.loop) @@ -1259,10 +1226,6 @@ def test_create_server_ssl_verified(self): server.close() self.loop.run_until_complete(proto.done) - def test_legacy_create_server_ssl_verified(self): - with test_utils.force_legacy_ssl_support(): - self.test_create_server_ssl_verified() - def test_create_server_sock(self): proto = asyncio.Future(loop=self.loop) @@ -1605,7 +1568,7 @@ def reader(data): @unittest.skipUnless(sys.platform != 'win32', "Don't support pipes for Windows") def test_write_pipe_disconnect_on_close(self): - rsock, wsock = test_utils.socketpair() + rsock, wsock = socket.socketpair() rsock.setblocking(False) pipeobj = io.open(wsock.detach(), 'wb', 1024) @@ -1743,7 +1706,7 @@ def reader(data): self.assertEqual('CLOSED', write_proto.state) def test_prompt_cancellation(self): - r, w = test_utils.socketpair() + r, w = socket.socketpair() r.setblocking(False) f = self.loop.sock_recv(r, 1) ov = getattr(f, 'ov', None) @@ -1808,7 +1771,7 @@ def wait(): def test_remove_fds_after_closing(self): loop = self.create_event_loop() callback = lambda: None - r, w = test_utils.socketpair() + r, w = socket.socketpair() self.addCleanup(r.close) self.addCleanup(w.close) loop.add_reader(r, callback) @@ -1820,7 +1783,7 @@ def test_remove_fds_after_closing(self): def test_add_fds_after_closing(self): loop = self.create_event_loop() callback = lambda: None - r, w = test_utils.socketpair() + r, w = socket.socketpair() self.addCleanup(r.close) self.addCleanup(w.close) loop.close() @@ -2159,37 +2122,6 @@ class ProactorEventLoopTests(EventLoopTestsMixin, def create_event_loop(self): return asyncio.ProactorEventLoop() - if not sslproto._is_sslproto_available(): - def test_create_ssl_connection(self): - raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") - - def test_create_server_ssl(self): - raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") - - def test_create_server_ssl_verify_failed(self): - raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") - - def test_create_server_ssl_match_failed(self): - raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") - - def test_create_server_ssl_verified(self): - raise unittest.SkipTest("need python 3.5 (ssl.MemoryBIO)") - - def test_legacy_create_ssl_connection(self): - raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") - - def test_legacy_create_server_ssl(self): - raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") - - def test_legacy_create_server_ssl_verify_failed(self): - raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") - - def test_legacy_create_server_ssl_match_failed(self): - raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") - - def test_legacy_create_server_ssl_verified(self): - raise unittest.SkipTest("IocpEventLoop incompatible with legacy SSL") - def test_reader_callback(self): raise unittest.SkipTest("IocpEventLoop does not have add_reader()") @@ -2209,7 +2141,7 @@ def test_create_datagram_endpoint(self): def test_remove_fds_after_closing(self): raise unittest.SkipTest("IocpEventLoop does not have add_reader()") else: - from asyncio import selectors + import selectors class UnixEventLoopTestsMixin(EventLoopTestsMixin): def setUp(self): @@ -2223,6 +2155,10 @@ def tearDown(self): super().tearDown() def test_get_event_loop_new_process(self): + # Issue bpo-32126: The multiprocessing module used by + # ProcessPoolExecutor is not functional when the + # multiprocessing.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') async def main(): pool = concurrent.futures.ProcessPoolExecutor() result = await self.loop.run_in_executor( diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 7a8b52352d6158..def08b96ab0ccd 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -444,15 +444,13 @@ def setUp(self): self.ssock, self.csock = mock.Mock(), mock.Mock() - class EventLoop(BaseProactorEventLoop): - def _socketpair(s): - return (self.ssock, self.csock) - - self.loop = EventLoop(self.proactor) + with mock.patch('asyncio.proactor_events.socket.socketpair', + return_value=(self.ssock, self.csock)): + self.loop = BaseProactorEventLoop(self.proactor) self.set_event_loop(self.loop) @mock.patch.object(BaseProactorEventLoop, 'call_soon') - @mock.patch.object(BaseProactorEventLoop, '_socketpair') + @mock.patch('asyncio.proactor_events.socket.socketpair') def test_ctor(self, socketpair, call_soon): ssock, csock = socketpair.return_value = ( mock.Mock(), mock.Mock()) @@ -506,14 +504,6 @@ def test_sock_accept(self): self.loop.sock_accept(self.sock) self.proactor.accept.assert_called_with(self.sock) - def test_socketpair(self): - class EventLoop(BaseProactorEventLoop): - # override the destructor to not log a ResourceWarning - def __del__(self): - pass - self.assertRaises( - NotImplementedError, EventLoop, self.proactor) - def test_make_socket_transport(self): tr = self.loop._make_socket_transport(self.sock, asyncio.Protocol()) self.assertIsInstance(tr, _ProactorSocketTransport) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c50b3e49565c92..616eb6f2630503 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1,6 +1,7 @@ """Tests for selector_events.py""" import errno +import selectors import socket import unittest from unittest import mock @@ -10,11 +11,9 @@ ssl = None import asyncio -from asyncio import selectors from asyncio import test_utils from asyncio.selector_events import BaseSelectorEventLoop from asyncio.selector_events import _SelectorTransport -from asyncio.selector_events import _SelectorSslTransport from asyncio.selector_events import _SelectorSocketTransport from asyncio.selector_events import _SelectorDatagramTransport @@ -155,9 +154,6 @@ def test_close_no_selector(self): self.loop.close() self.assertIsNone(self.loop._selector) - def test_socketpair(self): - self.assertRaises(NotImplementedError, self.loop._socketpair) - def test_read_from_self_tryagain(self): self.loop._ssock.recv.side_effect = BlockingIOError self.assertIsNone(self.loop._read_from_self()) @@ -182,7 +178,28 @@ def test_sock_recv(self): f = self.loop.sock_recv(sock, 1024) self.assertIsInstance(f, asyncio.Future) - self.loop._sock_recv.assert_called_with(f, False, sock, 1024) + self.loop._sock_recv.assert_called_with(f, None, sock, 1024) + + def test_sock_recv_reconnection(self): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.recv.side_effect = BlockingIOError + sock.gettimeout.return_value = 0.0 + + self.loop.add_reader = mock.Mock() + self.loop.remove_reader = mock.Mock() + fut = self.loop.sock_recv(sock, 1024) + callback = self.loop.add_reader.call_args[0][1] + params = self.loop.add_reader.call_args[0][2:] + + # emulate the old socket has closed, but the new one has + # the same fileno, so callback is called with old (closed) socket + sock.fileno.return_value = -1 + sock.recv.side_effect = OSError(9) + callback(*params) + + self.assertIsInstance(fut.exception(), OSError) + self.assertEqual((10,), self.loop.remove_reader.call_args[0]) def test__sock_recv_canceled_fut(self): sock = mock.Mock() @@ -190,7 +207,7 @@ def test__sock_recv_canceled_fut(self): f = asyncio.Future(loop=self.loop) f.cancel() - self.loop._sock_recv(f, False, sock, 1024) + self.loop._sock_recv(f, None, sock, 1024) self.assertFalse(sock.recv.called) def test__sock_recv_unregister(self): @@ -201,7 +218,7 @@ def test__sock_recv_unregister(self): f.cancel() self.loop.remove_reader = mock.Mock() - self.loop._sock_recv(f, True, sock, 1024) + self.loop._sock_recv(f, 10, sock, 1024) self.assertEqual((10,), self.loop.remove_reader.call_args[0]) def test__sock_recv_tryagain(self): @@ -211,8 +228,8 @@ def test__sock_recv_tryagain(self): sock.recv.side_effect = BlockingIOError self.loop.add_reader = mock.Mock() - self.loop._sock_recv(f, False, sock, 1024) - self.assertEqual((10, self.loop._sock_recv, f, True, sock, 1024), + self.loop._sock_recv(f, None, sock, 1024) + self.assertEqual((10, self.loop._sock_recv, f, 10, sock, 1024), self.loop.add_reader.call_args[0]) def test__sock_recv_exception(self): @@ -221,7 +238,7 @@ def test__sock_recv_exception(self): sock.fileno.return_value = 10 err = sock.recv.side_effect = OSError() - self.loop._sock_recv(f, False, sock, 1024) + self.loop._sock_recv(f, None, sock, 1024) self.assertIs(err, f.exception()) def test_sock_sendall(self): @@ -231,7 +248,7 @@ def test_sock_sendall(self): f = self.loop.sock_sendall(sock, b'data') self.assertIsInstance(f, asyncio.Future) self.assertEqual( - (f, False, sock, b'data'), + (f, None, sock, b'data'), self.loop._sock_sendall.call_args[0]) def test_sock_sendall_nodata(self): @@ -244,13 +261,34 @@ def test_sock_sendall_nodata(self): self.assertIsNone(f.result()) self.assertFalse(self.loop._sock_sendall.called) + def test_sock_sendall_reconnection(self): + sock = mock.Mock() + sock.fileno.return_value = 10 + sock.send.side_effect = BlockingIOError + sock.gettimeout.return_value = 0.0 + + self.loop.add_writer = mock.Mock() + self.loop.remove_writer = mock.Mock() + fut = self.loop.sock_sendall(sock, b'data') + callback = self.loop.add_writer.call_args[0][1] + params = self.loop.add_writer.call_args[0][2:] + + # emulate the old socket has closed, but the new one has + # the same fileno, so callback is called with old (closed) socket + sock.fileno.return_value = -1 + sock.send.side_effect = OSError(9) + callback(*params) + + self.assertIsInstance(fut.exception(), OSError) + self.assertEqual((10,), self.loop.remove_writer.call_args[0]) + def test__sock_sendall_canceled_fut(self): sock = mock.Mock() f = asyncio.Future(loop=self.loop) f.cancel() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertFalse(sock.send.called) def test__sock_sendall_unregister(self): @@ -261,7 +299,7 @@ def test__sock_sendall_unregister(self): f.cancel() self.loop.remove_writer = mock.Mock() - self.loop._sock_sendall(f, True, sock, b'data') + self.loop._sock_sendall(f, 10, sock, b'data') self.assertEqual((10,), self.loop.remove_writer.call_args[0]) def test__sock_sendall_tryagain(self): @@ -271,9 +309,9 @@ def test__sock_sendall_tryagain(self): sock.send.side_effect = BlockingIOError self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'data'), + (10, self.loop._sock_sendall, f, 10, sock, b'data'), self.loop.add_writer.call_args[0]) def test__sock_sendall_interrupted(self): @@ -283,9 +321,9 @@ def test__sock_sendall_interrupted(self): sock.send.side_effect = InterruptedError self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'data'), + (10, self.loop._sock_sendall, f, 10, sock, b'data'), self.loop.add_writer.call_args[0]) def test__sock_sendall_exception(self): @@ -294,7 +332,7 @@ def test__sock_sendall_exception(self): sock.fileno.return_value = 10 err = sock.send.side_effect = OSError() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertIs(f.exception(), err) def test__sock_sendall(self): @@ -304,7 +342,7 @@ def test__sock_sendall(self): sock.fileno.return_value = 10 sock.send.return_value = 4 - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertTrue(f.done()) self.assertIsNone(f.result()) @@ -316,10 +354,10 @@ def test__sock_sendall_partial(self): sock.send.return_value = 2 self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertFalse(f.done()) self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'ta'), + (10, self.loop._sock_sendall, f, 10, sock, b'ta'), self.loop.add_writer.call_args[0]) def test__sock_sendall_none(self): @@ -330,10 +368,10 @@ def test__sock_sendall_none(self): sock.send.return_value = 0 self.loop.add_writer = mock.Mock() - self.loop._sock_sendall(f, False, sock, b'data') + self.loop._sock_sendall(f, None, sock, b'data') self.assertFalse(f.done()) self.assertEqual( - (10, self.loop._sock_sendall, f, True, sock, b'data'), + (10, self.loop._sock_sendall, f, 10, sock, b'data'), self.loop.add_writer.call_args[0]) def test_sock_connect_timeout(self): @@ -1138,368 +1176,6 @@ def test_transport_close_remove_writer(self, m_log): remove_writer.assert_called_with(self.sock_fd) -@unittest.skipIf(ssl is None, 'No ssl module') -class SelectorSslTransportTests(test_utils.TestCase): - - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() - self.protocol = test_utils.make_test_protocol(asyncio.Protocol) - self.sock = mock.Mock(socket.socket) - self.sock.fileno.return_value = 7 - self.sslsock = mock.Mock() - self.sslsock.fileno.return_value = 1 - self.sslcontext = mock.Mock() - self.sslcontext.wrap_socket.return_value = self.sslsock - - def ssl_transport(self, waiter=None, server_hostname=None): - transport = _SelectorSslTransport(self.loop, self.sock, self.protocol, - self.sslcontext, waiter=waiter, - server_hostname=server_hostname) - self.addCleanup(close_transport, transport) - return transport - - def _make_one(self, create_waiter=None): - transport = self.ssl_transport() - self.sock.reset_mock() - self.sslsock.reset_mock() - self.sslcontext.reset_mock() - self.loop.reset_counters() - return transport - - def test_on_handshake(self): - waiter = asyncio.Future(loop=self.loop) - tr = self.ssl_transport(waiter=waiter) - self.assertTrue(self.sslsock.do_handshake.called) - self.loop.assert_reader(1, tr._read_ready) - test_utils.run_briefly(self.loop) - self.assertIsNone(waiter.result()) - - def test_on_handshake_reader_retry(self): - self.loop.set_debug(False) - self.sslsock.do_handshake.side_effect = ssl.SSLWantReadError - transport = self.ssl_transport() - self.loop.assert_reader(1, transport._on_handshake, None) - - def test_on_handshake_writer_retry(self): - self.loop.set_debug(False) - self.sslsock.do_handshake.side_effect = ssl.SSLWantWriteError - transport = self.ssl_transport() - self.loop.assert_writer(1, transport._on_handshake, None) - - def test_on_handshake_exc(self): - exc = ValueError() - self.sslsock.do_handshake.side_effect = exc - with test_utils.disable_logger(): - waiter = asyncio.Future(loop=self.loop) - self.ssl_transport(waiter=waiter) - self.assertTrue(waiter.done()) - self.assertIs(exc, waiter.exception()) - self.assertTrue(self.sslsock.close.called) - - def test_on_handshake_base_exc(self): - waiter = asyncio.Future(loop=self.loop) - transport = self.ssl_transport(waiter=waiter) - exc = BaseException() - self.sslsock.do_handshake.side_effect = exc - with test_utils.disable_logger(): - self.assertRaises(BaseException, transport._on_handshake, 0) - self.assertTrue(self.sslsock.close.called) - self.assertTrue(waiter.done()) - self.assertIs(exc, waiter.exception()) - - def test_cancel_handshake(self): - # Python issue #23197: cancelling a handshake must not raise an - # exception or log an error, even if the handshake failed - waiter = asyncio.Future(loop=self.loop) - transport = self.ssl_transport(waiter=waiter) - waiter.cancel() - exc = ValueError() - self.sslsock.do_handshake.side_effect = exc - with test_utils.disable_logger(): - transport._on_handshake(0) - transport.close() - test_utils.run_briefly(self.loop) - - def test_pause_resume_reading(self): - tr = self._make_one() - self.assertFalse(tr._paused) - self.loop.assert_reader(1, tr._read_ready) - tr.pause_reading() - self.assertTrue(tr._paused) - self.assertFalse(1 in self.loop.readers) - tr.resume_reading() - self.assertFalse(tr._paused) - self.loop.assert_reader(1, tr._read_ready) - with self.assertRaises(RuntimeError): - tr.resume_reading() - - def test_write(self): - transport = self._make_one() - transport.write(b'data') - self.assertEqual(list_to_buffer([b'data']), transport._buffer) - - def test_write_bytearray(self): - transport = self._make_one() - data = bytearray(b'data') - transport.write(data) - self.assertEqual(list_to_buffer([b'data']), transport._buffer) - self.assertEqual(data, bytearray(b'data')) # Hasn't been mutated. - self.assertIsNot(data, transport._buffer) # Hasn't been incorporated. - - def test_write_memoryview(self): - transport = self._make_one() - data = memoryview(b'data') - transport.write(data) - self.assertEqual(list_to_buffer([b'data']), transport._buffer) - - def test_write_no_data(self): - transport = self._make_one() - transport._buffer.extend(b'data') - transport.write(b'') - self.assertEqual(list_to_buffer([b'data']), transport._buffer) - - def test_write_str(self): - transport = self._make_one() - self.assertRaises(TypeError, transport.write, 'str') - - def test_write_closing(self): - transport = self._make_one() - transport.close() - self.assertEqual(transport._conn_lost, 1) - transport.write(b'data') - self.assertEqual(transport._conn_lost, 2) - - @mock.patch('asyncio.selector_events.logger') - def test_write_exception(self, m_log): - transport = self._make_one() - transport._conn_lost = 1 - transport.write(b'data') - self.assertEqual(transport._buffer, list_to_buffer()) - transport.write(b'data') - transport.write(b'data') - transport.write(b'data') - transport.write(b'data') - m_log.warning.assert_called_with('socket.send() raised exception.') - - def test_read_ready_recv(self): - self.sslsock.recv.return_value = b'data' - transport = self._make_one() - transport._read_ready() - self.assertTrue(self.sslsock.recv.called) - self.assertEqual((b'data',), self.protocol.data_received.call_args[0]) - - def test_read_ready_write_wants_read(self): - self.loop._add_writer = mock.Mock() - self.sslsock.recv.side_effect = BlockingIOError - transport = self._make_one() - transport._write_wants_read = True - transport._write_ready = mock.Mock() - transport._buffer.extend(b'data') - transport._read_ready() - - self.assertFalse(transport._write_wants_read) - transport._write_ready.assert_called_with() - self.loop._add_writer.assert_called_with( - transport._sock_fd, transport._write_ready) - - def test_read_ready_recv_eof(self): - self.sslsock.recv.return_value = b'' - transport = self._make_one() - transport.close = mock.Mock() - transport._read_ready() - transport.close.assert_called_with() - self.protocol.eof_received.assert_called_with() - - def test_read_ready_recv_conn_reset(self): - err = self.sslsock.recv.side_effect = ConnectionResetError() - transport = self._make_one() - transport._force_close = mock.Mock() - with test_utils.disable_logger(): - transport._read_ready() - transport._force_close.assert_called_with(err) - - def test_read_ready_recv_retry(self): - self.sslsock.recv.side_effect = ssl.SSLWantReadError - transport = self._make_one() - transport._read_ready() - self.assertTrue(self.sslsock.recv.called) - self.assertFalse(self.protocol.data_received.called) - - self.sslsock.recv.side_effect = BlockingIOError - transport._read_ready() - self.assertFalse(self.protocol.data_received.called) - - self.sslsock.recv.side_effect = InterruptedError - transport._read_ready() - self.assertFalse(self.protocol.data_received.called) - - def test_read_ready_recv_write(self): - self.loop._remove_reader = mock.Mock() - self.loop._add_writer = mock.Mock() - self.sslsock.recv.side_effect = ssl.SSLWantWriteError - transport = self._make_one() - transport._read_ready() - self.assertFalse(self.protocol.data_received.called) - self.assertTrue(transport._read_wants_write) - - self.loop._remove_reader.assert_called_with(transport._sock_fd) - self.loop._add_writer.assert_called_with( - transport._sock_fd, transport._write_ready) - - def test_read_ready_recv_exc(self): - err = self.sslsock.recv.side_effect = OSError() - transport = self._make_one() - transport._fatal_error = mock.Mock() - transport._read_ready() - transport._fatal_error.assert_called_with( - err, - 'Fatal read error on SSL transport') - - def test_write_ready_send(self): - self.sslsock.send.return_value = 4 - transport = self._make_one() - transport._buffer = list_to_buffer([b'data']) - transport._write_ready() - self.assertEqual(list_to_buffer(), transport._buffer) - self.assertTrue(self.sslsock.send.called) - - def test_write_ready_send_none(self): - self.sslsock.send.return_value = 0 - transport = self._make_one() - transport._buffer = list_to_buffer([b'data1', b'data2']) - transport._write_ready() - self.assertTrue(self.sslsock.send.called) - self.assertEqual(list_to_buffer([b'data1data2']), transport._buffer) - - def test_write_ready_send_partial(self): - self.sslsock.send.return_value = 2 - transport = self._make_one() - transport._buffer = list_to_buffer([b'data1', b'data2']) - transport._write_ready() - self.assertTrue(self.sslsock.send.called) - self.assertEqual(list_to_buffer([b'ta1data2']), transport._buffer) - - def test_write_ready_send_closing_partial(self): - self.sslsock.send.return_value = 2 - transport = self._make_one() - transport._buffer = list_to_buffer([b'data1', b'data2']) - transport._write_ready() - self.assertTrue(self.sslsock.send.called) - self.assertFalse(self.sslsock.close.called) - - def test_write_ready_send_closing(self): - self.sslsock.send.return_value = 4 - transport = self._make_one() - transport._buffer = list_to_buffer([b'data']) - transport.close() - transport._write_ready() - self.protocol.connection_lost.assert_called_with(None) - - def test_write_ready_send_closing_empty_buffer(self): - self.sslsock.send.return_value = 4 - call_soon = self.loop.call_soon = mock.Mock() - transport = self._make_one() - transport._buffer = list_to_buffer() - transport.close() - transport._write_ready() - call_soon.assert_called_with(transport._call_connection_lost, None) - - def test_write_ready_send_retry(self): - transport = self._make_one() - transport._buffer = list_to_buffer([b'data']) - - self.sslsock.send.side_effect = ssl.SSLWantWriteError - transport._write_ready() - self.assertEqual(list_to_buffer([b'data']), transport._buffer) - - self.sslsock.send.side_effect = BlockingIOError() - transport._write_ready() - self.assertEqual(list_to_buffer([b'data']), transport._buffer) - - def test_write_ready_send_read(self): - transport = self._make_one() - transport._buffer = list_to_buffer([b'data']) - - self.loop._remove_writer = mock.Mock() - self.sslsock.send.side_effect = ssl.SSLWantReadError - transport._write_ready() - self.assertFalse(self.protocol.data_received.called) - self.assertTrue(transport._write_wants_read) - self.loop._remove_writer.assert_called_with(transport._sock_fd) - - def test_write_ready_send_exc(self): - err = self.sslsock.send.side_effect = OSError() - - transport = self._make_one() - transport._buffer = list_to_buffer([b'data']) - transport._fatal_error = mock.Mock() - transport._write_ready() - transport._fatal_error.assert_called_with( - err, - 'Fatal write error on SSL transport') - self.assertEqual(list_to_buffer(), transport._buffer) - - def test_write_ready_read_wants_write(self): - self.loop._add_reader = mock.Mock() - self.sslsock.send.side_effect = BlockingIOError - transport = self._make_one() - transport._read_wants_write = True - transport._read_ready = mock.Mock() - transport._write_ready() - - self.assertFalse(transport._read_wants_write) - transport._read_ready.assert_called_with() - self.loop._add_reader.assert_called_with( - transport._sock_fd, transport._read_ready) - - def test_write_eof(self): - tr = self._make_one() - self.assertFalse(tr.can_write_eof()) - self.assertRaises(NotImplementedError, tr.write_eof) - - def check_close(self): - tr = self._make_one() - tr.close() - - self.assertTrue(tr.is_closing()) - self.assertEqual(1, self.loop.remove_reader_count[1]) - self.assertEqual(tr._conn_lost, 1) - - tr.close() - self.assertEqual(tr._conn_lost, 1) - self.assertEqual(1, self.loop.remove_reader_count[1]) - - test_utils.run_briefly(self.loop) - - def test_close(self): - self.check_close() - self.assertTrue(self.protocol.connection_made.called) - self.assertTrue(self.protocol.connection_lost.called) - - def test_close_not_connected(self): - self.sslsock.do_handshake.side_effect = ssl.SSLWantReadError - self.check_close() - self.assertFalse(self.protocol.connection_made.called) - self.assertFalse(self.protocol.connection_lost.called) - - @unittest.skipIf(ssl is None, 'No SSL support') - def test_server_hostname(self): - self.ssl_transport(server_hostname='localhost') - self.sslcontext.wrap_socket.assert_called_with( - self.sock, do_handshake_on_connect=False, server_side=False, - server_hostname='localhost') - - -class SelectorSslWithoutSslTransportTests(unittest.TestCase): - - @mock.patch('asyncio.selector_events.ssl', None) - def test_ssl_transport_requires_ssl_module(self): - Mock = mock.Mock - with self.assertRaises(RuntimeError): - _SelectorSslTransport(Mock(), Mock(), Mock(), Mock()) - - class SelectorDatagramTransportTests(test_utils.TestCase): def setUp(self): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index b47433a4cfdbd4..a1e5bd7fab6c8e 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -3,11 +3,13 @@ import gc import os import queue +import pickle import socket import sys import threading import unittest from unittest import mock +from test import support try: import ssl except ImportError: @@ -56,7 +58,7 @@ def test_open_connection(self): loop=self.loop) self._basetest_open_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -85,8 +87,8 @@ def test_open_connection_no_loop_ssl(self): self._basetest_open_connection_no_loop_ssl(conn_fut) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_open_unix_connection_no_loop_ssl(self): with test_utils.run_test_unix_server(use_ssl=True) as httpd: conn_fut = asyncio.open_unix_connection( @@ -112,7 +114,7 @@ def test_open_connection_error(self): loop=self.loop) self._basetest_open_connection_error(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection_error(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -633,7 +635,7 @@ def client(addr): server.stop() self.assertEqual(msg, b"hello world!\n") - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_start_unix_server(self): class MyServer: @@ -845,6 +847,23 @@ def test___repr__transport(self): stream._transport.__repr__.return_value = "" self.assertEqual(">", repr(stream)) + def test_IncompleteReadError_pickleable(self): + e = asyncio.IncompleteReadError(b'abc', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.partial, e2.partial) + self.assertEqual(e.expected, e2.expected) + + def test_LimitOverrunError_pickleable(self): + e = asyncio.LimitOverrunError('message', 10) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_protocol=proto): + e2 = pickle.loads(pickle.dumps(e, protocol=proto)) + self.assertEqual(str(e), str(e2)) + self.assertEqual(e.consumed, e2.consumed) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 1d77f9f2c4d180..f66f7f1e1706fb 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2350,6 +2350,10 @@ def test_env_var_debug(self): PYTHONPATH=aio_path) self.assertEqual(stdout.rstrip(), b'False') + sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', + '-c', code) + self.assertEqual(stdout.rstrip(), b'True') + class FutureGatherTests(GatherTestsBase, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 11f0890d65f964..284c73d8daf653 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -13,6 +13,7 @@ import threading import unittest from unittest import mock +from test import support if sys.platform == 'win32': raise unittest.SkipTest('UNIX only') @@ -239,6 +240,7 @@ def setUp(self): self.loop = asyncio.SelectorEventLoop() self.set_event_loop(self.loop) + @support.skip_unless_bind_unix_socket def test_create_unix_server_existing_path_sock(self): with test_utils.unix_socket_path() as path: sock = socket.socket(socket.AF_UNIX) @@ -251,7 +253,7 @@ def test_create_unix_server_existing_path_sock(self): srv.close() self.loop.run_until_complete(srv.wait_closed()) - @unittest.skipUnless(hasattr(os, 'fspath'), 'no os.fspath') + @support.skip_unless_bind_unix_socket def test_create_unix_server_pathlib(self): with test_utils.unix_socket_path() as path: path = pathlib.Path(path) @@ -260,6 +262,15 @@ def test_create_unix_server_pathlib(self): srv.close() self.loop.run_until_complete(srv.wait_closed()) + def test_create_unix_connection_pathlib(self): + with test_utils.unix_socket_path() as path: + path = pathlib.Path(path) + coro = self.loop.create_unix_connection(lambda: None, path) + with self.assertRaises(FileNotFoundError): + # If pathlib.Path wasn't supported, the exception would be + # different. + self.loop.run_until_complete(coro) + def test_create_unix_server_existing_path_nonsock(self): with tempfile.NamedTemporaryFile() as file: coro = self.loop.create_unix_server(lambda: None, file.name) @@ -300,6 +311,7 @@ def test_create_unix_server_path_dgram(self): @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'no socket.SOCK_NONBLOCK (linux only)') + @support.skip_unless_bind_unix_socket def test_create_unix_server_path_stream_bittype(self): sock = socket.socket( socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_NONBLOCK) @@ -319,7 +331,7 @@ def test_create_unix_server_path_stream_bittype(self): def test_create_unix_connection_path_inetsock(self): sock = socket.socket() with sock: - coro = self.loop.create_unix_connection(lambda: None, path=None, + coro = self.loop.create_unix_connection(lambda: None, sock=sock) with self.assertRaisesRegex(ValueError, 'A UNIX Domain Stream.*was expected'): @@ -382,7 +394,7 @@ def setUp(self): self.pipe = mock.Mock(spec_set=io.RawIOBase) self.pipe.fileno.return_value = 5 - blocking_patcher = mock.patch('asyncio.unix_events._set_nonblocking') + blocking_patcher = mock.patch('os.set_blocking') blocking_patcher.start() self.addCleanup(blocking_patcher.stop) @@ -532,7 +544,7 @@ def setUp(self): self.pipe = mock.Mock(spec_set=io.RawIOBase) self.pipe.fileno.return_value = 5 - blocking_patcher = mock.patch('asyncio.unix_events._set_nonblocking') + blocking_patcher = mock.patch('os.set_blocking') blocking_patcher.start() self.addCleanup(blocking_patcher.stop) @@ -1616,5 +1628,75 @@ def test_child_watcher_replace_mainloop_existing(self): new_loop.close() +class TestFunctional(unittest.TestCase): + + def setUp(self): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + asyncio.set_event_loop(None) + + def test_add_reader_invalid_argument(self): + def assert_raises(): + return self.assertRaisesRegex(ValueError, r'Invalid file object') + + cb = lambda: None + + with assert_raises(): + self.loop.add_reader(object(), cb) + with assert_raises(): + self.loop.add_writer(object(), cb) + + with assert_raises(): + self.loop.remove_reader(object()) + with assert_raises(): + self.loop.remove_writer(object()) + + def test_add_reader_or_writer_transport_fd(self): + def assert_raises(): + return self.assertRaisesRegex( + RuntimeError, + r'File descriptor .* is used by transport') + + async def runner(): + tr, pr = await self.loop.create_connection( + lambda: asyncio.Protocol(), sock=rsock) + + try: + cb = lambda: None + + with assert_raises(): + self.loop.add_reader(rsock, cb) + with assert_raises(): + self.loop.add_reader(rsock.fileno(), cb) + + with assert_raises(): + self.loop.remove_reader(rsock) + with assert_raises(): + self.loop.remove_reader(rsock.fileno()) + + with assert_raises(): + self.loop.add_writer(rsock, cb) + with assert_raises(): + self.loop.add_writer(rsock.fileno(), cb) + + with assert_raises(): + self.loop.remove_writer(rsock) + with assert_raises(): + self.loop.remove_writer(rsock.fileno()) + + finally: + tr.close() + + rsock, wsock = socket.socketpair() + try: + self.loop.run_until_complete(runner()) + finally: + rsock.close() + wsock.close() + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index c72eef1afdb454..5fdf5ff5e440ae 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -1,4 +1,5 @@ import os +import socket import sys import unittest from unittest import mock @@ -6,10 +7,10 @@ if sys.platform != 'win32': raise unittest.SkipTest('Windows only') +import _overlapped import _winapi import asyncio -from asyncio import _overlapped from asyncio import test_utils from asyncio import windows_events @@ -36,7 +37,7 @@ def setUp(self): self.set_event_loop(self.loop) def test_close(self): - a, b = self.loop._socketpair() + a, b = socket.socketpair() trans = self.loop._make_socket_transport(a, asyncio.Protocol()) f = asyncio.ensure_future(self.loop.sock_recv(b, 100)) trans.close() diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index d48b8bcbb0874f..952f95e5730a02 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -9,9 +9,9 @@ if sys.platform != 'win32': raise unittest.SkipTest('Windows only') +import _overlapped import _winapi -from asyncio import _overlapped from asyncio import windows_utils try: from test import support @@ -19,56 +19,6 @@ from asyncio import test_support as support -class WinsocketpairTests(unittest.TestCase): - - def check_winsocketpair(self, ssock, csock): - csock.send(b'xxx') - self.assertEqual(b'xxx', ssock.recv(1024)) - csock.close() - ssock.close() - - def test_winsocketpair(self): - ssock, csock = windows_utils.socketpair() - self.check_winsocketpair(ssock, csock) - - @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled') - def test_winsocketpair_ipv6(self): - ssock, csock = windows_utils.socketpair(family=socket.AF_INET6) - self.check_winsocketpair(ssock, csock) - - @unittest.skipIf(hasattr(socket, 'socketpair'), - 'socket.socketpair is available') - @mock.patch('asyncio.windows_utils.socket') - def test_winsocketpair_exc(self, m_socket): - m_socket.AF_INET = socket.AF_INET - m_socket.SOCK_STREAM = socket.SOCK_STREAM - m_socket.socket.return_value.getsockname.return_value = ('', 12345) - m_socket.socket.return_value.accept.return_value = object(), object() - m_socket.socket.return_value.connect.side_effect = OSError() - - self.assertRaises(OSError, windows_utils.socketpair) - - def test_winsocketpair_invalid_args(self): - self.assertRaises(ValueError, - windows_utils.socketpair, family=socket.AF_UNSPEC) - self.assertRaises(ValueError, - windows_utils.socketpair, type=socket.SOCK_DGRAM) - self.assertRaises(ValueError, - windows_utils.socketpair, proto=1) - - @unittest.skipIf(hasattr(socket, 'socketpair'), - 'socket.socketpair is available') - @mock.patch('asyncio.windows_utils.socket') - def test_winsocketpair_close(self, m_socket): - m_socket.AF_INET = socket.AF_INET - m_socket.SOCK_STREAM = socket.SOCK_STREAM - sock = mock.Mock() - m_socket.socket.return_value = sock - sock.bind.side_effect = OSError - self.assertRaises(OSError, windows_utils.socketpair) - self.assertTrue(sock.close.called) - - class PipeTests(unittest.TestCase): def test_pipe_overlapped(self): diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index 27d514fe2ee517..c055ee3d83c347 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -92,7 +92,7 @@ def test_interface_single_arg(self): exc = Exception(arg) results = ([len(exc.args), 1], [exc.args[0], arg], [str(exc), str(arg)], - [repr(exc), exc.__class__.__name__ + repr(exc.args)]) + [repr(exc), '%s(%r)' % (exc.__class__.__name__, arg)]) self.interface_test_driver(results) def test_interface_multi_arg(self): diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 635c98faced9cd..2a22739fb0f069 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -6,7 +6,6 @@ import sys import sysconfig import shutil -import subprocess from collections import namedtuple import test.support @@ -18,9 +17,12 @@ # Set our expectation for the default encoding used in the C locale # for the filesystem encoding and the standard streams -# AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII +# While most *nix platforms default to ASCII in the C locale, some use a +# different encoding. if sys.platform.startswith("aix"): C_LOCALE_STREAM_ENCODING = "iso8859-1" +elif test.support.is_android: + C_LOCALE_STREAM_ENCODING = "utf-8" else: C_LOCALE_STREAM_ENCODING = "ascii" @@ -301,6 +303,19 @@ def _check_c_locale_coercion(self, # See https://bugs.python.org/issue30672 for discussion if locale_to_set == "POSIX": continue + + # Platforms using UTF-8 in the C locale do not print + # CLI_COERCION_WARNING when all the locale envt variables are + # not set or set to the empty string. + _expected_warnings = expected_warnings + for _env_var in base_var_dict: + if base_var_dict[_env_var]: + break + else: + if (C_LOCALE_STREAM_ENCODING == "utf-8" and + locale_to_set == "" and coerce_c_locale == "warn"): + _expected_warnings = None + with self.subTest(env_var=env_var, nominal_locale=locale_to_set, PYTHONCOERCECLOCALE=coerce_c_locale): @@ -312,7 +327,7 @@ def _check_c_locale_coercion(self, self._check_child_encoding_details(var_dict, fs_encoding, stream_encoding, - expected_warnings, + _expected_warnings, coercion_expected) def test_test_PYTHONCOERCECLOCALE_not_set(self): diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index bb5b2a3b9f0d73..2a6de3c5aa9780 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -1,10 +1,9 @@ # Run the _testcapi module tests (tests for the Python/C API): by defn, # these are all functions _testcapi exports whose name begins with 'test_'. -from collections import namedtuple, OrderedDict +from collections import OrderedDict import os import pickle -import platform import random import re import subprocess @@ -420,180 +419,6 @@ def test(self): self.assertEqual(_testcapi.argparsing("Hello", "World"), 1) -class EmbeddingTests(unittest.TestCase): - def setUp(self): - here = os.path.abspath(__file__) - basepath = os.path.dirname(os.path.dirname(os.path.dirname(here))) - exename = "_testembed" - if sys.platform.startswith("win"): - ext = ("_d" if "_d" in sys.executable else "") + ".exe" - exename += ext - exepath = os.path.dirname(sys.executable) - else: - exepath = os.path.join(basepath, "Programs") - self.test_exe = exe = os.path.join(exepath, exename) - if not os.path.exists(exe): - self.skipTest("%r doesn't exist" % exe) - # This is needed otherwise we get a fatal error: - # "Py_Initialize: Unable to get the locale encoding - # LookupError: no codec search functions registered: can't find encoding" - self.oldcwd = os.getcwd() - os.chdir(basepath) - - def tearDown(self): - os.chdir(self.oldcwd) - - def run_embedded_interpreter(self, *args, env=None): - """Runs a test in the embedded interpreter""" - cmd = [self.test_exe] - cmd.extend(args) - if env is not None and sys.platform == 'win32': - # Windows requires at least the SYSTEMROOT environment variable to - # start Python. - env = env.copy() - env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] - - p = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=env) - (out, err) = p.communicate() - self.assertEqual(p.returncode, 0, - "bad returncode %d, stderr is %r" % - (p.returncode, err)) - return out, err - - def run_repeated_init_and_subinterpreters(self): - out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters") - self.assertEqual(err, "") - - # The output from _testembed looks like this: - # --- Pass 0 --- - # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 - # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 - # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 - # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 - # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 - # --- Pass 1 --- - # ... - - interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " - r"thread state <(0x[\dA-F]+)>: " - r"id\(modules\) = ([\d]+)$") - Interp = namedtuple("Interp", "id interp tstate modules") - - numloops = 0 - current_run = [] - for line in out.splitlines(): - if line == "--- Pass {} ---".format(numloops): - self.assertEqual(len(current_run), 0) - if support.verbose: - print(line) - numloops += 1 - continue - - self.assertLess(len(current_run), 5) - match = re.match(interp_pat, line) - if match is None: - self.assertRegex(line, interp_pat) - - # Parse the line from the loop. The first line is the main - # interpreter and the 3 afterward are subinterpreters. - interp = Interp(*match.groups()) - if support.verbose: - print(interp) - self.assertTrue(interp.interp) - self.assertTrue(interp.tstate) - self.assertTrue(interp.modules) - current_run.append(interp) - - # The last line in the loop should be the same as the first. - if len(current_run) == 5: - main = current_run[0] - self.assertEqual(interp, main) - yield current_run - current_run = [] - - def test_subinterps_main(self): - for run in self.run_repeated_init_and_subinterpreters(): - main = run[0] - - self.assertEqual(main.id, '0') - - def test_subinterps_different_ids(self): - for run in self.run_repeated_init_and_subinterpreters(): - main, *subs, _ = run - - mainid = int(main.id) - for i, sub in enumerate(subs): - self.assertEqual(sub.id, str(mainid + i + 1)) - - def test_subinterps_distinct_state(self): - for run in self.run_repeated_init_and_subinterpreters(): - main, *subs, _ = run - - if '0x0' in main: - # XXX Fix on Windows (and other platforms): something - # is going on with the pointers in Programs/_testembed.c. - # interp.interp is 0x0 and interp.modules is the same - # between interpreters. - raise unittest.SkipTest('platform prints pointers as 0x0') - - for sub in subs: - # A new subinterpreter may have the same - # PyInterpreterState pointer as a previous one if - # the earlier one has already been destroyed. So - # we compare with the main interpreter. The same - # applies to tstate. - self.assertNotEqual(sub.interp, main.interp) - self.assertNotEqual(sub.tstate, main.tstate) - self.assertNotEqual(sub.modules, main.modules) - - def test_forced_io_encoding(self): - # Checks forced configuration of embedded interpreter IO streams - env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") - out, err = self.run_embedded_interpreter("forced_io_encoding", env=env) - if support.verbose > 1: - print() - print(out) - print(err) - expected_stream_encoding = "utf-8" - expected_errors = "surrogateescape" - expected_output = '\n'.join([ - "--- Use defaults ---", - "Expected encoding: default", - "Expected errors: default", - "stdin: {in_encoding}:{errors}", - "stdout: {out_encoding}:{errors}", - "stderr: {out_encoding}:backslashreplace", - "--- Set errors only ---", - "Expected encoding: default", - "Expected errors: ignore", - "stdin: {in_encoding}:ignore", - "stdout: {out_encoding}:ignore", - "stderr: {out_encoding}:backslashreplace", - "--- Set encoding only ---", - "Expected encoding: latin-1", - "Expected errors: default", - "stdin: latin-1:{errors}", - "stdout: latin-1:{errors}", - "stderr: latin-1:backslashreplace", - "--- Set encoding and errors ---", - "Expected encoding: latin-1", - "Expected errors: replace", - "stdin: latin-1:replace", - "stdout: latin-1:replace", - "stderr: latin-1:backslashreplace"]) - expected_output = expected_output.format( - in_encoding=expected_stream_encoding, - out_encoding=expected_stream_encoding, - errors=expected_errors) - # This is useful if we ever trip over odd platform behaviour - self.maxDiff = None - self.assertEqual(out.strip(), expected_output) - - class SkipitemTest(unittest.TestCase): def test_skipitem(self): @@ -829,8 +654,7 @@ class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' -@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, - 'need pymalloc') +@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') class PyMemPymallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'pymalloc_debug' diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1b584ebb7c5ec3..383302bad8b549 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,14 +2,16 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution -import test.support, unittest import os -import sys import subprocess +import sys +import sysconfig import tempfile -from test.support import script_helper, is_android +import unittest +from test import support from test.support.script_helper import ( - spawn_python, kill_python, assert_python_ok, assert_python_failure + spawn_python, kill_python, assert_python_ok, assert_python_failure, + interpreter_requires_environment ) # XXX (ncoghlan): Move to script_helper and make consistent with run_python @@ -132,11 +134,11 @@ def test_run_code(self): # All good if execution is successful assert_python_ok('-c', 'pass') - @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII') + @unittest.skipUnless(support.FS_NONASCII, 'need support.FS_NONASCII') def test_non_ascii(self): # Test handling of non-ascii data command = ("assert(ord(%r) == %s)" - % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII))) + % (support.FS_NONASCII, ord(support.FS_NONASCII))) assert_python_ok('-c', command) # On Windows, pass bytes to subprocess doesn't test how Python decodes the @@ -179,7 +181,7 @@ def test_undecodable_code(self): raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) @unittest.skipUnless((sys.platform == 'darwin' or - is_android), 'test specific to Mac OS X and Android') + support.is_android), 'test specific to Mac OS X and Android') def test_osx_android_utf8(self): def check_output(text): decoded = text.decode('utf-8', 'surrogateescape') @@ -385,7 +387,7 @@ def preexec(): stderr=subprocess.PIPE, preexec_fn=preexec) out, err = p.communicate() - self.assertEqual(test.support.strip_python_stderr(err), b'') + self.assertEqual(support.strip_python_stderr(err), b'') self.assertEqual(p.returncode, 42) def test_no_stdin(self): @@ -433,8 +435,8 @@ def test_del___main__(self): # Issue #15001: PyRun_SimpleFileExFlags() did crash because it kept a # borrowed reference to the dict of __main__ module and later modify # the dict whereas the module was destroyed - filename = test.support.TESTFN - self.addCleanup(test.support.unlink, filename) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) with open(filename, "w") as script: print("import sys", file=script) print("del sys.modules['__main__']", file=script) @@ -458,7 +460,7 @@ def test_unknown_options(self): self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1) self.assertEqual(b'', out) - @unittest.skipIf(script_helper.interpreter_requires_environment(), + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') def test_isolatedmode(self): self.verify_valid_flag('-I') @@ -469,7 +471,7 @@ def test_isolatedmode(self): # dummyvar to prevent extraneous -E dummyvar="") self.assertEqual(out.strip(), b'1 1 1') - with test.support.temp_cwd() as tmpdir: + with support.temp_cwd() as tmpdir: fake = os.path.join(tmpdir, "uuid.py") main = os.path.join(tmpdir, "main.py") with open(fake, "w") as f: @@ -506,6 +508,153 @@ def test_sys_flags_set(self): with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) + def run_xdev(self, *args, check_exitcode=True, xdev=True): + env = dict(os.environ) + env.pop('PYTHONWARNINGS', None) + env.pop('PYTHONDEVMODE', None) + # Force malloc() to disable the debug hooks which are enabled + # by default for Python compiled in debug mode + env['PYTHONMALLOC'] = 'malloc' + + if xdev: + args = (sys.executable, '-X', 'dev', *args) + else: + args = (sys.executable, *args) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + if check_exitcode: + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() + + def test_xdev(self): + # sys.flags.dev_mode + code = "import sys; print(sys.flags.dev_mode)" + out = self.run_xdev("-c", code, xdev=False) + self.assertEqual(out, "False") + out = self.run_xdev("-c", code) + self.assertEqual(out, "True") + + # Warnings + code = ("import sys, warnings; " + "print(' '.join('%s::%s' % (f[0], f[2].__name__) " + "for f in warnings.filters))") + + out = self.run_xdev("-c", code) + self.assertEqual(out, + "ignore::BytesWarning " + "default::ResourceWarning " + "default::Warning") + + out = self.run_xdev("-b", "-c", code) + self.assertEqual(out, + "default::BytesWarning " + "default::ResourceWarning " + "default::Warning") + + out = self.run_xdev("-bb", "-c", code) + self.assertEqual(out, + "error::BytesWarning " + "default::ResourceWarning " + "default::Warning") + + out = self.run_xdev("-Werror", "-c", code) + self.assertEqual(out, + "error::Warning " + "ignore::BytesWarning " + "default::ResourceWarning " + "default::Warning") + + # Memory allocator debug hooks + try: + import _testcapi + except ImportError: + pass + else: + code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())" + with support.SuppressCrashReport(): + out = self.run_xdev("-c", code, check_exitcode=False) + if support.with_pymalloc(): + alloc_name = "pymalloc_debug" + else: + alloc_name = "malloc_debug" + self.assertEqual(out, alloc_name) + + # Faulthandler + try: + import faulthandler + except ImportError: + pass + else: + code = "import faulthandler; print(faulthandler.is_enabled())" + out = self.run_xdev("-c", code) + self.assertEqual(out, "True") + + def check_pythonmalloc(self, env_var, name): + code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + if env_var is not None: + env['PYTHONMALLOC'] = env_var + else: + env.pop('PYTHONMALLOC', None) + args = (sys.executable, '-c', code) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.stdout.rstrip(), name) + self.assertEqual(proc.returncode, 0) + + def test_pythonmalloc(self): + # Test the PYTHONMALLOC environment variable + pydebug = hasattr(sys, "gettotalrefcount") + pymalloc = support.with_pymalloc() + if pymalloc: + default_name = 'pymalloc_debug' if pydebug else 'pymalloc' + default_name_debug = 'pymalloc_debug' + else: + default_name = 'malloc_debug' if pydebug else 'malloc' + default_name_debug = 'malloc_debug' + + tests = [ + (None, default_name), + ('debug', default_name_debug), + ('malloc', 'malloc'), + ('malloc_debug', 'malloc_debug'), + ] + if pymalloc: + tests.extend(( + ('pymalloc', 'pymalloc'), + ('pymalloc_debug', 'pymalloc_debug'), + )) + + for env_var, name in tests: + with self.subTest(env_var=env_var, name=name): + self.check_pythonmalloc(env_var, name) + + def test_pythondevmode_env(self): + # Test the PYTHONDEVMODE environment variable + code = "import sys; print(sys.flags.dev_mode)" + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + args = (sys.executable, '-c', code) + + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'False') + self.assertEqual(proc.returncode, 0, proc) + + env['PYTHONDEVMODE'] = '1' + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'True') + self.assertEqual(proc.returncode, 0, proc) + + class IgnoreEnvironmentTest(unittest.TestCase): def run_ignoring_vars(self, predicate, **env_vars): @@ -541,8 +690,8 @@ def test_sys_flags_not_set(self): def test_main(): - test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest) - test.support.reap_children() + support.run_unittest(CmdLineTest, IgnoreEnvironmentTest) + support.reap_children() if __name__ == "__main__": test_main() diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 90cb584ac40a01..55faf4c4279fb6 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -102,6 +102,7 @@ """ +import inspect import sys import threading import unittest @@ -130,6 +131,10 @@ def dump(co): print("%s: %s" % (attr, getattr(co, "co_" + attr))) print("consts:", tuple(consts(co.co_consts))) +# Needed for test_closure_injection below +# Defined at global scope to avoid implicitly closing over __class__ +def external_getitem(self, i): + return f"Foreign getitem: {super().__getitem__(i)}" class CodeTest(unittest.TestCase): @@ -141,6 +146,46 @@ def test_newempty(self): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) + @cpython_only + def test_closure_injection(self): + # From https://bugs.python.org/issue32176 + from types import FunctionType, CodeType + + def create_closure(__class__): + return (lambda: __class__).__closure__ + + def new_code(c): + '''A new code object with a __class__ cell added to freevars''' + return CodeType( + c.co_argcount, c.co_kwonlyargcount, c.co_nlocals, + c.co_stacksize, c.co_flags, c.co_code, c.co_consts, c.co_names, + c.co_varnames, c.co_filename, c.co_name, c.co_firstlineno, + c.co_lnotab, c.co_freevars + ('__class__',), c.co_cellvars) + + def add_foreign_method(cls, name, f): + code = new_code(f.__code__) + assert not f.__closure__ + closure = create_closure(cls) + defaults = f.__defaults__ + setattr(cls, name, FunctionType(code, globals(), name, defaults, closure)) + + class List(list): + pass + + add_foreign_method(List, "__getitem__", external_getitem) + + # Ensure the closure injection actually worked + function = List.__getitem__ + class_ref = function.__closure__[0].cell_contents + self.assertIs(class_ref, List) + + # Ensure the code correctly indicates it accesses a free variable + self.assertFalse(function.__code__.co_flags & inspect.CO_NOFREE, + hex(function.__code__.co_flags)) + + # Ensure the zero-arg super() call in the injected method works + obj = List([1, 2, 3]) + self.assertEqual(obj[0], "Foreign getitem: 1") def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index de6868a46c4746..eb21a3915b938a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -196,19 +196,33 @@ def getreader(): self.assertEqual(f.read(), ''.join(lines[1:])) self.assertEqual(f.read(), '') + # Issue #32110: Test readline() followed by read(n) + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.read(1), lines[1][0]) + self.assertEqual(f.read(0), '') + self.assertEqual(f.read(100), data[len(lines[0]) + 1:][:100]) + # Issue #16636: Test readline() followed by readlines() f = getreader() self.assertEqual(f.readline(), lines[0]) self.assertEqual(f.readlines(), lines[1:]) self.assertEqual(f.read(), '') - # Test read() followed by read() + # Test read(n) followed by read() f = getreader() self.assertEqual(f.read(size=40, chars=5), data[:5]) self.assertEqual(f.read(), data[5:]) self.assertEqual(f.read(), '') - # Issue #12446: Test read() followed by readlines() + # Issue #32110: Test read(n) followed by read(n) + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.read(1), data[5]) + self.assertEqual(f.read(0), '') + self.assertEqual(f.read(100), data[6:106]) + + # Issue #12446: Test read(n) followed by readlines() f = getreader() self.assertEqual(f.read(size=40, chars=5), data[:5]) self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:]) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 64b6578ff94eb3..1a5e6edad9b28f 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -252,6 +252,16 @@ def close(self): 1 / 0 self.assertEqual(state, [1]) + +class NullcontextTestCase(unittest.TestCase): + def test_nullcontext(self): + class C: + pass + c = C() + with nullcontext(c) as c_in: + self.assertIs(c_in, c) + + class FileContextTestCase(unittest.TestCase): def testWithOpen(self): diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index 796fd076c6a5e5..d9189fc6f28b47 100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -39,12 +39,26 @@ def test_methods(self): else: self.assertEqual(crypt.methods[-1], crypt.METHOD_CRYPT) + @unittest.skipUnless(crypt.METHOD_SHA256 in crypt.methods or + crypt.METHOD_SHA512 in crypt.methods, + 'requires support of SHA-2') + def test_sha2_rounds(self): + for method in (crypt.METHOD_SHA256, crypt.METHOD_SHA512): + for rounds in 1000, 10_000, 100_000: + salt = crypt.mksalt(method, rounds=rounds) + self.assertIn('$rounds=%d$' % rounds, salt) + self.assertEqual(len(salt) - method.salt_chars, + 11 + len(str(rounds))) + cr = crypt.crypt('mypassword', salt) + self.assertTrue(cr) + cr2 = crypt.crypt('mypassword', cr) + self.assertEqual(cr2, cr) + @unittest.skipUnless(crypt.METHOD_BLOWFISH in crypt.methods, 'requires support of Blowfish') - def test_log_rounds(self): - self.assertEqual(len(crypt._saltchars), 64) + def test_blowfish_rounds(self): for log_rounds in range(4, 11): - salt = crypt.mksalt(crypt.METHOD_BLOWFISH, log_rounds=log_rounds) + salt = crypt.mksalt(crypt.METHOD_BLOWFISH, rounds=1 << log_rounds) self.assertIn('$%02d$' % log_rounds, salt) self.assertIn(len(salt) - crypt.METHOD_BLOWFISH.salt_chars, {6, 7}) cr = crypt.crypt('mypassword', salt) @@ -52,18 +66,21 @@ def test_log_rounds(self): cr2 = crypt.crypt('mypassword', cr) self.assertEqual(cr2, cr) - @unittest.skipUnless(crypt.METHOD_BLOWFISH in crypt.methods, - 'requires support of Blowfish') - def test_invalid_log_rounds(self): - for log_rounds in (1, -1, 999): - salt = crypt.mksalt(crypt.METHOD_BLOWFISH, log_rounds=log_rounds) - cr = crypt.crypt('mypassword', salt) - if cr is not None: - # On failure the openwall implementation returns a magic - # string that is shorter than 13 characters and is guaranteed - # to differ from a salt. - self.assertNotEqual(cr, salt) - self.assertLess(len(cr), 13) + def test_invalid_rounds(self): + for method in (crypt.METHOD_SHA256, crypt.METHOD_SHA512, + crypt.METHOD_BLOWFISH): + with self.assertRaises(TypeError): + crypt.mksalt(method, rounds='4096') + with self.assertRaises(TypeError): + crypt.mksalt(method, rounds=4096.0) + for rounds in (0, 1, -1, 1<<999): + with self.assertRaises(ValueError): + crypt.mksalt(method, rounds=rounds) + with self.assertRaises(ValueError): + crypt.mksalt(crypt.METHOD_BLOWFISH, rounds=1000) + for method in (crypt.METHOD_CRYPT, crypt.METHOD_MD5): + with self.assertRaisesRegex(ValueError, 'support'): + crypt.mksalt(method, rounds=4096) if __name__ == "__main__": diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py new file mode 100755 index 00000000000000..caea98a13b06b7 --- /dev/null +++ b/Lib/test/test_dataclasses.py @@ -0,0 +1,2076 @@ +from dataclasses import ( + dataclass, field, FrozenInstanceError, fields, asdict, astuple, + make_dataclass, replace, InitVar, Field +) + +import pickle +import inspect +import unittest +from unittest.mock import Mock +from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar +from collections import deque, OrderedDict, namedtuple + +# Just any custom exception we can catch. +class CustomError(Exception): pass + +class TestCase(unittest.TestCase): + def test_no_fields(self): + @dataclass + class C: + pass + + o = C() + self.assertEqual(len(fields(C)), 0) + + def test_one_field_no_default(self): + @dataclass + class C: + x: int + + o = C(42) + self.assertEqual(o.x, 42) + + def test_named_init_params(self): + @dataclass + class C: + x: int + + o = C(x=32) + self.assertEqual(o.x, 32) + + def test_two_fields_one_default(self): + @dataclass + class C: + x: int + y: int = 0 + + o = C(3) + self.assertEqual((o.x, o.y), (3, 0)) + + # Non-defaults following defaults. + with self.assertRaisesRegex(TypeError, + "non-default argument 'y' follows " + "default argument"): + @dataclass + class C: + x: int = 0 + y: int + + # A derived class adds a non-default field after a default one. + with self.assertRaisesRegex(TypeError, + "non-default argument 'y' follows " + "default argument"): + @dataclass + class B: + x: int = 0 + + @dataclass + class C(B): + y: int + + # Override a base class field and add a default to + # a field which didn't use to have a default. + with self.assertRaisesRegex(TypeError, + "non-default argument 'y' follows " + "default argument"): + @dataclass + class B: + x: int + y: int + + @dataclass + class C(B): + x: int = 0 + + def test_overwriting_init(self): + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __init__ ' + 'in C'): + @dataclass + class C: + x: int + def __init__(self, x): + self.x = 2 * x + + @dataclass(init=False) + class C: + x: int + def __init__(self, x): + self.x = 2 * x + self.assertEqual(C(5).x, 10) + + def test_overwriting_repr(self): + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __repr__ ' + 'in C'): + @dataclass + class C: + x: int + def __repr__(self): + pass + + @dataclass(repr=False) + class C: + x: int + def __repr__(self): + return 'x' + self.assertEqual(repr(C(0)), 'x') + + def test_overwriting_cmp(self): + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __eq__ ' + 'in C'): + # This will generate the comparison functions, make sure we can't + # overwrite them. + @dataclass(hash=False, frozen=False) + class C: + x: int + def __eq__(self): + pass + + @dataclass(order=False, eq=False) + class C: + x: int + def __eq__(self, other): + return True + self.assertEqual(C(0), 'x') + + def test_overwriting_hash(self): + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __hash__ ' + 'in C'): + @dataclass(frozen=True) + class C: + x: int + def __hash__(self): + pass + + @dataclass(frozen=True,hash=False) + class C: + x: int + def __hash__(self): + return 600 + self.assertEqual(hash(C(0)), 600) + + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __hash__ ' + 'in C'): + @dataclass(frozen=True) + class C: + x: int + def __hash__(self): + pass + + @dataclass(frozen=True, hash=False) + class C: + x: int + def __hash__(self): + return 600 + self.assertEqual(hash(C(0)), 600) + + def test_overwriting_frozen(self): + # frozen uses __setattr__ and __delattr__ + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __setattr__ ' + 'in C'): + @dataclass(frozen=True) + class C: + x: int + def __setattr__(self): + pass + + with self.assertRaisesRegex(TypeError, + 'Cannot overwrite attribute __delattr__ ' + 'in C'): + @dataclass(frozen=True) + class C: + x: int + def __delattr__(self): + pass + + @dataclass(frozen=False) + class C: + x: int + def __setattr__(self, name, value): + self.__dict__['x'] = value * 2 + self.assertEqual(C(10).x, 20) + + def test_overwrite_fields_in_derived_class(self): + # Note that x from C1 replaces x in Base, but the order remains + # the same as defined in Base. + @dataclass + class Base: + x: Any = 15.0 + y: int = 0 + + @dataclass + class C1(Base): + z: int = 10 + x: int = 15 + + o = Base() + self.assertEqual(repr(o), 'TestCase.test_overwrite_fields_in_derived_class..Base(x=15.0, y=0)') + + o = C1() + self.assertEqual(repr(o), 'TestCase.test_overwrite_fields_in_derived_class..C1(x=15, y=0, z=10)') + + o = C1(x=5) + self.assertEqual(repr(o), 'TestCase.test_overwrite_fields_in_derived_class..C1(x=5, y=0, z=10)') + + def test_field_named_self(self): + @dataclass + class C: + self: str + c=C('foo') + self.assertEqual(c.self, 'foo') + + # Make sure the first parameter is not named 'self'. + sig = inspect.signature(C.__init__) + first = next(iter(sig.parameters)) + self.assertNotEqual('self', first) + + # But we do use 'self' if no field named self. + @dataclass + class C: + selfx: str + + # Make sure the first parameter is named 'self'. + sig = inspect.signature(C.__init__) + first = next(iter(sig.parameters)) + self.assertEqual('self', first) + + def test_repr(self): + @dataclass + class B: + x: int + + @dataclass + class C(B): + y: int = 10 + + o = C(4) + self.assertEqual(repr(o), 'TestCase.test_repr..C(x=4, y=10)') + + @dataclass + class D(C): + x: int = 20 + self.assertEqual(repr(D()), 'TestCase.test_repr..D(x=20, y=10)') + + @dataclass + class C: + @dataclass + class D: + i: int + @dataclass + class E: + pass + self.assertEqual(repr(C.D(0)), 'TestCase.test_repr..C.D(i=0)') + self.assertEqual(repr(C.E()), 'TestCase.test_repr..C.E()') + + def test_0_field_compare(self): + # Ensure that order=False is the default. + @dataclass + class C0: + pass + + @dataclass(order=False) + class C1: + pass + + for cls in [C0, C1]: + with self.subTest(cls=cls): + self.assertEqual(cls(), cls()) + for idx, fn in enumerate([lambda a, b: a < b, + lambda a, b: a <= b, + lambda a, b: a > b, + lambda a, b: a >= b]): + with self.subTest(idx=idx): + with self.assertRaisesRegex(TypeError, + f"not supported between instances of '{cls.__name__}' and '{cls.__name__}'"): + fn(cls(), cls()) + + @dataclass(order=True) + class C: + pass + self.assertLessEqual(C(), C()) + self.assertGreaterEqual(C(), C()) + + def test_1_field_compare(self): + # Ensure that order=False is the default. + @dataclass + class C0: + x: int + + @dataclass(order=False) + class C1: + x: int + + for cls in [C0, C1]: + with self.subTest(cls=cls): + self.assertEqual(cls(1), cls(1)) + self.assertNotEqual(cls(0), cls(1)) + for idx, fn in enumerate([lambda a, b: a < b, + lambda a, b: a <= b, + lambda a, b: a > b, + lambda a, b: a >= b]): + with self.subTest(idx=idx): + with self.assertRaisesRegex(TypeError, + f"not supported between instances of '{cls.__name__}' and '{cls.__name__}'"): + fn(cls(0), cls(0)) + + @dataclass(order=True) + class C: + x: int + self.assertLess(C(0), C(1)) + self.assertLessEqual(C(0), C(1)) + self.assertLessEqual(C(1), C(1)) + self.assertGreater(C(1), C(0)) + self.assertGreaterEqual(C(1), C(0)) + self.assertGreaterEqual(C(1), C(1)) + + def test_simple_compare(self): + # Ensure that order=False is the default. + @dataclass + class C0: + x: int + y: int + + @dataclass(order=False) + class C1: + x: int + y: int + + for cls in [C0, C1]: + with self.subTest(cls=cls): + self.assertEqual(cls(0, 0), cls(0, 0)) + self.assertEqual(cls(1, 2), cls(1, 2)) + self.assertNotEqual(cls(1, 0), cls(0, 0)) + self.assertNotEqual(cls(1, 0), cls(1, 1)) + for idx, fn in enumerate([lambda a, b: a < b, + lambda a, b: a <= b, + lambda a, b: a > b, + lambda a, b: a >= b]): + with self.subTest(idx=idx): + with self.assertRaisesRegex(TypeError, + f"not supported between instances of '{cls.__name__}' and '{cls.__name__}'"): + fn(cls(0, 0), cls(0, 0)) + + @dataclass(order=True) + class C: + x: int + y: int + + for idx, fn in enumerate([lambda a, b: a == b, + lambda a, b: a <= b, + lambda a, b: a >= b]): + with self.subTest(idx=idx): + self.assertTrue(fn(C(0, 0), C(0, 0))) + + for idx, fn in enumerate([lambda a, b: a < b, + lambda a, b: a <= b, + lambda a, b: a != b]): + with self.subTest(idx=idx): + self.assertTrue(fn(C(0, 0), C(0, 1))) + self.assertTrue(fn(C(0, 1), C(1, 0))) + self.assertTrue(fn(C(1, 0), C(1, 1))) + + for idx, fn in enumerate([lambda a, b: a > b, + lambda a, b: a >= b, + lambda a, b: a != b]): + with self.subTest(idx=idx): + self.assertTrue(fn(C(0, 1), C(0, 0))) + self.assertTrue(fn(C(1, 0), C(0, 1))) + self.assertTrue(fn(C(1, 1), C(1, 0))) + + def test_compare_subclasses(self): + # Comparisons fail for subclasses, even if no fields + # are added. + @dataclass + class B: + i: int + + @dataclass + class C(B): + pass + + for idx, (fn, expected) in enumerate([(lambda a, b: a == b, False), + (lambda a, b: a != b, True)]): + with self.subTest(idx=idx): + self.assertEqual(fn(B(0), C(0)), expected) + + for idx, fn in enumerate([lambda a, b: a < b, + lambda a, b: a <= b, + lambda a, b: a > b, + lambda a, b: a >= b]): + with self.subTest(idx=idx): + with self.assertRaisesRegex(TypeError, + "not supported between instances of 'B' and 'C'"): + fn(B(0), C(0)) + + def test_0_field_hash(self): + @dataclass(hash=True) + class C: + pass + self.assertEqual(hash(C()), hash(())) + + def test_1_field_hash(self): + @dataclass(hash=True) + class C: + x: int + self.assertEqual(hash(C(4)), hash((4,))) + self.assertEqual(hash(C(42)), hash((42,))) + + def test_hash(self): + @dataclass(hash=True) + class C: + x: int + y: str + self.assertEqual(hash(C(1, 'foo')), hash((1, 'foo'))) + + def test_no_hash(self): + @dataclass(hash=None) + class C: + x: int + with self.assertRaisesRegex(TypeError, + "unhashable type: 'C'"): + hash(C(1)) + + def test_hash_rules(self): + # There are 24 cases of: + # hash=True/False/None + # eq=True/False + # order=True/False + # frozen=True/False + for (hash, eq, order, frozen, result ) in [ + (False, False, False, False, 'absent'), + (False, False, False, True, 'absent'), + (False, False, True, False, 'exception'), + (False, False, True, True, 'exception'), + (False, True, False, False, 'absent'), + (False, True, False, True, 'absent'), + (False, True, True, False, 'absent'), + (False, True, True, True, 'absent'), + (True, False, False, False, 'fn'), + (True, False, False, True, 'fn'), + (True, False, True, False, 'exception'), + (True, False, True, True, 'exception'), + (True, True, False, False, 'fn'), + (True, True, False, True, 'fn'), + (True, True, True, False, 'fn'), + (True, True, True, True, 'fn'), + (None, False, False, False, 'absent'), + (None, False, False, True, 'absent'), + (None, False, True, False, 'exception'), + (None, False, True, True, 'exception'), + (None, True, False, False, 'none'), + (None, True, False, True, 'fn'), + (None, True, True, False, 'none'), + (None, True, True, True, 'fn'), + ]: + with self.subTest(hash=hash, eq=eq, order=order, frozen=frozen): + if result == 'exception': + with self.assertRaisesRegex(ValueError, 'eq must be true if order is true'): + @dataclass(hash=hash, eq=eq, order=order, frozen=frozen) + class C: + pass + else: + @dataclass(hash=hash, eq=eq, order=order, frozen=frozen) + class C: + pass + + # See if the result matches what's expected. + if result == 'fn': + # __hash__ contains the function we generated. + self.assertIn('__hash__', C.__dict__) + self.assertIsNotNone(C.__dict__['__hash__']) + elif result == 'absent': + # __hash__ is not present in our class. + self.assertNotIn('__hash__', C.__dict__) + elif result == 'none': + # __hash__ is set to None. + self.assertIn('__hash__', C.__dict__) + self.assertIsNone(C.__dict__['__hash__']) + else: + assert False, f'unknown result {result!r}' + + def test_eq_order(self): + for (eq, order, result ) in [ + (False, False, 'neither'), + (False, True, 'exception'), + (True, False, 'eq_only'), + (True, True, 'both'), + ]: + with self.subTest(eq=eq, order=order): + if result == 'exception': + with self.assertRaisesRegex(ValueError, 'eq must be true if order is true'): + @dataclass(eq=eq, order=order) + class C: + pass + else: + @dataclass(eq=eq, order=order) + class C: + pass + + if result == 'neither': + self.assertNotIn('__eq__', C.__dict__) + self.assertNotIn('__ne__', C.__dict__) + self.assertNotIn('__lt__', C.__dict__) + self.assertNotIn('__le__', C.__dict__) + self.assertNotIn('__gt__', C.__dict__) + self.assertNotIn('__ge__', C.__dict__) + elif result == 'both': + self.assertIn('__eq__', C.__dict__) + self.assertIn('__ne__', C.__dict__) + self.assertIn('__lt__', C.__dict__) + self.assertIn('__le__', C.__dict__) + self.assertIn('__gt__', C.__dict__) + self.assertIn('__ge__', C.__dict__) + elif result == 'eq_only': + self.assertIn('__eq__', C.__dict__) + self.assertIn('__ne__', C.__dict__) + self.assertNotIn('__lt__', C.__dict__) + self.assertNotIn('__le__', C.__dict__) + self.assertNotIn('__gt__', C.__dict__) + self.assertNotIn('__ge__', C.__dict__) + else: + assert False, f'unknown result {result!r}' + + def test_field_no_default(self): + @dataclass + class C: + x: int = field() + + self.assertEqual(C(5).x, 5) + + with self.assertRaisesRegex(TypeError, + r"__init__\(\) missing 1 required " + "positional argument: 'x'"): + C() + + def test_field_default(self): + default = object() + @dataclass + class C: + x: object = field(default=default) + + self.assertIs(C.x, default) + c = C(10) + self.assertEqual(c.x, 10) + + # If we delete the instance attribute, we should then see the + # class attribute. + del c.x + self.assertIs(c.x, default) + + self.assertIs(C().x, default) + + def test_not_in_repr(self): + @dataclass + class C: + x: int = field(repr=False) + with self.assertRaises(TypeError): + C() + c = C(10) + self.assertEqual(repr(c), 'TestCase.test_not_in_repr..C()') + + @dataclass + class C: + x: int = field(repr=False) + y: int + c = C(10, 20) + self.assertEqual(repr(c), 'TestCase.test_not_in_repr..C(y=20)') + + def test_not_in_compare(self): + @dataclass + class C: + x: int = 0 + y: int = field(compare=False, default=4) + + self.assertEqual(C(), C(0, 20)) + self.assertEqual(C(1, 10), C(1, 20)) + self.assertNotEqual(C(3), C(4, 10)) + self.assertNotEqual(C(3, 10), C(4, 10)) + + def test_hash_field_rules(self): + # Test all 6 cases of: + # hash=True/False/None + # compare=True/False + for (hash_val, compare, result ) in [ + (True, False, 'field' ), + (True, True, 'field' ), + (False, False, 'absent'), + (False, True, 'absent'), + (None, False, 'absent'), + (None, True, 'field' ), + ]: + with self.subTest(hash_val=hash_val, compare=compare): + @dataclass(hash=True) + class C: + x: int = field(compare=compare, hash=hash_val, default=5) + + if result == 'field': + # __hash__ contains the field. + self.assertEqual(C(5).__hash__(), hash((5,))) + elif result == 'absent': + # The field is not present in the hash. + self.assertEqual(C(5).__hash__(), hash(())) + else: + assert False, f'unknown result {result!r}' + + def test_init_false_no_default(self): + # If init=False and no default value, then the field won't be + # present in the instance. + @dataclass + class C: + x: int = field(init=False) + + self.assertNotIn('x', C().__dict__) + + @dataclass + class C: + x: int + y: int = 0 + z: int = field(init=False) + t: int = 10 + + self.assertNotIn('z', C(0).__dict__) + self.assertEqual(vars(C(5)), {'t': 10, 'x': 5, 'y': 0}) + + def test_class_marker(self): + @dataclass + class C: + x: int + y: str = field(init=False, default=None) + z: str = field(repr=False) + + the_fields = fields(C) + # the_fields is a tuple of 3 items, each value + # is in __annotations__. + self.assertIsInstance(the_fields, tuple) + for f in the_fields: + self.assertIs(type(f), Field) + self.assertIn(f.name, C.__annotations__) + + self.assertEqual(len(the_fields), 3) + + self.assertEqual(the_fields[0].name, 'x') + self.assertEqual(the_fields[0].type, int) + self.assertFalse(hasattr(C, 'x')) + self.assertTrue (the_fields[0].init) + self.assertTrue (the_fields[0].repr) + self.assertEqual(the_fields[1].name, 'y') + self.assertEqual(the_fields[1].type, str) + self.assertIsNone(getattr(C, 'y')) + self.assertFalse(the_fields[1].init) + self.assertTrue (the_fields[1].repr) + self.assertEqual(the_fields[2].name, 'z') + self.assertEqual(the_fields[2].type, str) + self.assertFalse(hasattr(C, 'z')) + self.assertTrue (the_fields[2].init) + self.assertFalse(the_fields[2].repr) + + def test_field_order(self): + @dataclass + class B: + a: str = 'B:a' + b: str = 'B:b' + c: str = 'B:c' + + @dataclass + class C(B): + b: str = 'C:b' + + self.assertEqual([(f.name, f.default) for f in fields(C)], + [('a', 'B:a'), + ('b', 'C:b'), + ('c', 'B:c')]) + + @dataclass + class D(B): + c: str = 'D:c' + + self.assertEqual([(f.name, f.default) for f in fields(D)], + [('a', 'B:a'), + ('b', 'B:b'), + ('c', 'D:c')]) + + @dataclass + class E(D): + a: str = 'E:a' + d: str = 'E:d' + + self.assertEqual([(f.name, f.default) for f in fields(E)], + [('a', 'E:a'), + ('b', 'B:b'), + ('c', 'D:c'), + ('d', 'E:d')]) + + def test_class_attrs(self): + # We only have a class attribute if a default value is + # specified, either directly or via a field with a default. + default = object() + @dataclass + class C: + x: int + y: int = field(repr=False) + z: object = default + t: int = field(default=100) + + self.assertFalse(hasattr(C, 'x')) + self.assertFalse(hasattr(C, 'y')) + self.assertIs (C.z, default) + self.assertEqual(C.t, 100) + + def test_disallowed_mutable_defaults(self): + # For the known types, don't allow mutable default values. + for typ, empty, non_empty in [(list, [], [1]), + (dict, {}, {0:1}), + (set, set(), set([1])), + ]: + with self.subTest(typ=typ): + # Can't use a zero-length value. + with self.assertRaisesRegex(ValueError, + f'mutable default {typ} for field ' + 'x is not allowed'): + @dataclass + class Point: + x: typ = empty + + + # Nor a non-zero-length value + with self.assertRaisesRegex(ValueError, + f'mutable default {typ} for field ' + 'y is not allowed'): + @dataclass + class Point: + y: typ = non_empty + + # Check subtypes also fail. + class Subclass(typ): pass + + with self.assertRaisesRegex(ValueError, + f"mutable default .*Subclass'>" + ' for field z is not allowed' + ): + @dataclass + class Point: + z: typ = Subclass() + + # Because this is a ClassVar, it can be mutable. + @dataclass + class C: + z: ClassVar[typ] = typ() + + # Because this is a ClassVar, it can be mutable. + @dataclass + class C: + x: ClassVar[typ] = Subclass() + + + def test_deliberately_mutable_defaults(self): + # If a mutable default isn't in the known list of + # (list, dict, set), then it's okay. + class Mutable: + def __init__(self): + self.l = [] + + @dataclass + class C: + x: Mutable + + # These 2 instances will share this value of x. + lst = Mutable() + o1 = C(lst) + o2 = C(lst) + self.assertEqual(o1, o2) + o1.x.l.extend([1, 2]) + self.assertEqual(o1, o2) + self.assertEqual(o1.x.l, [1, 2]) + self.assertIs(o1.x, o2.x) + + def test_no_options(self): + # call with dataclass() + @dataclass() + class C: + x: int + + self.assertEqual(C(42).x, 42) + + def test_not_tuple(self): + # Make sure we can't be compared to a tuple. + @dataclass + class Point: + x: int + y: int + self.assertNotEqual(Point(1, 2), (1, 2)) + + # And that we can't compare to another unrelated dataclass + @dataclass + class C: + x: int + y: int + self.assertNotEqual(Point(1, 3), C(1, 3)) + + def test_base_has_init(self): + class B: + def __init__(self): + pass + + # Make sure that declaring this class doesn't raise an error. + # The issue is that we can't override __init__ in our class, + # but it should be okay to add __init__ to us if our base has + # an __init__. + @dataclass + class C(B): + x: int = 0 + + def test_frozen(self): + @dataclass(frozen=True) + class C: + i: int + + c = C(10) + self.assertEqual(c.i, 10) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertEqual(c.i, 10) + + # Check that a derived class is still frozen, even if not + # marked so. + @dataclass + class D(C): + pass + + d = D(20) + self.assertEqual(d.i, 20) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + self.assertEqual(d.i, 20) + + def test_not_tuple(self): + # Test that some of the problems with namedtuple don't happen + # here. + @dataclass + class Point3D: + x: int + y: int + z: int + + @dataclass + class Date: + year: int + month: int + day: int + + self.assertNotEqual(Point3D(2017, 6, 3), Date(2017, 6, 3)) + self.assertNotEqual(Point3D(1, 2, 3), (1, 2, 3)) + + # Make sure we can't unpack + with self.assertRaisesRegex(TypeError, 'is not iterable'): + x, y, z = Point3D(4, 5, 6) + + # Maka sure another class with the same field names isn't + # equal. + @dataclass + class Point3Dv1: + x: int = 0 + y: int = 0 + z: int = 0 + self.assertNotEqual(Point3D(0, 0, 0), Point3Dv1()) + + def test_function_annotations(self): + # Some dummy class and instance to use as a default. + class F: + pass + f = F() + + def validate_class(cls): + # First, check __annotations__, even though they're not + # function annotations. + self.assertEqual(cls.__annotations__['i'], int) + self.assertEqual(cls.__annotations__['j'], str) + self.assertEqual(cls.__annotations__['k'], F) + self.assertEqual(cls.__annotations__['l'], float) + self.assertEqual(cls.__annotations__['z'], complex) + + # Verify __init__. + + signature = inspect.signature(cls.__init__) + # Check the return type, should be None + self.assertIs(signature.return_annotation, None) + + # Check each parameter. + params = iter(signature.parameters.values()) + param = next(params) + # This is testing an internal name, and probably shouldn't be tested. + self.assertEqual(param.name, 'self') + param = next(params) + self.assertEqual(param.name, 'i') + self.assertIs (param.annotation, int) + self.assertEqual(param.default, inspect.Parameter.empty) + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) + param = next(params) + self.assertEqual(param.name, 'j') + self.assertIs (param.annotation, str) + self.assertEqual(param.default, inspect.Parameter.empty) + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) + param = next(params) + self.assertEqual(param.name, 'k') + self.assertIs (param.annotation, F) + # Don't test for the default, since it's set to _MISSING + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) + param = next(params) + self.assertEqual(param.name, 'l') + self.assertIs (param.annotation, float) + # Don't test for the default, since it's set to _MISSING + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) + self.assertRaises(StopIteration, next, params) + + + @dataclass + class C: + i: int + j: str + k: F = f + l: float=field(default=None) + z: complex=field(default=3+4j, init=False) + + validate_class(C) + + # Now repeat with __hash__. + @dataclass(frozen=True, hash=True) + class C: + i: int + j: str + k: F = f + l: float=field(default=None) + z: complex=field(default=3+4j, init=False) + + validate_class(C) + + def test_dont_include_other_annotations(self): + @dataclass + class C: + i: int + def foo(self) -> int: + return 4 + @property + def bar(self) -> int: + return 5 + self.assertEqual(list(C.__annotations__), ['i']) + self.assertEqual(C(10).foo(), 4) + self.assertEqual(C(10).bar, 5) + + def test_post_init(self): + # Just make sure it gets called + @dataclass + class C: + def __post_init__(self): + raise CustomError() + with self.assertRaises(CustomError): + C() + + @dataclass + class C: + i: int = 10 + def __post_init__(self): + if self.i == 10: + raise CustomError() + with self.assertRaises(CustomError): + C() + # post-init gets called, but doesn't raise. This is just + # checking that self is used correctly. + C(5) + + # If there's not an __init__, then post-init won't get called. + @dataclass(init=False) + class C: + def __post_init__(self): + raise CustomError() + # Creating the class won't raise + C() + + @dataclass + class C: + x: int = 0 + def __post_init__(self): + self.x *= 2 + self.assertEqual(C().x, 0) + self.assertEqual(C(2).x, 4) + + # Make sure that if we'r frozen, post-init can't set + # attributes. + @dataclass(frozen=True) + class C: + x: int = 0 + def __post_init__(self): + self.x *= 2 + with self.assertRaises(FrozenInstanceError): + C() + + def test_post_init_super(self): + # Make sure super() post-init isn't called by default. + class B: + def __post_init__(self): + raise CustomError() + + @dataclass + class C(B): + def __post_init__(self): + self.x = 5 + + self.assertEqual(C().x, 5) + + # Now call super(), and it will raise + @dataclass + class C(B): + def __post_init__(self): + super().__post_init__() + + with self.assertRaises(CustomError): + C() + + # Make sure post-init is called, even if not defined in our + # class. + @dataclass + class C(B): + pass + + with self.assertRaises(CustomError): + C() + + def test_post_init_staticmethod(self): + flag = False + @dataclass + class C: + x: int + y: int + @staticmethod + def __post_init__(): + nonlocal flag + flag = True + + self.assertFalse(flag) + c = C(3, 4) + self.assertEqual((c.x, c.y), (3, 4)) + self.assertTrue(flag) + + def test_post_init_classmethod(self): + @dataclass + class C: + flag = False + x: int + y: int + @classmethod + def __post_init__(cls): + cls.flag = True + + self.assertFalse(C.flag) + c = C(3, 4) + self.assertEqual((c.x, c.y), (3, 4)) + self.assertTrue(C.flag) + + def test_class_var(self): + # Make sure ClassVars are ignored in __init__, __repr__, etc. + @dataclass + class C: + x: int + y: int = 10 + z: ClassVar[int] = 1000 + w: ClassVar[int] = 2000 + t: ClassVar[int] = 3000 + + c = C(5) + self.assertEqual(repr(c), 'TestCase.test_class_var..C(x=5, y=10)') + self.assertEqual(len(fields(C)), 2) # We have 2 fields + self.assertEqual(len(C.__annotations__), 5) # And 3 ClassVars + self.assertEqual(c.z, 1000) + self.assertEqual(c.w, 2000) + self.assertEqual(c.t, 3000) + C.z += 1 + self.assertEqual(c.z, 1001) + c = C(20) + self.assertEqual((c.x, c.y), (20, 10)) + self.assertEqual(c.z, 1001) + self.assertEqual(c.w, 2000) + self.assertEqual(c.t, 3000) + + def test_class_var_no_default(self): + # If a ClassVar has no default value, it should not be set on the class. + @dataclass + class C: + x: ClassVar[int] + + self.assertNotIn('x', C.__dict__) + + def test_class_var_default_factory(self): + # It makes no sense for a ClassVar to have a default factory. When + # would it be called? Call it yourself, since it's class-wide. + with self.assertRaisesRegex(TypeError, + 'cannot have a default factory'): + @dataclass + class C: + x: ClassVar[int] = field(default_factory=int) + + self.assertNotIn('x', C.__dict__) + + def test_class_var_with_default(self): + # If a ClassVar has a default value, it should be set on the class. + @dataclass + class C: + x: ClassVar[int] = 10 + self.assertEqual(C.x, 10) + + @dataclass + class C: + x: ClassVar[int] = field(default=10) + self.assertEqual(C.x, 10) + + def test_class_var_frozen(self): + # Make sure ClassVars work even if we're frozen. + @dataclass(frozen=True) + class C: + x: int + y: int = 10 + z: ClassVar[int] = 1000 + w: ClassVar[int] = 2000 + t: ClassVar[int] = 3000 + + c = C(5) + self.assertEqual(repr(C(5)), 'TestCase.test_class_var_frozen..C(x=5, y=10)') + self.assertEqual(len(fields(C)), 2) # We have 2 fields + self.assertEqual(len(C.__annotations__), 5) # And 3 ClassVars + self.assertEqual(c.z, 1000) + self.assertEqual(c.w, 2000) + self.assertEqual(c.t, 3000) + # We can still modify the ClassVar, it's only instances that are + # frozen. + C.z += 1 + self.assertEqual(c.z, 1001) + c = C(20) + self.assertEqual((c.x, c.y), (20, 10)) + self.assertEqual(c.z, 1001) + self.assertEqual(c.w, 2000) + self.assertEqual(c.t, 3000) + + def test_init_var_no_default(self): + # If an InitVar has no default value, it should not be set on the class. + @dataclass + class C: + x: InitVar[int] + + self.assertNotIn('x', C.__dict__) + + def test_init_var_default_factory(self): + # It makes no sense for an InitVar to have a default factory. When + # would it be called? Call it yourself, since it's class-wide. + with self.assertRaisesRegex(TypeError, + 'cannot have a default factory'): + @dataclass + class C: + x: InitVar[int] = field(default_factory=int) + + self.assertNotIn('x', C.__dict__) + + def test_init_var_with_default(self): + # If an InitVar has a default value, it should be set on the class. + @dataclass + class C: + x: InitVar[int] = 10 + self.assertEqual(C.x, 10) + + @dataclass + class C: + x: InitVar[int] = field(default=10) + self.assertEqual(C.x, 10) + + def test_init_var(self): + @dataclass + class C: + x: int = None + init_param: InitVar[int] = None + + def __post_init__(self, init_param): + if self.x is None: + self.x = init_param*2 + + c = C(init_param=10) + self.assertEqual(c.x, 20) + + def test_init_var_inheritance(self): + # Note that this deliberately tests that a dataclass need not + # have a __post_init__ function if it has an InitVar field. + # It could just be used in a derived class, as shown here. + @dataclass + class Base: + x: int + init_base: InitVar[int] + + # We can instantiate by passing the InitVar, even though + # it's not used. + b = Base(0, 10) + self.assertEqual(vars(b), {'x': 0}) + + @dataclass + class C(Base): + y: int + init_derived: InitVar[int] + + def __post_init__(self, init_base, init_derived): + self.x = self.x + init_base + self.y = self.y + init_derived + + c = C(10, 11, 50, 51) + self.assertEqual(vars(c), {'x': 21, 'y': 101}) + + def test_default_factory(self): + # Test a factory that returns a new list. + @dataclass + class C: + x: int + y: list = field(default_factory=list) + + c0 = C(3) + c1 = C(3) + self.assertEqual(c0.x, 3) + self.assertEqual(c0.y, []) + self.assertEqual(c0, c1) + self.assertIsNot(c0.y, c1.y) + self.assertEqual(astuple(C(5, [1])), (5, [1])) + + # Test a factory that returns a shared list. + l = [] + @dataclass + class C: + x: int + y: list = field(default_factory=lambda: l) + + c0 = C(3) + c1 = C(3) + self.assertEqual(c0.x, 3) + self.assertEqual(c0.y, []) + self.assertEqual(c0, c1) + self.assertIs(c0.y, c1.y) + self.assertEqual(astuple(C(5, [1])), (5, [1])) + + # Test various other field flags. + # repr + @dataclass + class C: + x: list = field(default_factory=list, repr=False) + self.assertEqual(repr(C()), 'TestCase.test_default_factory..C()') + self.assertEqual(C().x, []) + + # hash + @dataclass(hash=True) + class C: + x: list = field(default_factory=list, hash=False) + self.assertEqual(astuple(C()), ([],)) + self.assertEqual(hash(C()), hash(())) + + # init (see also test_default_factory_with_no_init) + @dataclass + class C: + x: list = field(default_factory=list, init=False) + self.assertEqual(astuple(C()), ([],)) + + # compare + @dataclass + class C: + x: list = field(default_factory=list, compare=False) + self.assertEqual(C(), C([1])) + + def test_default_factory_with_no_init(self): + # We need a factory with a side effect. + factory = Mock() + + @dataclass + class C: + x: list = field(default_factory=factory, init=False) + + # Make sure the default factory is called for each new instance. + C().x + self.assertEqual(factory.call_count, 1) + C().x + self.assertEqual(factory.call_count, 2) + + def test_default_factory_not_called_if_value_given(self): + # We need a factory that we can test if it's been called. + factory = Mock() + + @dataclass + class C: + x: int = field(default_factory=factory) + + # Make sure that if a field has a default factory function, + # it's not called if a value is specified. + C().x + self.assertEqual(factory.call_count, 1) + self.assertEqual(C(10).x, 10) + self.assertEqual(factory.call_count, 1) + C().x + self.assertEqual(factory.call_count, 2) + + def x_test_classvar_default_factory(self): + # XXX: it's an error for a ClassVar to have a factory function + @dataclass + class C: + x: ClassVar[int] = field(default_factory=int) + + self.assertIs(C().x, int) + + def test_isdataclass(self): + # There is no isdataclass() helper any more, but the PEP + # describes how to write it, so make sure that works. Note + # that this version returns True for both classes and + # instances. + def isdataclass(obj): + try: + fields(obj) + return True + except TypeError: + return False + + self.assertFalse(isdataclass(0)) + self.assertFalse(isdataclass(int)) + + @dataclass + class C: + x: int + + self.assertTrue(isdataclass(C)) + self.assertTrue(isdataclass(C(0))) + + def test_helper_fields_with_class_instance(self): + # Check that we can call fields() on either a class or instance, + # and get back the same thing. + @dataclass + class C: + x: int + y: float + + self.assertEqual(fields(C), fields(C(0, 0.0))) + + def test_helper_fields_exception(self): + # Check that TypeError is raised if not passed a dataclass or + # instance. + with self.assertRaisesRegex(TypeError, 'dataclass type or instance'): + fields(0) + + class C: pass + with self.assertRaisesRegex(TypeError, 'dataclass type or instance'): + fields(C) + with self.assertRaisesRegex(TypeError, 'dataclass type or instance'): + fields(C()) + + def test_helper_asdict(self): + # Basic tests for asdict(), it should return a new dictionary + @dataclass + class C: + x: int + y: int + c = C(1, 2) + + self.assertEqual(asdict(c), {'x': 1, 'y': 2}) + self.assertEqual(asdict(c), asdict(c)) + self.assertIsNot(asdict(c), asdict(c)) + c.x = 42 + self.assertEqual(asdict(c), {'x': 42, 'y': 2}) + self.assertIs(type(asdict(c)), dict) + + def test_helper_asdict_raises_on_classes(self): + # asdict() should raise on a class object + @dataclass + class C: + x: int + y: int + with self.assertRaisesRegex(TypeError, 'dataclass instance'): + asdict(C) + with self.assertRaisesRegex(TypeError, 'dataclass instance'): + asdict(int) + + def test_helper_asdict_copy_values(self): + @dataclass + class C: + x: int + y: List[int] = field(default_factory=list) + initial = [] + c = C(1, initial) + d = asdict(c) + self.assertEqual(d['y'], initial) + self.assertIsNot(d['y'], initial) + c = C(1) + d = asdict(c) + d['y'].append(1) + self.assertEqual(c.y, []) + + def test_helper_asdict_nested(self): + @dataclass + class UserId: + token: int + group: int + @dataclass + class User: + name: str + id: UserId + u = User('Joe', UserId(123, 1)) + d = asdict(u) + self.assertEqual(d, {'name': 'Joe', 'id': {'token': 123, 'group': 1}}) + self.assertIsNot(asdict(u), asdict(u)) + u.id.group = 2 + self.assertEqual(asdict(u), {'name': 'Joe', + 'id': {'token': 123, 'group': 2}}) + + def test_helper_asdict_builtin_containers(self): + @dataclass + class User: + name: str + id: int + @dataclass + class GroupList: + id: int + users: List[User] + @dataclass + class GroupTuple: + id: int + users: Tuple[User, ...] + @dataclass + class GroupDict: + id: int + users: Dict[str, User] + a = User('Alice', 1) + b = User('Bob', 2) + gl = GroupList(0, [a, b]) + gt = GroupTuple(0, (a, b)) + gd = GroupDict(0, {'first': a, 'second': b}) + self.assertEqual(asdict(gl), {'id': 0, 'users': [{'name': 'Alice', 'id': 1}, + {'name': 'Bob', 'id': 2}]}) + self.assertEqual(asdict(gt), {'id': 0, 'users': ({'name': 'Alice', 'id': 1}, + {'name': 'Bob', 'id': 2})}) + self.assertEqual(asdict(gd), {'id': 0, 'users': {'first': {'name': 'Alice', 'id': 1}, + 'second': {'name': 'Bob', 'id': 2}}}) + + def test_helper_asdict_builtin_containers(self): + @dataclass + class Child: + d: object + + @dataclass + class Parent: + child: Child + + self.assertEqual(asdict(Parent(Child([1]))), {'child': {'d': [1]}}) + self.assertEqual(asdict(Parent(Child({1: 2}))), {'child': {'d': {1: 2}}}) + + def test_helper_asdict_factory(self): + @dataclass + class C: + x: int + y: int + c = C(1, 2) + d = asdict(c, dict_factory=OrderedDict) + self.assertEqual(d, OrderedDict([('x', 1), ('y', 2)])) + self.assertIsNot(d, asdict(c, dict_factory=OrderedDict)) + c.x = 42 + d = asdict(c, dict_factory=OrderedDict) + self.assertEqual(d, OrderedDict([('x', 42), ('y', 2)])) + self.assertIs(type(d), OrderedDict) + + def test_helper_astuple(self): + # Basic tests for astuple(), it should return a new tuple + @dataclass + class C: + x: int + y: int = 0 + c = C(1) + + self.assertEqual(astuple(c), (1, 0)) + self.assertEqual(astuple(c), astuple(c)) + self.assertIsNot(astuple(c), astuple(c)) + c.y = 42 + self.assertEqual(astuple(c), (1, 42)) + self.assertIs(type(astuple(c)), tuple) + + def test_helper_astuple_raises_on_classes(self): + # astuple() should raise on a class object + @dataclass + class C: + x: int + y: int + with self.assertRaisesRegex(TypeError, 'dataclass instance'): + astuple(C) + with self.assertRaisesRegex(TypeError, 'dataclass instance'): + astuple(int) + + def test_helper_astuple_copy_values(self): + @dataclass + class C: + x: int + y: List[int] = field(default_factory=list) + initial = [] + c = C(1, initial) + t = astuple(c) + self.assertEqual(t[1], initial) + self.assertIsNot(t[1], initial) + c = C(1) + t = astuple(c) + t[1].append(1) + self.assertEqual(c.y, []) + + def test_helper_astuple_nested(self): + @dataclass + class UserId: + token: int + group: int + @dataclass + class User: + name: str + id: UserId + u = User('Joe', UserId(123, 1)) + t = astuple(u) + self.assertEqual(t, ('Joe', (123, 1))) + self.assertIsNot(astuple(u), astuple(u)) + u.id.group = 2 + self.assertEqual(astuple(u), ('Joe', (123, 2))) + + def test_helper_astuple_builtin_containers(self): + @dataclass + class User: + name: str + id: int + @dataclass + class GroupList: + id: int + users: List[User] + @dataclass + class GroupTuple: + id: int + users: Tuple[User, ...] + @dataclass + class GroupDict: + id: int + users: Dict[str, User] + a = User('Alice', 1) + b = User('Bob', 2) + gl = GroupList(0, [a, b]) + gt = GroupTuple(0, (a, b)) + gd = GroupDict(0, {'first': a, 'second': b}) + self.assertEqual(astuple(gl), (0, [('Alice', 1), ('Bob', 2)])) + self.assertEqual(astuple(gt), (0, (('Alice', 1), ('Bob', 2)))) + self.assertEqual(astuple(gd), (0, {'first': ('Alice', 1), 'second': ('Bob', 2)})) + + def test_helper_astuple_builtin_containers(self): + @dataclass + class Child: + d: object + + @dataclass + class Parent: + child: Child + + self.assertEqual(astuple(Parent(Child([1]))), (([1],),)) + self.assertEqual(astuple(Parent(Child({1: 2}))), (({1: 2},),)) + + def test_helper_astuple_factory(self): + @dataclass + class C: + x: int + y: int + NT = namedtuple('NT', 'x y') + def nt(lst): + return NT(*lst) + c = C(1, 2) + t = astuple(c, tuple_factory=nt) + self.assertEqual(t, NT(1, 2)) + self.assertIsNot(t, astuple(c, tuple_factory=nt)) + c.x = 42 + t = astuple(c, tuple_factory=nt) + self.assertEqual(t, NT(42, 2)) + self.assertIs(type(t), NT) + + def test_dynamic_class_creation(self): + cls_dict = {'__annotations__': OrderedDict(x=int, y=int), + } + + # Create the class. + cls = type('C', (), cls_dict) + + # Make it a dataclass. + cls1 = dataclass(cls) + + self.assertEqual(cls1, cls) + self.assertEqual(asdict(cls(1, 2)), {'x': 1, 'y': 2}) + + def test_dynamic_class_creation_using_field(self): + cls_dict = {'__annotations__': OrderedDict(x=int, y=int), + 'y': field(default=5), + } + + # Create the class. + cls = type('C', (), cls_dict) + + # Make it a dataclass. + cls1 = dataclass(cls) + + self.assertEqual(cls1, cls) + self.assertEqual(asdict(cls1(1)), {'x': 1, 'y': 5}) + + def test_init_in_order(self): + @dataclass + class C: + a: int + b: int = field() + c: list = field(default_factory=list, init=False) + d: list = field(default_factory=list) + e: int = field(default=4, init=False) + f: int = 4 + + calls = [] + def setattr(self, name, value): + calls.append((name, value)) + + C.__setattr__ = setattr + c = C(0, 1) + self.assertEqual(('a', 0), calls[0]) + self.assertEqual(('b', 1), calls[1]) + self.assertEqual(('c', []), calls[2]) + self.assertEqual(('d', []), calls[3]) + self.assertNotIn(('e', 4), calls) + self.assertEqual(('f', 4), calls[4]) + + def test_items_in_dicts(self): + @dataclass + class C: + a: int + b: list = field(default_factory=list, init=False) + c: list = field(default_factory=list) + d: int = field(default=4, init=False) + e: int = 0 + + c = C(0) + # Class dict + self.assertNotIn('a', C.__dict__) + self.assertNotIn('b', C.__dict__) + self.assertNotIn('c', C.__dict__) + self.assertIn('d', C.__dict__) + self.assertEqual(C.d, 4) + self.assertIn('e', C.__dict__) + self.assertEqual(C.e, 0) + # Instance dict + self.assertIn('a', c.__dict__) + self.assertEqual(c.a, 0) + self.assertIn('b', c.__dict__) + self.assertEqual(c.b, []) + self.assertIn('c', c.__dict__) + self.assertEqual(c.c, []) + self.assertNotIn('d', c.__dict__) + self.assertIn('e', c.__dict__) + self.assertEqual(c.e, 0) + + def test_alternate_classmethod_constructor(self): + # Since __post_init__ can't take params, use a classmethod + # alternate constructor. This is mostly an example to show how + # to use this technique. + @dataclass + class C: + x: int + @classmethod + def from_file(cls, filename): + # In a real example, create a new instance + # and populate 'x' from contents of a file. + value_in_file = 20 + return cls(value_in_file) + + self.assertEqual(C.from_file('filename').x, 20) + + def test_field_metadata_default(self): + # Make sure the default metadata is read-only and of + # zero length. + @dataclass + class C: + i: int + + self.assertFalse(fields(C)[0].metadata) + self.assertEqual(len(fields(C)[0].metadata), 0) + with self.assertRaisesRegex(TypeError, + 'does not support item assignment'): + fields(C)[0].metadata['test'] = 3 + + def test_field_metadata_mapping(self): + # Make sure only a mapping can be passed as metadata + # zero length. + with self.assertRaises(TypeError): + @dataclass + class C: + i: int = field(metadata=0) + + # Make sure an empty dict works + @dataclass + class C: + i: int = field(metadata={}) + self.assertFalse(fields(C)[0].metadata) + self.assertEqual(len(fields(C)[0].metadata), 0) + with self.assertRaisesRegex(TypeError, + 'does not support item assignment'): + fields(C)[0].metadata['test'] = 3 + + # Make sure a non-empty dict works. + @dataclass + class C: + i: int = field(metadata={'test': 10, 'bar': '42', 3: 'three'}) + self.assertEqual(len(fields(C)[0].metadata), 3) + self.assertEqual(fields(C)[0].metadata['test'], 10) + self.assertEqual(fields(C)[0].metadata['bar'], '42') + self.assertEqual(fields(C)[0].metadata[3], 'three') + with self.assertRaises(KeyError): + # Non-existent key. + fields(C)[0].metadata['baz'] + with self.assertRaisesRegex(TypeError, + 'does not support item assignment'): + fields(C)[0].metadata['test'] = 3 + + def test_field_metadata_custom_mapping(self): + # Try a custom mapping. + class SimpleNameSpace: + def __init__(self, **kw): + self.__dict__.update(kw) + + def __getitem__(self, item): + if item == 'xyzzy': + return 'plugh' + return getattr(self, item) + + def __len__(self): + return self.__dict__.__len__() + + @dataclass + class C: + i: int = field(metadata=SimpleNameSpace(a=10)) + + self.assertEqual(len(fields(C)[0].metadata), 1) + self.assertEqual(fields(C)[0].metadata['a'], 10) + with self.assertRaises(AttributeError): + fields(C)[0].metadata['b'] + # Make sure we're still talking to our custom mapping. + self.assertEqual(fields(C)[0].metadata['xyzzy'], 'plugh') + + def test_generic_dataclasses(self): + T = TypeVar('T') + + @dataclass + class LabeledBox(Generic[T]): + content: T + label: str = '' + + box = LabeledBox(42) + self.assertEqual(box.content, 42) + self.assertEqual(box.label, '') + + # subscripting the resulting class should work, etc. + Alias = List[LabeledBox[int]] + + def test_generic_extending(self): + S = TypeVar('S') + T = TypeVar('T') + + @dataclass + class Base(Generic[T, S]): + x: T + y: S + + @dataclass + class DataDerived(Base[int, T]): + new_field: str + Alias = DataDerived[str] + c = Alias(0, 'test1', 'test2') + self.assertEqual(astuple(c), (0, 'test1', 'test2')) + + class NonDataDerived(Base[int, T]): + def new_method(self): + return self.y + Alias = NonDataDerived[float] + c = Alias(10, 1.0) + self.assertEqual(c.new_method(), 1.0) + + def test_helper_replace(self): + @dataclass(frozen=True) + class C: + x: int + y: int + + c = C(1, 2) + c1 = replace(c, x=3) + self.assertEqual(c1.x, 3) + self.assertEqual(c1.y, 2) + + def test_helper_replace_frozen(self): + @dataclass(frozen=True) + class C: + x: int + y: int + z: int = field(init=False, default=10) + t: int = field(init=False, default=100) + + c = C(1, 2) + c1 = replace(c, x=3) + self.assertEqual((c.x, c.y, c.z, c.t), (1, 2, 10, 100)) + self.assertEqual((c1.x, c1.y, c1.z, c1.t), (3, 2, 10, 100)) + + + with self.assertRaisesRegex(ValueError, 'init=False'): + replace(c, x=3, z=20, t=50) + with self.assertRaisesRegex(ValueError, 'init=False'): + replace(c, z=20) + replace(c, x=3, z=20, t=50) + + # Make sure the result is still frozen. + with self.assertRaisesRegex(FrozenInstanceError, "cannot assign to field 'x'"): + c1.x = 3 + + # Make sure we can't replace an attribute that doesn't exist, + # if we're also replacing one that does exist. Test this + # here, because setting attributes on frozen instances is + # handled slightly differently from non-frozen ones. + with self.assertRaisesRegex(TypeError, "__init__\(\) got an unexpected " + "keyword argument 'a'"): + c1 = replace(c, x=20, a=5) + + def test_helper_replace_invalid_field_name(self): + @dataclass(frozen=True) + class C: + x: int + y: int + + c = C(1, 2) + with self.assertRaisesRegex(TypeError, "__init__\(\) got an unexpected " + "keyword argument 'z'"): + c1 = replace(c, z=3) + + def test_helper_replace_invalid_object(self): + @dataclass(frozen=True) + class C: + x: int + y: int + + with self.assertRaisesRegex(TypeError, 'dataclass instance'): + replace(C, x=3) + + with self.assertRaisesRegex(TypeError, 'dataclass instance'): + replace(0, x=3) + + def test_helper_replace_no_init(self): + @dataclass + class C: + x: int + y: int = field(init=False, default=10) + + c = C(1) + c.y = 20 + + # Make sure y gets the default value. + c1 = replace(c, x=5) + self.assertEqual((c1.x, c1.y), (5, 10)) + + # Trying to replace y is an error. + with self.assertRaisesRegex(ValueError, 'init=False'): + replace(c, x=2, y=30) + with self.assertRaisesRegex(ValueError, 'init=False'): + replace(c, y=30) + + def test_dataclassses_pickleable(self): + global P, Q, R + @dataclass + class P: + x: int + y: int = 0 + @dataclass + class Q: + x: int + y: int = field(default=0, init=False) + @dataclass + class R: + x: int + y: List[int] = field(default_factory=list) + q = Q(1) + q.y = 2 + samples = [P(1), P(1, 2), Q(1), q, R(1), R(1, [2, 3, 4])] + for sample in samples: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(sample=sample, proto=proto): + new_sample = pickle.loads(pickle.dumps(sample, proto)) + self.assertEqual(sample.x, new_sample.x) + self.assertEqual(sample.y, new_sample.y) + self.assertIsNot(sample, new_sample) + new_sample.x = 42 + another_new_sample = pickle.loads(pickle.dumps(new_sample, proto)) + self.assertEqual(new_sample.x, another_new_sample.x) + self.assertEqual(sample.y, another_new_sample.y) + + def test_helper_make_dataclass(self): + C = make_dataclass('C', + [('x', int), + ('y', int, field(default=5))], + namespace={'add_one': lambda self: self.x + 1}) + c = C(10) + self.assertEqual((c.x, c.y), (10, 5)) + self.assertEqual(c.add_one(), 11) + + + def test_helper_make_dataclass_no_mutate_namespace(self): + # Make sure a provided namespace isn't mutated. + ns = {} + C = make_dataclass('C', + [('x', int), + ('y', int, field(default=5))], + namespace=ns) + self.assertEqual(ns, {}) + + def test_helper_make_dataclass_base(self): + class Base1: + pass + class Base2: + pass + C = make_dataclass('C', + [('x', int)], + bases=(Base1, Base2)) + c = C(2) + self.assertIsInstance(c, C) + self.assertIsInstance(c, Base1) + self.assertIsInstance(c, Base2) + + def test_helper_make_dataclass_base_dataclass(self): + @dataclass + class Base1: + x: int + class Base2: + pass + C = make_dataclass('C', + [('y', int)], + bases=(Base1, Base2)) + with self.assertRaisesRegex(TypeError, 'required positional'): + c = C(2) + c = C(1, 2) + self.assertIsInstance(c, C) + self.assertIsInstance(c, Base1) + self.assertIsInstance(c, Base2) + + self.assertEqual((c.x, c.y), (1, 2)) + + def test_helper_make_dataclass_init_var(self): + def post_init(self, y): + self.x *= y + + C = make_dataclass('C', + [('x', int), + ('y', InitVar[int]), + ], + namespace={'__post_init__': post_init}, + ) + c = C(2, 3) + self.assertEqual(vars(c), {'x': 6}) + self.assertEqual(len(fields(c)), 1) + + def test_helper_make_dataclass_class_var(self): + C = make_dataclass('C', + [('x', int), + ('y', ClassVar[int], 10), + ('z', ClassVar[int], field(default=20)), + ]) + c = C(1) + self.assertEqual(vars(c), {'x': 1}) + self.assertEqual(len(fields(c)), 1) + self.assertEqual(C.y, 10) + self.assertEqual(C.z, 20) + + +class TestDocString(unittest.TestCase): + def assertDocStrEqual(self, a, b): + # Because 3.6 and 3.7 differ in how inspect.signature work + # (see bpo #32108), for the time being just compare them with + # whitespace stripped. + self.assertEqual(a.replace(' ', ''), b.replace(' ', '')) + + def test_existing_docstring_not_overridden(self): + @dataclass + class C: + """Lorem ipsum""" + x: int + + self.assertEqual(C.__doc__, "Lorem ipsum") + + def test_docstring_no_fields(self): + @dataclass + class C: + pass + + self.assertDocStrEqual(C.__doc__, "C()") + + def test_docstring_one_field(self): + @dataclass + class C: + x: int + + self.assertDocStrEqual(C.__doc__, "C(x:int)") + + def test_docstring_two_fields(self): + @dataclass + class C: + x: int + y: int + + self.assertDocStrEqual(C.__doc__, "C(x:int, y:int)") + + def test_docstring_three_fields(self): + @dataclass + class C: + x: int + y: int + z: str + + self.assertDocStrEqual(C.__doc__, "C(x:int, y:int, z:str)") + + def test_docstring_one_field_with_default(self): + @dataclass + class C: + x: int = 3 + + self.assertDocStrEqual(C.__doc__, "C(x:int=3)") + + def test_docstring_one_field_with_default_none(self): + @dataclass + class C: + x: Union[int, type(None)] = None + + self.assertDocStrEqual(C.__doc__, "C(x:Union[int, NoneType]=None)") + + def test_docstring_list_field(self): + @dataclass + class C: + x: List[int] + + self.assertDocStrEqual(C.__doc__, "C(x:List[int])") + + def test_docstring_list_field_with_default_factory(self): + @dataclass + class C: + x: List[int] = field(default_factory=list) + + self.assertDocStrEqual(C.__doc__, "C(x:List[int]=)") + + def test_docstring_deque_field(self): + @dataclass + class C: + x: deque + + self.assertDocStrEqual(C.__doc__, "C(x:collections.deque)") + + def test_docstring_deque_field_with_default_factory(self): + @dataclass + class C: + x: deque = field(default_factory=deque) + + self.assertDocStrEqual(C.__doc__, "C(x:collections.deque=)") + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 8013f37c88da0a..4386eda3ae48fd 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -468,6 +468,12 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + def test_repr_deep(self): + d = {} + for i in range(sys.getrecursionlimit() + 100): + d = {1: d} + self.assertRaises(RecursionError, repr, d) + def test_eq(self): self.assertEqual({}, {}) self.assertEqual({1: 2}, {1: 2}) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index e0ec87d2080185..1667617b9e465f 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -14,18 +14,7 @@ def test_EWWhiteSpaceTerminal(self): self.assertEqual(x, ' \t') self.assertEqual(str(x), '') self.assertEqual(x.value, '') - self.assertEqual(x.encoded, ' \t') - - # UnstructuredTokenList - - def test_undecodable_bytes_error_preserved(self): - badstr = b"le pouf c\xaflebre".decode('ascii', 'surrogateescape') - unst = parser.get_unstructured(badstr) - self.assertDefectsEqual(unst.all_defects, [errors.UndecodableBytesDefect]) - parts = list(unst.parts) - self.assertDefectsEqual(parts[0].all_defects, []) - self.assertDefectsEqual(parts[1].all_defects, []) - self.assertDefectsEqual(parts[2].all_defects, [errors.UndecodableBytesDefect]) + self.assertEqual(x.token_type, 'fws') class TestParserMixin: @@ -139,7 +128,6 @@ def test_get_encoded_word_sets_extra_attributes(self): 'first second', [], '') - self.assertEqual(ew.encoded, '=?us-ascii*jive?q?first_second?=') self.assertEqual(ew.charset, 'us-ascii') self.assertEqual(ew.lang, 'jive') @@ -150,7 +138,6 @@ def test_get_encoded_word_lang_default_is_blank(self): 'first second', [], '') - self.assertEqual(ew.encoded, '=?us-ascii?q?first_second?=') self.assertEqual(ew.charset, 'us-ascii') self.assertEqual(ew.lang, '') @@ -2700,28 +2687,37 @@ def test_address_list_with_unicode_names_in_quotes(self): # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. - def test_initial_whitespace_splitting(self): + def test_split_at_whitespace_after_header_before_long_token(self): body = parser.get_unstructured(' ' + 'x'*77) header = parser.Header([ parser.HeaderLabel([parser.ValueTerminal('test:', 'atext')]), parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), body]) self._test(header, 'test: \n ' + 'x'*77 + '\n') - def test_whitespace_splitting(self): + def test_split_at_whitespace_before_long_token(self): self._test(parser.get_unstructured('xxx ' + 'y'*77), 'xxx \n ' + 'y'*77 + '\n') + def test_overlong_encodeable_is_wrapped(self): + first_token_with_whitespace = 'xxx ' + chrome_leader = '=?utf-8?q?' + len_chrome = len(chrome_leader) + 2 + len_non_y = len_chrome + len(first_token_with_whitespace) + self._test(parser.get_unstructured(first_token_with_whitespace + + 'y'*80), + first_token_with_whitespace + chrome_leader + + 'y'*(78-len_non_y) + '?=\n' + + ' ' + chrome_leader + 'y'*(80-(78-len_non_y)) + '?=\n') + def test_long_filename_attachment(self): - folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"') - self.assertEqual( - 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"\n', - folded - ) - folded = self.policy.fold('Content-Disposition', 'attachment; filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"') - self.assertEqual( - 'Content-Disposition: attachment;\n filename="TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_T.txt"\n', - folded - ) + self._test(parser.parse_content_disposition_header( + 'attachment; filename="TEST_TEST_TEST_TEST' + '_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TEST_TES.txt"'), + "attachment;\n" + " filename*0*=us-ascii''TEST_TEST_TEST_TEST_TEST_TEST" + "_TEST_TEST_TEST_TEST_TEST;\n" + " filename*1*=_TEST_TES.txt\n", + ) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index c4f182903afefe..c1aeaefab775ff 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -27,7 +27,6 @@ def msgmaker(self, msg, policy=None): None """), - # From is wrapped because wrapped it fits in 40. 40: textwrap.dedent("""\ To: whom_it_may_concern@example.com From: @@ -40,11 +39,11 @@ def msgmaker(self, msg, policy=None): None """), - # Neither to nor from fit even if put on a new line, - # so we leave them sticking out on the first line. 20: textwrap.dedent("""\ - To: whom_it_may_concern@example.com - From: nobody_you_want_to_know@example.com + To: + whom_it_may_concern@example.com + From: + nobody_you_want_to_know@example.com Subject: We the willing led by the unknowing are doing @@ -169,6 +168,53 @@ def test_compat32_max_line_length_does_not_fold_when_none(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(self.refold_long_expected[0])) + def test_rfc2231_wrapping(self): + # This is pretty much just to make sure we don't have an infinite + # loop; I don't expect anyone to hit this in the field. + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename="afilenamelongenoghtowraphere" + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename*0*=us-ascii''afilename; + filename*1*=longenoghtowraphere + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=33)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + + def test_rfc2231_wrapping_switches_to_default_len_if_too_narrow(self): + # This is just to make sure we don't have an infinite loop; I don't + # expect anyone to hit this in the field, so I'm not bothering to make + # the result optimal (the encoding isn't needed). + msg = self.msgmaker(self.typ(textwrap.dedent("""\ + To: nobody + Content-Disposition: attachment; + filename="afilenamelongenoghtowraphere" + + None + """))) + expected = textwrap.dedent("""\ + To: nobody + Content-Disposition: + attachment; + filename*0*=us-ascii''afilenamelongenoghtowraphere + + None + """) + s = self.ioclass() + g = self.genclass(s, policy=self.policy.clone(max_line_length=20)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.typ(expected)) + class TestGenerator(TestGeneratorBase, TestEmailBase): diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index af836dc9726622..30ce0ba54e4728 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -229,14 +229,14 @@ def content_type_as_value(self, defects = args[1] if l>1 else [] decoded = args[2] if l>2 and args[2] is not DITTO else source header = 'Content-Type:' + ' ' if source else '' - folded = args[3] if l>3 else header + source + '\n' + folded = args[3] if l>3 else header + decoded + '\n' h = self.make_header('Content-Type', source) self.assertEqual(h.content_type, content_type) self.assertEqual(h.maintype, maintype) self.assertEqual(h.subtype, subtype) self.assertEqual(h.params, parmdict) with self.assertRaises(TypeError): - h.params['abc'] = 'xyz' # params is read-only. + h.params['abc'] = 'xyz' # make sure params is read-only. self.assertDefectsEqual(h.defects, defects) self.assertEqual(h, decoded) self.assertEqual(h.fold(policy=policy.default), folded) @@ -373,9 +373,10 @@ def content_type_as_value(self, 'text/plain; Charset="utf-8"'), # Since this is pretty much the ur-mimeheader, we'll put all the tests - # that exercise the parameter parsing and formatting here. - # - # XXX: question: is minimal quoting preferred? + # that exercise the parameter parsing and formatting here. Note that + # when we refold we may canonicalize, so things like whitespace, + # quoting, and rfc2231 encoding may change from what was in the input + # header. 'unquoted_param_value': ( 'text/plain; title=foo', @@ -384,7 +385,8 @@ def content_type_as_value(self, 'plain', {'title': 'foo'}, [], - 'text/plain; title="foo"'), + 'text/plain; title="foo"', + ), 'param_value_with_tspecials': ( 'text/plain; title="(bar)foo blue"', @@ -415,7 +417,8 @@ def content_type_as_value(self, 'mixed', {'boundary': 'CPIMSSMTPC06p5f3tG'}, [], - 'Multipart/mixed; boundary="CPIMSSMTPC06p5f3tG"'), + 'Multipart/mixed; boundary="CPIMSSMTPC06p5f3tG"', + ), 'spaces_around_semis': ( ('image/jpeg; name="wibble.JPG" ; x-mac-type="4A504547" ; ' @@ -429,14 +432,31 @@ def content_type_as_value(self, [], ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' 'x-mac-creator="474B4F4E"'), - # XXX: it could be that we will eventually prefer to fold starting - # from the decoded value, in which case these spaces and similar - # spaces in other tests will be wrong. - ('Content-Type: image/jpeg; name="wibble.JPG" ; ' - 'x-mac-type="4A504547" ;\n' + ('Content-Type: image/jpeg; name="wibble.JPG";' + ' x-mac-type="4A504547";\n' ' x-mac-creator="474B4F4E"\n'), ), + 'lots_of_mime_params': ( + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"; x-extrastuff="make it longer"'), + 'image/jpeg', + 'image', + 'jpeg', + {'name': 'wibble.JPG', + 'x-mac-type': '4A504547', + 'x-mac-creator': '474B4F4E', + 'x-extrastuff': 'make it longer'}, + [], + ('image/jpeg; name="wibble.JPG"; x-mac-type="4A504547"; ' + 'x-mac-creator="474B4F4E"; x-extrastuff="make it longer"'), + # In this case the whole of the MimeParameters does *not* fit + # one one line, so we break at a lower syntactic level. + ('Content-Type: image/jpeg; name="wibble.JPG";' + ' x-mac-type="4A504547";\n' + ' x-mac-creator="474B4F4E"; x-extrastuff="make it longer"\n'), + ), + 'semis_inside_quotes': ( 'image/jpeg; name="Jim&&Jill"', 'image/jpeg', @@ -460,19 +480,25 @@ def content_type_as_value(self, [], r'image/jpeg; name="Jim \"Bob\" Jill"'), - # XXX: This test works except for the refolding of the header. I'll - # deal with that bug when I deal with the other folding bugs. - #'non_ascii_in_params': ( - # ('foo\xa7/bar; b\xa7r=two; ' - # 'baz=thr\xa7e'.encode('latin-1').decode('us-ascii', - # 'surrogateescape')), - # 'foo\uFFFD/bar', - # 'foo\uFFFD', - # 'bar', - # {'b\uFFFDr': 'two', 'baz': 'thr\uFFFDe'}, - # [errors.UndecodableBytesDefect]*3, - # 'foo�/bar; b�r="two"; baz="thr�e"', - # ), + 'non_ascii_in_params': ( + ('foo\xa7/bar; b\xa7r=two; ' + 'baz=thr\xa7e'.encode('latin-1').decode('us-ascii', + 'surrogateescape')), + 'foo\uFFFD/bar', + 'foo\uFFFD', + 'bar', + {'b\uFFFDr': 'two', 'baz': 'thr\uFFFDe'}, + [errors.UndecodableBytesDefect]*3, + 'foo�/bar; b�r="two"; baz="thr�e"', + # XXX Two bugs here: the mime type is not allowed to be an encoded + # word, and we shouldn't be emitting surrogates in the parameter + # names. But I don't know what the behavior should be here, so I'm + # punting for now. In practice this is unlikely to be encountered + # since headers with binary in them only come from a binary source + # and are almost certain to be re-emitted without refolding. + 'Content-Type: =?unknown-8bit?q?foo=A7?=/bar; b\udca7r="two";\n' + " baz*=unknown-8bit''thr%A7e\n", + ), # RFC 2231 parameter tests. @@ -494,19 +520,20 @@ def content_type_as_value(self, [], r'image/jpeg; bar="baz\"foobar\"baz"'), - # XXX: This test works except for the refolding of the header. I'll - # deal with that bug when I deal with the other folding bugs. - #'non_ascii_rfc2231_value': ( - # ('text/plain; charset=us-ascii; ' - # "title*=us-ascii'en'This%20is%20" - # 'not%20f\xa7n').encode('latin-1').decode('us-ascii', - # 'surrogateescape'), - # 'text/plain', - # 'text', - # 'plain', - # {'charset': 'us-ascii', 'title': 'This is not f\uFFFDn'}, - # [errors.UndecodableBytesDefect], - # 'text/plain; charset="us-ascii"; title="This is not f�n"'), + 'non_ascii_rfc2231_value': ( + ('text/plain; charset=us-ascii; ' + "title*=us-ascii'en'This%20is%20" + 'not%20f\xa7n').encode('latin-1').decode('us-ascii', + 'surrogateescape'), + 'text/plain', + 'text', + 'plain', + {'charset': 'us-ascii', 'title': 'This is not f\uFFFDn'}, + [errors.UndecodableBytesDefect], + 'text/plain; charset="us-ascii"; title="This is not f�n"', + 'Content-Type: text/plain; charset="us-ascii";\n' + " title*=unknown-8bit''This%20is%20not%20f%A7n\n", + ), 'rfc2231_encoded_charset': ( 'text/plain; charset*=ansi-x3.4-1968\'\'us-ascii', @@ -529,8 +556,6 @@ def content_type_as_value(self, {'name': 'This is ***fun*** is it not.pdf'}, [], 'text/plain; name="This is ***fun*** is it not.pdf"', - ('Content-Type: text/plain;\tname*0*=\'\'This%20is%20;\n' - '\tname*1*=%2A%2A%2Afun%2A%2A%2A%20;\tname*2="is it not.pdf"\n'), ), # Make sure we also handle it if there are spurious double quotes. @@ -545,9 +570,6 @@ def content_type_as_value(self, {'name': 'This is even more ***fun*** is it not.pdf'}, [errors.InvalidHeaderDefect]*2, 'text/plain; name="This is even more ***fun*** is it not.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="us-ascii\'\'This%20is%20even%20more%20";\n' - '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it not.pdf"\n'), ), 'rfc2231_single_quote_inside_double_quotes': ( @@ -562,9 +584,8 @@ def content_type_as_value(self, [errors.InvalidHeaderDefect]*2, ('text/plain; charset="us-ascii"; ' 'title="This is really ***fun*** isn\'t it!"'), - ('Content-Type: text/plain; charset=us-ascii;\n' - '\ttitle*0*="us-ascii\'en\'This%20is%20really%20";\n' - '\ttitle*1*="%2A%2A%2Afun%2A%2A%2A%20";\ttitle*2="isn\'t it!"\n'), + ('Content-Type: text/plain; charset="us-ascii";\n' + ' title="This is really ***fun*** isn\'t it!"\n'), ), 'rfc2231_single_quote_in_value_with_charset_and_lang': ( @@ -576,9 +597,6 @@ def content_type_as_value(self, {'name': "Frank's Document"}, [errors.InvalidHeaderDefect]*2, 'application/x-foo; name="Frank\'s Document"', - ('Content-Type: application/x-foo;\t' - 'name*0*="us-ascii\'en-us\'Frank\'s";\n' - ' name*1*=" Document"\n'), ), 'rfc2231_single_quote_in_non_encoded_value': ( @@ -590,9 +608,6 @@ def content_type_as_value(self, {'name': "us-ascii'en-us'Frank's Document"}, [], 'application/x-foo; name="us-ascii\'en-us\'Frank\'s Document"', - ('Content-Type: application/x-foo;\t' - 'name*0="us-ascii\'en-us\'Frank\'s";\n' - ' name*1=" Document"\n'), ), 'rfc2231_no_language_or_charset': ( @@ -615,12 +630,8 @@ def content_type_as_value(self, {'name': 'This is even more ***fun*** is it.pdf'}, [errors.InvalidHeaderDefect]*2, 'text/plain; name="This is even more ***fun*** is it.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="\'\'This%20is%20even%20more%20";\n' - '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'), ), - # XXX: see below...the first name line here should be *0 not *0*. 'rfc2231_partly_encoded': ( ("text/plain;" '\tname*0*="\'\'This%20is%20even%20more%20";' @@ -632,9 +643,6 @@ def content_type_as_value(self, {'name': 'This is even more ***fun*** is it.pdf'}, [errors.InvalidHeaderDefect]*2, 'text/plain; name="This is even more ***fun*** is it.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="\'\'This%20is%20even%20more%20";\n' - '\tname*1*="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'), ), 'rfc2231_partly_encoded_2': ( @@ -647,10 +655,11 @@ def content_type_as_value(self, 'plain', {'name': 'This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf'}, [errors.InvalidHeaderDefect], - 'text/plain; name="This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf"', - ('Content-Type: text/plain;\t' - 'name*0*="\'\'This%20is%20even%20more%20";\n' - '\tname*1="%2A%2A%2Afun%2A%2A%2A%20";\tname*2="is it.pdf"\n'), + ('text/plain;' + ' name="This is even more %2A%2A%2Afun%2A%2A%2A%20is it.pdf"'), + ('Content-Type: text/plain;\n' + ' name="This is even more %2A%2A%2Afun%2A%2A%2A%20is' + ' it.pdf"\n'), ), 'rfc2231_unknown_charset_treated_as_ascii': ( @@ -669,9 +678,12 @@ def content_type_as_value(self, 'plain', {'charset': 'utf-8\uFFFD\uFFFD\uFFFD'}, [errors.UndecodableBytesDefect], - 'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"'), + 'text/plain; charset="utf-8\uFFFD\uFFFD\uFFFD"', + "Content-Type: text/plain;" + " charset*=unknown-8bit''utf-8%F1%F2%F3\n", + ), - 'rfc2231_utf_8_in_supposedly_ascii_charset_parameter_value': ( + 'rfc2231_utf8_in_supposedly_ascii_charset_parameter_value': ( "text/plain; charset*=ascii''utf-8%E2%80%9D", 'text/plain', 'text', @@ -679,9 +691,11 @@ def content_type_as_value(self, {'charset': 'utf-8”'}, [errors.UndecodableBytesDefect], 'text/plain; charset="utf-8”"', + # XXX Should folding change the charset to utf8? Currently it just + # reproduces the original, which is arguably fine. + "Content-Type: text/plain;" + " charset*=unknown-8bit''utf-8%E2%80%9D\n", ), - # XXX: if the above were *re*folded, it would get tagged as utf-8 - # instead of ascii in the param, since it now contains non-ASCII. 'rfc2231_encoded_then_unencoded_segments': ( ('application/x-foo;' @@ -694,9 +708,6 @@ def content_type_as_value(self, {'name': 'My Document For You'}, [errors.InvalidHeaderDefect], 'application/x-foo; name="My Document For You"', - ('Content-Type: application/x-foo;\t' - 'name*0*="us-ascii\'en-us\'My";\n' - '\tname*1=" Document";\tname*2=" For You"\n'), ), # My reading of the RFC is that this is an invalid header. The RFC @@ -713,11 +724,6 @@ def content_type_as_value(self, {'name': 'My Document For You'}, [errors.InvalidHeaderDefect]*3, 'application/x-foo; name="My Document For You"', - ("Content-Type: application/x-foo;\tname*0=us-ascii'en-us'My;\t" - # XXX: the newline is in the wrong place, come back and fix - # this when the rest of tests pass. - 'name*1*=" Document"\n;' - '\tname*2*=" For You"\n'), ), # XXX: I would say this one should default to ascii/en for the @@ -730,8 +736,7 @@ def content_type_as_value(self, # charset'lang'value pattern exactly *and* there is at least one # encoded segment. Implementing that algorithm will require some # refactoring, so I haven't done it (yet). - - 'rfc2231_qouted_unencoded_then_encoded_segments': ( + 'rfc2231_quoted_unencoded_then_encoded_segments': ( ('application/x-foo;' '\tname*0="us-ascii\'en-us\'My";' '\tname*1*=" Document";' @@ -742,9 +747,25 @@ def content_type_as_value(self, {'name': "us-ascii'en-us'My Document For You"}, [errors.InvalidHeaderDefect]*2, 'application/x-foo; name="us-ascii\'en-us\'My Document For You"', - ('Content-Type: application/x-foo;\t' - 'name*0="us-ascii\'en-us\'My";\n' - '\tname*1*=" Document";\tname*2*=" For You"\n'), + ), + + # Make sure our folding algorithm produces multiple sections correctly. + # We could mix encoded and non-encoded segments, but we don't, we just + # make them all encoded. It might be worth fixing that, since the + # sections can get used for wrapping ascii text. + 'rfc2231_folded_segments_correctly_formatted': ( + ('application/x-foo;' + '\tname="' + "with spaces"*8 + '"'), + 'application/x-foo', + 'application', + 'x-foo', + {'name': "with spaces"*8}, + [], + 'application/x-foo; name="' + "with spaces"*8 + '"', + "Content-Type: application/x-foo;\n" + " name*0*=us-ascii''with%20spaceswith%20spaceswith%20spaceswith" + "%20spaceswith;\n" + " name*1*=%20spaceswith%20spaceswith%20spaceswith%20spaces\n" ), } @@ -827,8 +848,8 @@ def content_disp_as_value(self, [], ('attachment; filename="genome.jpeg"; ' 'modification-date="Wed, 12 Feb 1997 16:29:51 -0500"'), - ('Content-Disposition: attachment; filename=genome.jpeg;\n' - ' modification-date="Wed, 12 Feb 1997 16:29:51 -0500";\n'), + ('Content-Disposition: attachment; filename="genome.jpeg";\n' + ' modification-date="Wed, 12 Feb 1997 16:29:51 -0500"\n'), ), 'no_value': ( @@ -873,7 +894,7 @@ def version_string_as_MIME_Version(self, if source: source = ' ' + source self.assertEqual(h.fold(policy=policy.default), - 'MIME-Version:' + source + '\n') + 'MIME-Version:' + source + '\n') version_string_params = { @@ -1546,15 +1567,39 @@ def test_fold_unstructured_with_overlong_word(self): 'singlewordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: thisisaverylonglineconsistingofasinglewordthatwontfit\n') + 'Subject: \n' + ' =?utf-8?q?thisisa?=\n' + ' =?utf-8?q?verylon?=\n' + ' =?utf-8?q?glineco?=\n' + ' =?utf-8?q?nsistin?=\n' + ' =?utf-8?q?gofasin?=\n' + ' =?utf-8?q?gleword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit?=\n' + ) def test_fold_unstructured_with_two_overlong_words(self): h = self.make_header('Subject', 'thisisaverylonglineconsistingofa' 'singlewordthatwontfit plusanotherverylongwordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: thisisaverylonglineconsistingofasinglewordthatwontfit\n' - ' plusanotherverylongwordthatwontfit\n') + 'Subject: \n' + ' =?utf-8?q?thisisa?=\n' + ' =?utf-8?q?verylon?=\n' + ' =?utf-8?q?glineco?=\n' + ' =?utf-8?q?nsistin?=\n' + ' =?utf-8?q?gofasin?=\n' + ' =?utf-8?q?gleword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit_pl?=\n' + ' =?utf-8?q?usanoth?=\n' + ' =?utf-8?q?erveryl?=\n' + ' =?utf-8?q?ongword?=\n' + ' =?utf-8?q?thatwon?=\n' + ' =?utf-8?q?tfit?=\n' + ) + + # XXX Need test for when max_line_length is less than the chrome size. def test_fold_unstructured_with_slightly_long_word(self): h = self.make_header('Subject', 'thislongwordislessthanmaxlinelen') @@ -1590,6 +1635,18 @@ def test_fold_date_header(self): self.assertEqual(h.fold(policy=policy.default), 'Date: Sat, 02 Feb 2002 17:00:06 -0800\n') + def test_fold_overlong_words_using_RFC2047(self): + h = self.make_header( + 'X-Report-Abuse', + '') + self.assertEqual( + h.fold(policy=policy.default), + 'X-Report-Abuse: =?utf-8?q?=3Chttps=3A//www=2Emailitapp=2E' + 'com/report=5F?=\n' + ' =?utf-8?q?abuse=2Ephp=3Fmid=3Dxxx-xxx-xxxx' + 'xxxxxxxxxxxxxxxxxxxx=3D=3D-xxx-?=\n' + ' =?utf-8?q?xx-xx=3E?=\n') if __name__ == '__main__': diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py new file mode 100644 index 00000000000000..c7f45b59acc8a1 --- /dev/null +++ b/Lib/test/test_embed.py @@ -0,0 +1,213 @@ +# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs) +from test import support +import unittest + +from collections import namedtuple +import os +import re +import subprocess +import sys + + +class EmbeddingTests(unittest.TestCase): + def setUp(self): + here = os.path.abspath(__file__) + basepath = os.path.dirname(os.path.dirname(os.path.dirname(here))) + exename = "_testembed" + if sys.platform.startswith("win"): + ext = ("_d" if "_d" in sys.executable else "") + ".exe" + exename += ext + exepath = os.path.dirname(sys.executable) + else: + exepath = os.path.join(basepath, "Programs") + self.test_exe = exe = os.path.join(exepath, exename) + if not os.path.exists(exe): + self.skipTest("%r doesn't exist" % exe) + # This is needed otherwise we get a fatal error: + # "Py_Initialize: Unable to get the locale encoding + # LookupError: no codec search functions registered: can't find encoding" + self.oldcwd = os.getcwd() + os.chdir(basepath) + + def tearDown(self): + os.chdir(self.oldcwd) + + def run_embedded_interpreter(self, *args, env=None): + """Runs a test in the embedded interpreter""" + cmd = [self.test_exe] + cmd.extend(args) + if env is not None and sys.platform == 'win32': + # Windows requires at least the SYSTEMROOT environment variable to + # start Python. + env = env.copy() + env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] + + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=env) + (out, err) = p.communicate() + if p.returncode != 0 and support.verbose: + print(f"--- {cmd} failed ---") + print(f"stdout:\n{out}") + print(f"stderr:\n{out}") + print(f"------") + + self.assertEqual(p.returncode, 0, + "bad returncode %d, stderr is %r" % + (p.returncode, err)) + return out, err + + def run_repeated_init_and_subinterpreters(self): + out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters") + self.assertEqual(err, "") + + # The output from _testembed looks like this: + # --- Pass 0 --- + # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 + # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 + # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 + # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 + # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 + # --- Pass 1 --- + # ... + + interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " + r"thread state <(0x[\dA-F]+)>: " + r"id\(modules\) = ([\d]+)$") + Interp = namedtuple("Interp", "id interp tstate modules") + + numloops = 0 + current_run = [] + for line in out.splitlines(): + if line == "--- Pass {} ---".format(numloops): + self.assertEqual(len(current_run), 0) + if support.verbose: + print(line) + numloops += 1 + continue + + self.assertLess(len(current_run), 5) + match = re.match(interp_pat, line) + if match is None: + self.assertRegex(line, interp_pat) + + # Parse the line from the loop. The first line is the main + # interpreter and the 3 afterward are subinterpreters. + interp = Interp(*match.groups()) + if support.verbose: + print(interp) + self.assertTrue(interp.interp) + self.assertTrue(interp.tstate) + self.assertTrue(interp.modules) + current_run.append(interp) + + # The last line in the loop should be the same as the first. + if len(current_run) == 5: + main = current_run[0] + self.assertEqual(interp, main) + yield current_run + current_run = [] + + def test_subinterps_main(self): + for run in self.run_repeated_init_and_subinterpreters(): + main = run[0] + + self.assertEqual(main.id, '0') + + def test_subinterps_different_ids(self): + for run in self.run_repeated_init_and_subinterpreters(): + main, *subs, _ = run + + mainid = int(main.id) + for i, sub in enumerate(subs): + self.assertEqual(sub.id, str(mainid + i + 1)) + + def test_subinterps_distinct_state(self): + for run in self.run_repeated_init_and_subinterpreters(): + main, *subs, _ = run + + if '0x0' in main: + # XXX Fix on Windows (and other platforms): something + # is going on with the pointers in Programs/_testembed.c. + # interp.interp is 0x0 and interp.modules is the same + # between interpreters. + raise unittest.SkipTest('platform prints pointers as 0x0') + + for sub in subs: + # A new subinterpreter may have the same + # PyInterpreterState pointer as a previous one if + # the earlier one has already been destroyed. So + # we compare with the main interpreter. The same + # applies to tstate. + self.assertNotEqual(sub.interp, main.interp) + self.assertNotEqual(sub.tstate, main.tstate) + self.assertNotEqual(sub.modules, main.modules) + + def test_forced_io_encoding(self): + # Checks forced configuration of embedded interpreter IO streams + env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") + out, err = self.run_embedded_interpreter("forced_io_encoding", env=env) + if support.verbose > 1: + print() + print(out) + print(err) + expected_stream_encoding = "utf-8" + expected_errors = "surrogateescape" + expected_output = '\n'.join([ + "--- Use defaults ---", + "Expected encoding: default", + "Expected errors: default", + "stdin: {in_encoding}:{errors}", + "stdout: {out_encoding}:{errors}", + "stderr: {out_encoding}:backslashreplace", + "--- Set errors only ---", + "Expected encoding: default", + "Expected errors: ignore", + "stdin: {in_encoding}:ignore", + "stdout: {out_encoding}:ignore", + "stderr: {out_encoding}:backslashreplace", + "--- Set encoding only ---", + "Expected encoding: latin-1", + "Expected errors: default", + "stdin: latin-1:{errors}", + "stdout: latin-1:{errors}", + "stderr: latin-1:backslashreplace", + "--- Set encoding and errors ---", + "Expected encoding: latin-1", + "Expected errors: replace", + "stdin: latin-1:replace", + "stdout: latin-1:replace", + "stderr: latin-1:backslashreplace"]) + expected_output = expected_output.format( + in_encoding=expected_stream_encoding, + out_encoding=expected_stream_encoding, + errors=expected_errors) + # This is useful if we ever trip over odd platform behaviour + self.maxDiff = None + self.assertEqual(out.strip(), expected_output) + + def test_pre_initialization_api(self): + """ + Checks the few parts of the C-API that work before the runtine + is initialized (via Py_Initialize()). + """ + env = dict(os.environ, PYTHONPATH=os.pathsep.join(sys.path)) + out, err = self.run_embedded_interpreter("pre_initialization_api", env=env) + self.assertEqual(out, '') + self.assertEqual(err, '') + + def test_bpo20891(self): + """ + bpo-20891: Calling PyGILState_Ensure in a non-Python thread before + calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must + call PyEval_InitThreads() for us in this case. + """ + out, err = self.run_embedded_interpreter("bpo20891") + self.assertEqual(out, '') + self.assertEqual(err, '') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index e0e53c2f829210..3647bfc719ecc6 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -6,7 +6,7 @@ import subprocess import sys from test import support -from test.support import script_helper, is_android, requires_android_level +from test.support import script_helper, is_android import tempfile import threading import unittest @@ -29,6 +29,11 @@ def expected_traceback(lineno1, lineno2, header, min_count=1): else: return '^' + regex + '$' +def skip_segfault_on_android(test): + # Issue #32138: Raising SIGSEGV on Android may not cause a crash. + return unittest.skipIf(is_android, + 'raising SIGSEGV on Android is unreliable')(test) + @contextmanager def temporary_filename(): filename = tempfile.mktemp() @@ -37,10 +42,6 @@ def temporary_filename(): finally: support.unlink(filename) -def requires_raise(test): - return (test if not is_android else - requires_android_level(24, 'raise() is buggy')(test)) - class FaultHandlerTests(unittest.TestCase): def get_output(self, code, filename=None, fd=None): """ @@ -140,7 +141,7 @@ def test_read_null(self): 3, 'access violation') - @requires_raise + @skip_segfault_on_android def test_sigsegv(self): self.check_fatal_error(""" import faulthandler @@ -182,7 +183,7 @@ def test_sigfpe(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') - @requires_raise + @skip_segfault_on_android def test_sigbus(self): self.check_fatal_error(""" import _testcapi @@ -197,7 +198,7 @@ def test_sigbus(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGILL'), 'need signal.SIGILL') - @requires_raise + @skip_segfault_on_android def test_sigill(self): self.check_fatal_error(""" import _testcapi @@ -241,7 +242,7 @@ def test_stack_overflow(self): '(?:Segmentation fault|Bus error)', other_regex='unable to raise a stack overflow') - @requires_raise + @skip_segfault_on_android def test_gil_released(self): self.check_fatal_error(""" import faulthandler @@ -251,7 +252,7 @@ def test_gil_released(self): 3, 'Segmentation fault') - @requires_raise + @skip_segfault_on_android def test_enable_file(self): with temporary_filename() as filename: self.check_fatal_error(""" @@ -266,7 +267,7 @@ def test_enable_file(self): @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") - @requires_raise + @skip_segfault_on_android def test_enable_fd(self): with tempfile.TemporaryFile('wb+') as fp: fd = fp.fileno() @@ -280,7 +281,7 @@ def test_enable_fd(self): 'Segmentation fault', fd=fd) - @requires_raise + @skip_segfault_on_android def test_enable_single_thread(self): self.check_fatal_error(""" import faulthandler @@ -291,7 +292,7 @@ def test_enable_single_thread(self): 'Segmentation fault', all_threads=False) - @requires_raise + @skip_segfault_on_android def test_disable(self): code = """ import faulthandler diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index a16c05cf5d99c5..17174dd295dfcc 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -51,7 +51,7 @@ def test_float(self): self.assertRaises(TypeError, float, {}) self.assertRaisesRegex(TypeError, "not 'dict'", float, {}) # Lone surrogate - self.assertRaises(UnicodeEncodeError, float, '\uD8F0') + self.assertRaises(ValueError, float, '\uD8F0') # check that we don't accept alternate exponent markers self.assertRaises(ValueError, float, "-1.7d29") self.assertRaises(ValueError, float, "3D-14") diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py new file mode 100644 index 00000000000000..a7c748422b1db3 --- /dev/null +++ b/Lib/test/test_frozen.py @@ -0,0 +1,30 @@ +"""Basic test of the frozen module (source is in Python/frozen.c).""" + +# The Python/frozen.c source code contains a marshalled Python module +# and therefore depends on the marshal format as well as the bytecode +# format. If those formats have been changed then frozen.c needs to be +# updated. +# +# The test_importlib also tests this module but because those tests +# are much more complicated, it might be unclear why they are failing. +# Invalid marshalled data in frozen.c could case the interpreter to +# crash when __hello__ is imported. + +import sys +import unittest +from test.support import captured_stdout +from importlib import util + + +class TestFrozen(unittest.TestCase): + def test_frozen(self): + name = '__hello__' + if name in sys.modules: + del sys.modules[name] + with captured_stdout() as out: + import __hello__ + self.assertEqual(out.getvalue(), 'Hello world!\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 7eac9d076be462..f88c762581da61 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1830,13 +1830,7 @@ def printsolution(self, x): [None] - -An obscene abuse of a yield expression within a generator expression: - ->>> list((yield 21) for i in range(4)) -[21, None, 21, None, 21, None, 21, None] - -And a more sane, but still weird usage: +Yield is allowed only in the outermost iterable in generator expression: >>> def f(): list(i for i in [(yield 26)]) >>> type(f()) @@ -2106,10 +2100,6 @@ def printsolution(self, x): >>> type(f()) ->>> def f(): x=(i for i in (yield) if (yield)) ->>> type(f()) - - >>> def f(d): d[(yield "a")] = d[(yield "b")] = 27 >>> data = [1,2] >>> g = f(data) diff --git a/Lib/test/test_genericclass.py b/Lib/test/test_genericclass.py new file mode 100644 index 00000000000000..214527b01fa871 --- /dev/null +++ b/Lib/test/test_genericclass.py @@ -0,0 +1,252 @@ +import unittest + + +class TestMROEntry(unittest.TestCase): + def test_mro_entry_signature(self): + tested = [] + class B: ... + class C: + def __mro_entries__(self, *args, **kwargs): + tested.extend([args, kwargs]) + return (C,) + c = C() + self.assertEqual(tested, []) + class D(B, c): ... + self.assertEqual(tested[0], ((B, c),)) + self.assertEqual(tested[1], {}) + + def test_mro_entry(self): + tested = [] + class A: ... + class B: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (self.__class__,) + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, C, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, C, B, object)) + d = D() + class E(d): ... + self.assertEqual(tested[-1], (d,)) + self.assertEqual(E.__bases__, (D,)) + + def test_mro_entry_none(self): + tested = [] + class A: ... + class B: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return () + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + class E(c): ... + self.assertEqual(tested[-1], (c,)) + self.assertEqual(E.__bases__, (object,)) + self.assertEqual(E.__orig_bases__, (c,)) + self.assertEqual(E.__mro__, (E, object)) + + def test_mro_entry_with_builtins(self): + tested = [] + class A: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (dict,) + c = C() + self.assertEqual(tested, []) + class D(A, c): ... + self.assertEqual(tested[-1], (A, c)) + self.assertEqual(D.__bases__, (A, dict)) + self.assertEqual(D.__orig_bases__, (A, c)) + self.assertEqual(D.__mro__, (D, A, dict, object)) + + def test_mro_entry_with_builtins_2(self): + tested = [] + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (C,) + c = C() + self.assertEqual(tested, []) + class D(c, dict): ... + self.assertEqual(tested[-1], (c, dict)) + self.assertEqual(D.__bases__, (C, dict)) + self.assertEqual(D.__orig_bases__, (c, dict)) + self.assertEqual(D.__mro__, (D, C, dict, object)) + + def test_mro_entry_errors(self): + class C_too_many: + def __mro_entries__(self, bases, something, other): + return () + c = C_too_many() + with self.assertRaises(TypeError): + class D(c): ... + class C_too_few: + def __mro_entries__(self): + return () + d = C_too_few() + with self.assertRaises(TypeError): + class D(d): ... + + def test_mro_entry_errors_2(self): + class C_not_callable: + __mro_entries__ = "Surprise!" + c = C_not_callable() + with self.assertRaises(TypeError): + class D(c): ... + class C_not_tuple: + def __mro_entries__(self): + return object + c = C_not_tuple() + with self.assertRaises(TypeError): + class D(c): ... + + def test_mro_entry_metaclass(self): + meta_args = [] + class Meta(type): + def __new__(mcls, name, bases, ns): + meta_args.extend([mcls, name, bases, ns]) + return super().__new__(mcls, name, bases, ns) + class A: ... + class C: + def __mro_entries__(self, bases): + return (A,) + c = C() + class D(c, metaclass=Meta): + x = 1 + self.assertEqual(meta_args[0], Meta) + self.assertEqual(meta_args[1], 'D') + self.assertEqual(meta_args[2], (A,)) + self.assertEqual(meta_args[3]['x'], 1) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + self.assertEqual(D.__class__, Meta) + + def test_mro_entry_type_call(self): + # Substitution should _not_ happen in direct type call + class C: + def __mro_entries__(self, bases): + return () + c = C() + with self.assertRaisesRegex(TypeError, + "MRO entry resolution; " + "use types.new_class()"): + type('Bad', (c,), {}) + + +class TestClassGetitem(unittest.TestCase): + def test_class_getitem(self): + getitem_args = [] + class C: + def __class_getitem__(*args, **kwargs): + getitem_args.extend([args, kwargs]) + return None + C[int, str] + self.assertEqual(getitem_args[0], (C, (int, str))) + self.assertEqual(getitem_args[1], {}) + + def test_class_getitem(self): + class C: + def __class_getitem__(cls, item): + return f'C[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + self.assertEqual(C[C], 'C[C]') + + def test_class_getitem_inheritance(self): + class C: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_inheritance_2(self): + class C: + def __class_getitem__(cls, item): + return 'Should not see this' + class D(C): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_patched(self): + class C: + def __init_subclass__(cls): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + cls.__class_getitem__ = __class_getitem__ + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_with_builtins(self): + class A(dict): + called_with = None + + def __class_getitem__(cls, item): + cls.called_with = item + class B(A): + pass + self.assertIs(B.called_with, None) + B[int] + self.assertIs(B.called_with, int) + + def test_class_getitem_errors(self): + class C_too_few: + def __class_getitem__(cls): + return None + with self.assertRaises(TypeError): + C_too_few[int] + class C_too_many: + def __class_getitem__(cls, one, two): + return None + with self.assertRaises(TypeError): + C_too_many[int] + + def test_class_getitem_errors_2(self): + class C: + def __class_getitem__(cls, item): + return None + with self.assertRaises(TypeError): + C()[int] + class E: ... + e = E() + e.__class_getitem__ = lambda cls, item: 'This will not work' + with self.assertRaises(TypeError): + e[int] + class C_not_callable: + __class_getitem__ = "Surprise!" + with self.assertRaises(TypeError): + C_not_callable[int] + + def test_class_getitem_metaclass(self): + class Meta(type): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(Meta[int], 'Meta[int]') + + def test_class_getitem_metaclass_2(self): + class Meta(type): + def __getitem__(cls, item): + return 'from metaclass' + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return 'from __class_getitem__' + self.assertEqual(C[int], 'from metaclass') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index f698e13f68a7bf..01e11da09805e7 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -8,7 +8,6 @@ import unittest import warnings from test import support -android_not_root = support.android_not_root def create_file(filename, data=b'foo'): @@ -213,9 +212,11 @@ def _test_samefile_on_link_func(self, func): def test_samefile_on_symlink(self): self._test_samefile_on_link_func(os.symlink) - @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_samefile_on_link(self): - self._test_samefile_on_link_func(os.link) + try: + self._test_samefile_on_link_func(os.link) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) def test_samestat(self): test_fn1 = support.TESTFN @@ -253,9 +254,11 @@ def _test_samestat_on_link_func(self, func): def test_samestat_on_symlink(self): self._test_samestat_on_link_func(os.symlink) - @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_samestat_on_link(self): - self._test_samestat_on_link_func(os.link) + try: + self._test_samestat_on_link_func(os.link) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) def test_sameopenfile(self): filename = support.TESTFN diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 65e26bfd383823..823315f8cd0dfc 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -841,6 +841,41 @@ def g(): f((yield from ()), 1) # Check annotation refleak on SyntaxError check_syntax_error(self, "def g(a:(yield)): pass") + def test_yield_in_comprehensions(self): + # Check yield in comprehensions + def g(): [x for x in [(yield 1)]] + def g(): [x for x in [(yield from ())]] + + def check(code, warntext): + with self.assertWarnsRegex(DeprecationWarning, warntext): + compile(code, '', 'exec') + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=DeprecationWarning) + with self.assertRaisesRegex(SyntaxError, warntext): + compile(code, '', 'exec') + + check("def g(): [(yield x) for x in ()]", + "'yield' inside list comprehension") + check("def g(): [x for x in () if not (yield x)]", + "'yield' inside list comprehension") + check("def g(): [y for x in () for y in [(yield x)]]", + "'yield' inside list comprehension") + check("def g(): {(yield x) for x in ()}", + "'yield' inside set comprehension") + check("def g(): {(yield x): x for x in ()}", + "'yield' inside dict comprehension") + check("def g(): {x: (yield x) for x in ()}", + "'yield' inside dict comprehension") + check("def g(): ((yield x) for x in ())", + "'yield' inside generator expression") + check("def g(): [(yield from x) for x in ()]", + "'yield' inside list comprehension") + check("class C: [(yield x) for x in ()]", + "'yield' inside list comprehension") + check("[(yield x) for x in ()]", + "'yield' inside list comprehension") + def test_raise(self): # 'raise' test [',' test] try: raise RuntimeError('just testing') diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 0d79cae5096cde..a3f8194dc44fcf 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -497,7 +497,7 @@ def test_status_lines(self): def test_bad_status_repr(self): exc = client.BadStatusLine('') - self.assertEqual(repr(exc), '''BadStatusLine("\'\'",)''') + self.assertEqual(repr(exc), '''BadStatusLine("''")''') def test_partial_reads(self): # if we have Content-Length, HTTPResponse knows when to close itself, diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 20e6f66766f676..cc829a522b88bc 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -384,7 +384,8 @@ def close_conn(): reader.close() return body - @support.requires_mac_ver(10, 5) + @unittest.skipIf(sys.platform == 'darwin', + 'undecodable name cannot always be decoded on macOS') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(support.TESTFN_UNDECODABLE, diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index e64215d4677720..13a86b12dd3d51 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -2875,12 +2875,12 @@ def test_signature_str(self): def foo(a:int=1, *, b, c=None, **kwargs) -> 42: pass self.assertEqual(str(inspect.signature(foo)), - '(a:int=1, *, b, c=None, **kwargs) -> 42') + '(a: int = 1, *, b, c=None, **kwargs) -> 42') def foo(a:int=1, *args, b, c=None, **kwargs) -> 42: pass self.assertEqual(str(inspect.signature(foo)), - '(a:int=1, *args, b, c=None, **kwargs) -> 42') + '(a: int = 1, *args, b, c=None, **kwargs) -> 42') def foo(): pass diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 791052102140b7..eb9064edea9115 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -93,12 +93,15 @@ def test_failures(self): def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} + with self.assertRaisesRegex(TypeError, + 'keys must be str, int, float, bool or None, not tuple'): + self.dumps(data) - #This is for c encoder - self.assertRaises(TypeError, self.dumps, data) - - #This is for python encoder - self.assertRaises(TypeError, self.dumps, data, indent=True) + def test_not_serializable(self): + import sys + with self.assertRaisesRegex(TypeError, + 'Object of type module is not JSON serializable'): + self.dumps(sys) def test_truncated_input(self): test_cases = [ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1968b4253c2abc..7101e3b15760c2 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3211,9 +3211,11 @@ def test_handle_called_with_queue_queue(self, mock_handle): self.assertEqual(mock_handle.call_count, 5 * self.repeat, 'correct number of handled log messages') - @support.requires_multiprocessing_queue @patch.object(logging.handlers.QueueListener, 'handle') def test_handle_called_with_mp_queue(self, mock_handle): + # Issue 28668: The multiprocessing (mp) module is not functional + # when the mp.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') for i in range(self.repeat): log_queue = multiprocessing.Queue() self.setup_and_log(log_queue, '%s_%s' % (self.id(), i)) @@ -3230,7 +3232,6 @@ def get_all_from_queue(log_queue): except queue.Empty: return [] - @support.requires_multiprocessing_queue def test_no_messages_in_queue_after_stop(self): """ Five messages are logged then the QueueListener is stopped. This @@ -3238,6 +3239,9 @@ def test_no_messages_in_queue_after_stop(self): indicates that messages were not registered on the queue until _after_ the QueueListener stopped. """ + # Issue 28668: The multiprocessing (mp) module is not functional + # when the mp.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') for i in range(self.repeat): queue = multiprocessing.Queue() self.setup_and_log(queue, '%s_%s' %(self.id(), i)) diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index b378ffea00c0a7..29dda987d0bb7f 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -34,6 +34,29 @@ def test_ints(self): self.helper(expected) n = n >> 1 + def test_int64(self): + # Simulate int marshaling with TYPE_INT64. + maxint64 = (1 << 63) - 1 + minint64 = -maxint64-1 + for base in maxint64, minint64, -maxint64, -(minint64 >> 1): + while base: + s = b'I' + int.to_bytes(base, 8, 'little', signed=True) + got = marshal.loads(s) + self.assertEqual(base, got) + if base == -1: # a fixed-point for shifting right 1 + base = 0 + else: + base >>= 1 + + got = marshal.loads(b'I\xfe\xdc\xba\x98\x76\x54\x32\x10') + self.assertEqual(got, 0x1032547698badcfe) + got = marshal.loads(b'I\x01\x23\x45\x67\x89\xab\xcd\xef') + self.assertEqual(got, -0x1032547698badcff) + got = marshal.loads(b'I\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f') + self.assertEqual(got, 0x7f6e5d4c3b2a1908) + got = marshal.loads(b'I\xf7\xe6\xd5\xc4\xb3\xa2\x91\x80') + self.assertEqual(got, -0x7f6e5d4c3b2a1909) + def test_bool(self): for b in (True, False): self.helper(b) diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py index f656f7234e2d33..6d89ca4b77321e 100644 --- a/Lib/test/test_msilib.py +++ b/Lib/test/test_msilib.py @@ -1,7 +1,65 @@ """ Test suite for the code in msilib """ +import os.path import unittest -from test.support import import_module +from test.support import TESTFN, import_module, unlink msilib = import_module('msilib') +import msilib.schema + + +def init_database(): + path = TESTFN + '.msi' + db = msilib.init_database( + path, + msilib.schema, + 'Python Tests', + 'product_code', + '1.0', + 'PSF', + ) + return db, path + + +class MsiDatabaseTestCase(unittest.TestCase): + + def test_view_fetch_returns_none(self): + db, db_path = init_database() + properties = [] + view = db.OpenView('SELECT Property, Value FROM Property') + view.Execute(None) + while True: + record = view.Fetch() + if record is None: + break + properties.append(record.GetString(1)) + view.Close() + db.Close() + self.assertEqual( + properties, + [ + 'ProductName', 'ProductCode', 'ProductVersion', + 'Manufacturer', 'ProductLanguage', + ] + ) + self.addCleanup(unlink, db_path) + + def test_database_open_failed(self): + with self.assertRaises(msilib.MSIError) as cm: + msilib.OpenDatabase('non-existent.msi', msilib.MSIDBOPEN_READONLY) + self.assertEqual(str(cm.exception), 'open failed') + + def test_database_create_failed(self): + db_path = os.path.join(TESTFN, 'test.msi') + with self.assertRaises(msilib.MSIError) as cm: + msilib.OpenDatabase(db_path, msilib.MSIDBOPEN_CREATE) + self.assertEqual(str(cm.exception), 'create failed') + + def test_get_property_vt_empty(self): + db, db_path = init_database() + summary = db.GetSummaryInformation(0) + self.assertIsNone(summary.GetProperty(msilib.PID_SECURITY)) + db.Close() + self.addCleanup(unlink, db_path) + class Test_make_id(unittest.TestCase): #http://msdn.microsoft.com/en-us/library/aa369212(v=vs.85).aspx diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index ca6f27d03c3afd..f59e5371acad3b 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,4 +1,5 @@ import netrc, os, unittest, sys, tempfile, textwrap +from unittest import mock from test import support @@ -126,8 +127,44 @@ def test_security(self): os.chmod(fn, 0o622) self.assertRaises(netrc.NetrcParseError, netrc.netrc) -def test_main(): - support.run_unittest(NetrcTestCase) + def test_file_not_found_in_home(self): + d = support.TESTFN + os.mkdir(d) + self.addCleanup(support.rmtree, d) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + self.assertRaises(FileNotFoundError, netrc.netrc) + + def test_file_not_found_explicit(self): + self.assertRaises(FileNotFoundError, netrc.netrc, + file='unlikely_netrc') + + def test_home_not_set(self): + fake_home = support.TESTFN + os.mkdir(fake_home) + self.addCleanup(support.rmtree, fake_home) + fake_netrc_path = os.path.join(fake_home, '.netrc') + with open(fake_netrc_path, 'w') as f: + f.write('machine foo.domain.com login bar password pass') + os.chmod(fake_netrc_path, 0o600) + + orig_expanduser = os.path.expanduser + called = [] + + def fake_expanduser(s): + called.append(s) + with support.EnvironmentVarGuard() as environ: + environ.set('HOME', fake_home) + result = orig_expanduser(s) + return result + + with support.swap_attr(os.path, 'expanduser', fake_expanduser): + nrc = netrc.netrc() + login, account, password = nrc.authenticators('foo.domain.com') + self.assertEqual(login, 'bar') + + self.assertTrue(called) + if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index bb780bfa004b57..8c1032b986bf61 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -165,6 +165,7 @@ def check_article_resp(self, resp, article, art_num=None): # XXX this could exceptionally happen... self.assertNotIn(article.lines[-1], (b".", b".\n", b".\r\n")) + @unittest.skipIf(True, "FIXME: see bpo-32128") def test_article_head_body(self): resp, count, first, last, name = self.server.group(self.GROUP_NAME) # Try to find an available article diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ff19fac0f77af6..96ee3ee577fb9e 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1736,7 +1736,10 @@ def tearDown(self): def _test_link(self, file1, file2): create_file(file1) - os.link(file1, file2) + try: + os.link(file1, file2) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) with open(file1, "r") as f1, open(file2, "r") as f2: self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) @@ -2888,7 +2891,8 @@ def test_stty_match(self): """ try: size = subprocess.check_output(['stty', 'size']).decode().split() - except (FileNotFoundError, subprocess.CalledProcessError): + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError): self.skipTest("stty invocation failed") expected = (int(size[1]), int(size[0])) # reversed order @@ -3242,7 +3246,10 @@ def test_attributes(self): os.mkdir(dirname) filename = self.create_file("file.txt") if link: - os.link(filename, os.path.join(self.path, "link_file.txt")) + try: + os.link(filename, os.path.join(self.path, "link_file.txt")) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) if symlink: os.symlink(dirname, os.path.join(self.path, "symlink_dir"), target_is_directory=True) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 962adde38da541..e56e0d20844ec5 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -11,7 +11,6 @@ from unittest import mock from test import support -android_not_root = support.android_not_root TESTFN = support.TESTFN try: @@ -1911,10 +1910,12 @@ def test_is_fifo_false(self): self.assertFalse((P / 'fileA' / 'bah').is_fifo()) @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_is_fifo_true(self): P = self.cls(BASE, 'myfifo') - os.mkfifo(str(P)) + try: + os.mkfifo(str(P)) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) self.assertTrue(P.is_fifo()) self.assertFalse(P.is_socket()) self.assertFalse(P.is_file()) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index e6c5d08522aa8e..895ed48df1d8d2 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -6,6 +6,7 @@ import collections import struct import sys +import weakref import unittest from test import support @@ -26,8 +27,13 @@ has_c_implementation = False -class PickleTests(AbstractPickleModuleTests): - pass +class PyPickleTests(AbstractPickleModuleTests): + dump = staticmethod(pickle._dump) + dumps = staticmethod(pickle._dumps) + load = staticmethod(pickle._load) + loads = staticmethod(pickle._loads) + Pickler = pickle._Pickler + Unpickler = pickle._Unpickler class PyUnpicklerTests(AbstractUnpickleTests): @@ -112,6 +118,66 @@ class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, pickler = pickle._Pickler unpickler = pickle._Unpickler + @support.cpython_only + def test_pickler_reference_cycle(self): + def check(Pickler): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = Pickler(f, proto) + pickler.dump('abc') + self.assertEqual(self.loads(f.getvalue()), 'abc') + pickler = Pickler(io.BytesIO()) + self.assertEqual(pickler.persistent_id('def'), 'def') + r = weakref.ref(pickler) + del pickler + self.assertIsNone(r()) + + class PersPickler(self.pickler): + def persistent_id(subself, obj): + return obj + check(PersPickler) + + class PersPickler(self.pickler): + @classmethod + def persistent_id(cls, obj): + return obj + check(PersPickler) + + class PersPickler(self.pickler): + @staticmethod + def persistent_id(obj): + return obj + check(PersPickler) + + @support.cpython_only + def test_unpickler_reference_cycle(self): + def check(Unpickler): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = Unpickler(io.BytesIO(self.dumps('abc', proto))) + self.assertEqual(unpickler.load(), 'abc') + unpickler = Unpickler(io.BytesIO()) + self.assertEqual(unpickler.persistent_load('def'), 'def') + r = weakref.ref(unpickler) + del unpickler + self.assertIsNone(r()) + + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + return pid + check(PersUnpickler) + + class PersUnpickler(self.unpickler): + @classmethod + def persistent_load(cls, pid): + return pid + check(PersUnpickler) + + class PersUnpickler(self.unpickler): + @staticmethod + def persistent_load(pid): + return pid + check(PersUnpickler) + class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests): @@ -136,6 +202,9 @@ def get_dispatch_table(self): if has_c_implementation: + class CPickleTests(AbstractPickleModuleTests): + from _pickle import dump, dumps, load, loads, Pickler, Unpickler + class CUnpicklerTests(PyUnpicklerTests): unpickler = _pickle.Unpickler bad_stack_errors = (pickle.UnpicklingError,) @@ -189,7 +258,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('5P2n3i2n3iP') + basesize = support.calcobjsize('6P2n3i2n3iP') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -206,7 +275,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2Pn2P 2P2n2i5P 2P3n6P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n6P2n2i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -426,12 +495,12 @@ def test_multiprocessing_exceptions(self): def test_main(): - tests = [PickleTests, PyUnpicklerTests, PyPicklerTests, + tests = [PyPickleTests, PyUnpicklerTests, PyPicklerTests, PyPersPicklerTests, PyIdPersPicklerTests, PyDispatchTableTests, PyChainDispatchTableTests, CompatPickleTests] if has_c_implementation: - tests.extend([CUnpicklerTests, CPicklerTests, + tests.extend([CPickleTests, CUnpicklerTests, CPicklerTests, CPersPicklerTests, CIdPersPicklerTests, CDumpPickle_LoadPickle, DumpPickle_CLoadPickle, PyPicklerUnpicklerObjectTests, diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 86bebfa0266862..b3cab0e4fb338a 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -2,10 +2,9 @@ import pickletools from test import support from test.pickletester import AbstractPickleTests -from test.pickletester import AbstractPickleModuleTests import unittest -class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests): +class OptimizedPickleTests(AbstractPickleTests): def dumps(self, arg, proto=None): return pickletools.optimize(pickle.dumps(arg, proto)) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index ea6752b61d8f77..2cf4d3f5dfdb64 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -145,14 +145,14 @@ def test_sys_version(self): ("PyPy", "2.5.2", "trunk", "63378", ('63378', 'Mar 26 2009'), "") } - for (version_tag, subversion, sys_platform), info in \ + for (version_tag, scm, sys_platform), info in \ sys_versions.items(): sys.version = version_tag - if subversion is None: + if scm is None: if hasattr(sys, "_git"): del sys._git else: - sys._git = subversion + sys._git = scm if sys_platform is not None: sys.platform = sys_platform self.assertEqual(platform.python_implementation(), info[0]) diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index 7dd179a2c71e9e..ae8b6239690d32 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -169,6 +169,17 @@ def test_int(self): self.assertRaises(OverflowError, plistlib.dumps, pl, fmt=fmt) + def test_bytearray(self): + for pl in (b'', b"\0\1\2\3" * 10): + for fmt in ALL_FORMATS: + with self.subTest(pl=pl, fmt=fmt): + data = plistlib.dumps(bytearray(pl), fmt=fmt) + pl2 = plistlib.loads(data) + self.assertIsInstance(pl2, bytes) + self.assertEqual(pl2, pl) + data2 = plistlib.dumps(pl2, fmt=fmt) + self.assertEqual(data, data2) + def test_bytes(self): pl = self._create() data = plistlib.dumps(pl) @@ -431,6 +442,9 @@ def test_xml_encodings(self): pl2 = plistlib.loads(data) self.assertEqual(dict(pl), dict(pl2)) + +class TestBinaryPlistlib(unittest.TestCase): + def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers data = (b'bplist00' @@ -443,6 +457,47 @@ def test_nonstandard_refs_size(self): b'\x00\x00\x00\x00\x00\x00\x00\x13') self.assertEqual(plistlib.loads(data), {'a': 'b'}) + def test_dump_duplicates(self): + # Test effectiveness of saving duplicated objects + for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde', + datetime.datetime(2004, 10, 26, 10, 33, 33), + plistlib.Data(b'abcde'), bytearray(b'abcde'), + [12, 345], (12, 345), {'12': 345}): + with self.subTest(x=x): + data = plistlib.dumps([x]*1000, fmt=plistlib.FMT_BINARY) + self.assertLess(len(data), 1100, repr(data)) + + def test_identity(self): + for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde', + datetime.datetime(2004, 10, 26, 10, 33, 33), + plistlib.Data(b'abcde'), bytearray(b'abcde'), + [12, 345], (12, 345), {'12': 345}): + with self.subTest(x=x): + data = plistlib.dumps([x]*2, fmt=plistlib.FMT_BINARY) + a, b = plistlib.loads(data) + if isinstance(x, tuple): + x = list(x) + self.assertEqual(a, x) + self.assertEqual(b, x) + self.assertIs(a, b) + + def test_cycles(self): + # recursive list + a = [] + a.append(a) + b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) + self.assertIs(b[0], b) + # recursive tuple + a = ([],) + a[0].append(a) + b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) + self.assertIs(b[0][0], b) + # recursive dict + a = {} + a['x'] = a + b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) + self.assertIs(b['x'], b) + def test_large_timestamp(self): # Issue #26709: 32-bit timestamp out of range for ts in -2**31-1, 2**31: diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index dba50e02d94d03..44b8d6a7ad0796 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -2,7 +2,6 @@ from test import support from test.support.script_helper import assert_python_ok -android_not_root = support.android_not_root # Skip these tests if there is no posix module. posix = support.import_module('posix') @@ -504,15 +503,16 @@ def test_stat(self): posix.stat, list(os.fsencode(support.TESTFN))) @unittest.skipUnless(hasattr(posix, 'mkfifo'), "don't have mkfifo()") - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_mkfifo(self): support.unlink(support.TESTFN) - posix.mkfifo(support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) + try: + posix.mkfifo(support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) + except PermissionError as e: + self.skipTest('posix.mkfifo(): %s' % e) self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) @unittest.skipUnless(hasattr(posix, 'mknod') and hasattr(stat, 'S_IFIFO'), "don't have mknod()/S_IFIFO") - @unittest.skipIf(android_not_root, "mknod not allowed, non root user") def test_mknod(self): # Test using mknod() to create a FIFO (the only use specified # by POSIX). @@ -523,7 +523,7 @@ def test_mknod(self): except OSError as e: # Some old systems don't allow unprivileged users to use # mknod(), or only support creating device nodes. - self.assertIn(e.errno, (errno.EPERM, errno.EINVAL)) + self.assertIn(e.errno, (errno.EPERM, errno.EINVAL, errno.EACCES)) else: self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) @@ -533,7 +533,7 @@ def test_mknod(self): posix.mknod(path=support.TESTFN, mode=mode, device=0, dir_fd=None) except OSError as e: - self.assertIn(e.errno, (errno.EPERM, errno.EINVAL)) + self.assertIn(e.errno, (errno.EPERM, errno.EINVAL, errno.EACCES)) @unittest.skipUnless(hasattr(posix, 'stat'), 'test needs posix.stat()') @unittest.skipUnless(hasattr(posix, 'makedev'), 'test needs posix.makedev()') @@ -1018,11 +1018,13 @@ def test_utime_dir_fd(self): posix.close(f) @unittest.skipUnless(os.link in os.supports_dir_fd, "test needs dir_fd support in os.link()") - @unittest.skipIf(android_not_root, "hard link not allowed, non root user") def test_link_dir_fd(self): f = posix.open(posix.getcwd(), posix.O_RDONLY) try: posix.link(support.TESTFN, support.TESTFN + 'link', src_dir_fd=f, dst_dir_fd=f) + except PermissionError as e: + self.skipTest('posix.link(): %s' % e) + else: # should have same inodes self.assertEqual(posix.stat(support.TESTFN)[1], posix.stat(support.TESTFN + 'link')[1]) @@ -1042,7 +1044,6 @@ def test_mkdir_dir_fd(self): @unittest.skipUnless((os.mknod in os.supports_dir_fd) and hasattr(stat, 'S_IFIFO'), "test requires both stat.S_IFIFO and dir_fd support for os.mknod()") - @unittest.skipIf(android_not_root, "mknod not allowed, non root user") def test_mknod_dir_fd(self): # Test using mknodat() to create a FIFO (the only use specified # by POSIX). @@ -1054,7 +1055,7 @@ def test_mknod_dir_fd(self): except OSError as e: # Some old systems don't allow unprivileged users to use # mknod(), or only support creating device nodes. - self.assertIn(e.errno, (errno.EPERM, errno.EINVAL)) + self.assertIn(e.errno, (errno.EPERM, errno.EINVAL, errno.EACCES)) else: self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) finally: @@ -1126,12 +1127,15 @@ def test_unlink_dir_fd(self): posix.close(f) @unittest.skipUnless(os.mkfifo in os.supports_dir_fd, "test needs dir_fd support in os.mkfifo()") - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_mkfifo_dir_fd(self): support.unlink(support.TESTFN) f = posix.open(posix.getcwd(), posix.O_RDONLY) try: - posix.mkfifo(support.TESTFN, stat.S_IRUSR | stat.S_IWUSR, dir_fd=f) + try: + posix.mkfifo(support.TESTFN, + stat.S_IRUSR | stat.S_IWUSR, dir_fd=f) + except PermissionError as e: + self.skipTest('posix.mkfifo(): %s' % e) self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) finally: posix.close(f) diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index ac9cff789eea28..c13a7c9294f983 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -4,19 +4,11 @@ pwd = support.import_module('pwd') -def _getpwall(): - # Android does not have getpwall. - if hasattr(pwd, 'getpwall'): - return pwd.getpwall() - elif hasattr(pwd, 'getpwuid'): - return [pwd.getpwuid(0)] - else: - return [] - +@unittest.skipUnless(hasattr(pwd, 'getpwall'), 'Does not have getpwall()') class PwdTest(unittest.TestCase): def test_values(self): - entries = _getpwall() + entries = pwd.getpwall() for e in entries: self.assertEqual(len(e), 7) @@ -42,7 +34,7 @@ def test_values(self): # and check afterwards (done in test_values_extended) def test_values_extended(self): - entries = _getpwall() + entries = pwd.getpwall() entriesbyname = {} entriesbyuid = {} @@ -66,13 +58,12 @@ def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid, 3.14) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - if hasattr(pwd, 'getpwall'): - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwall, 42) # try to get some errors bynames = {} byuids = {} - for (n, p, u, g, gecos, d, s) in _getpwall(): + for (n, p, u, g, gecos, d, s) in pwd.getpwall(): bynames[n] = u byuids[u] = n @@ -106,17 +97,13 @@ def test_errors(self): # loop, say), pwd.getpwuid() might still be able to find data for that # uid. Using sys.maxint may provoke the same problems, but hopefully # it will be a more repeatable failure. - # Android accepts a very large span of uids including sys.maxsize and - # -1; it raises KeyError with 1 or 2 for example. fakeuid = sys.maxsize self.assertNotIn(fakeuid, byuids) - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + self.assertRaises(KeyError, pwd.getpwuid, fakeuid) # -1 shouldn't be a valid uid because it has a special meaning in many # uid-related functions - if not support.is_android: - self.assertRaises(KeyError, pwd.getpwuid, -1) + self.assertRaises(KeyError, pwd.getpwuid, -1) # should be out of uid_t range self.assertRaises(KeyError, pwd.getpwuid, 2**128) self.assertRaises(KeyError, pwd.getpwuid, -2**128) diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 52830b49aea31f..1926cffba263a2 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -357,7 +357,7 @@ def get_pydoc_html(module): def get_pydoc_link(module): "Returns a documentation web link of a module" dirname = os.path.dirname - basedir = dirname(dirname(os.path.realpath(__file__))) + basedir = dirname(dirname(__file__)) doc = pydoc.TextDoc() loc = doc.getdocloc(module, basedir=basedir) return loc @@ -824,7 +824,7 @@ def foo(data: typing.List[typing.Any], T = typing.TypeVar('T') class C(typing.Generic[T], typing.Mapping[int, str]): ... self.assertEqual(pydoc.render_doc(foo).splitlines()[-1], - 'f\x08fo\x08oo\x08o(data:List[Any], x:int)' + 'f\x08fo\x08oo\x08o(data: List[Any], x: int)' ' -> Iterator[Tuple[int, Any]]') self.assertEqual(pydoc.render_doc(C).splitlines()[2], 'class C\x08C(typing.Mapping)') diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 35466c1eae39d0..6ee906c4d2665a 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -54,7 +54,7 @@ def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): self.result = block_func(*block_args) # If block_func returned before our thread made the call, we failed! if not thread.startedEvent.is_set(): - self.fail("blocking function '%r' appeared not to block" % + self.fail("blocking function %r appeared not to block" % block_func) return self.result finally: diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index fc015e4ed9be0f..2344d71abf209d 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -331,21 +331,21 @@ def test_re_split(self): ['', 'a', '', '', 'c']) for sep, expected in [ - (':*', ['', 'a', 'b', 'c']), - ('(?::*)', ['', 'a', 'b', 'c']), - ('(:*)', ['', ':', 'a', ':', 'b', '::', 'c']), - ('(:)*', ['', ':', 'a', ':', 'b', ':', 'c']), + (':*', ['', 'a', 'b', 'c', '']), + ('(?::*)', ['', 'a', 'b', 'c', '']), + ('(:*)', ['', ':', 'a', ':', 'b', '::', 'c', '', '']), + ('(:)*', ['', ':', 'a', ':', 'b', ':', 'c', None, '']), ]: - with self.subTest(sep=sep), self.assertWarns(FutureWarning): + with self.subTest(sep=sep): self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) for sep, expected in [ - ('', [':a:b::c']), - (r'\b', [':a:b::c']), - (r'(?=:)', [':a:b::c']), - (r'(?<=:)', [':a:b::c']), + ('', ['', ':', 'a', ':', 'b', ':', ':', 'c', '']), + (r'\b', [':', 'a', ':', 'b', '::', 'c', '']), + (r'(?=:)', ['', ':a', ':b', ':', ':c']), + (r'(?<=:)', [':', 'a:', 'b:', ':', 'c']), ]: - with self.subTest(sep=sep), self.assertRaises(ValueError): + with self.subTest(sep=sep): self.assertTypedEqual(re.split(sep, ':a:b::c'), expected) def test_qualified_re_split(self): @@ -356,9 +356,8 @@ def test_qualified_re_split(self): ['', ':', 'a', ':', 'b::c']) self.assertEqual(re.split("(:+)", ":a:b::c", maxsplit=2), ['', ':', 'a', ':', 'b::c']) - with self.assertWarns(FutureWarning): - self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), - ['', ':', 'a', ':', 'b::c']) + self.assertEqual(re.split("(:*)", ":a:b::c", maxsplit=2), + ['', ':', 'a', ':', 'b::c']) def test_re_findall(self): self.assertEqual(re.findall(":+", "abc"), []) @@ -914,6 +913,51 @@ def test_not_literal(self): self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b") self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb") + def test_possible_set_operations(self): + s = bytes(range(128)).decode() + with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9--1]') + self.assertEqual(p.findall(s), list('-./0123456789')) + self.assertEqual(re.findall(r'[--1]', s), list('-./01')) + with self.assertWarns(FutureWarning): + p = re.compile(r'[%--1]') + self.assertEqual(p.findall(s), list("%&'()*+,-1")) + with self.assertWarns(FutureWarning): + p = re.compile(r'[%--]') + self.assertEqual(p.findall(s), list("%&'()*+,-")) + + with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9&&1]') + self.assertEqual(p.findall(s), list('&0123456789')) + with self.assertWarns(FutureWarning): + p = re.compile(r'[\d&&1]') + self.assertEqual(p.findall(s), list('&0123456789')) + self.assertEqual(re.findall(r'[&&1]', s), list('&1')) + + with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9||a]') + self.assertEqual(p.findall(s), list('0123456789a|')) + with self.assertWarns(FutureWarning): + p = re.compile(r'[\d||a]') + self.assertEqual(p.findall(s), list('0123456789a|')) + self.assertEqual(re.findall(r'[||1]', s), list('1|')) + + with self.assertWarns(FutureWarning): + p = re.compile(r'[0-9~~1]') + self.assertEqual(p.findall(s), list('0123456789~')) + with self.assertWarns(FutureWarning): + p = re.compile(r'[\d~~1]') + self.assertEqual(p.findall(s), list('0123456789~')) + self.assertEqual(re.findall(r'[~~1]', s), list('1~')) + + with self.assertWarns(FutureWarning): + p = re.compile(r'[[0-9]|]') + self.assertEqual(p.findall(s), list('0123456789[]')) + + with self.assertWarns(FutureWarning): + p = re.compile(r'[[:digit:]|]') + self.assertEqual(p.findall(s), list(':[]dgit')) + def test_search_coverage(self): self.assertEqual(re.search(r"\s(b)", " b").group(1), "b") self.assertEqual(re.search(r"a\s", "a ").group(0), "a ") @@ -932,7 +976,7 @@ def assertMatch(self, pattern, text, match=None, span=None, self.assertEqual(m.group(), match) self.assertEqual(m.span(), span) - LITERAL_CHARS = string.ascii_letters + string.digits + '!"%&\',/:;<=>@_`~' + LITERAL_CHARS = string.ascii_letters + string.digits + '!"%\',/:;<=>@_`' def test_re_escape(self): p = ''.join(chr(i) for i in range(256)) @@ -1706,6 +1750,25 @@ def test_match_repr(self): "span=(3, 5), match='bb'>" % (type(second).__module__, type(second).__qualname__)) + def test_zerowidth(self): + # Issues 852532, 1647489, 3262, 25054. + self.assertEqual(re.split(r"\b", "a::bc"), ['', 'a', '::', 'bc', '']) + self.assertEqual(re.split(r"\b|:+", "a::bc"), ['', 'a', '', 'bc', '']) + self.assertEqual(re.split(r"(?'. + # The directory of argv[0] must match the directory of the Python + # executable for the Popen() call to python to succeed as the directory + # path may be used by Py_GetPath() to build the default module search + # path. + stdin_fname = os.path.join(os.path.dirname(sys.executable), "") + cmd_line = [stdin_fname, '-E', '-i'] + cmd_line.extend(args) + + # Set TERM=vt100, for the rationale see the comments in spawn_python() of + # test.support.script_helper. + env = kw.setdefault('env', dict(os.environ)) + env['TERM'] = 'vt100' + return subprocess.Popen(cmd_line, executable=sys.executable, + stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + **kw) + +class TestInteractiveInterpreter(unittest.TestCase): + + @cpython_only + def test_no_memory(self): + # Issue #30696: Fix the interactive interpreter looping endlessly when + # no memory. Check also that the fix does not break the interactive + # loop when an exception is raised. + user_input = """ + import sys, _testcapi + 1/0 + print('After the exception.') + _testcapi.set_nomemory(0) + sys.exit(0) + """ + user_input = dedent(user_input) + user_input = user_input.encode() + p = spawn_repl() + with SuppressCrashReport(): + p.stdin.write(user_input) + output = kill_python(p) + self.assertIn(b'After the exception.', output) + # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr. + self.assertIn(p.returncode, (1, 120)) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 5c1a571f1b6d70..75198b70ad4ff5 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -3,7 +3,6 @@ import threading import unittest import urllib.robotparser -from collections import namedtuple from test import support from http.server import BaseHTTPRequestHandler, HTTPServer @@ -87,6 +86,10 @@ def test_request_rate(self): self.parser.crawl_delay(agent), self.crawl_delay ) if self.request_rate: + self.assertIsInstance( + self.parser.request_rate(agent), + urllib.robotparser.RequestRate + ) self.assertEqual( self.parser.request_rate(agent).requests, self.request_rate.requests @@ -108,7 +111,7 @@ class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): Disallow: /%7ejoe/index.html """ agent = 'figtree' - request_rate = namedtuple('req_rate', 'requests seconds')(9, 30) + request_rate = urllib.robotparser.RequestRate(9, 30) crawl_delay = 3 good = [('figtree', '/foo.html')] bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', @@ -237,7 +240,7 @@ class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): Request-rate: 3/15 Disallow: /cyberworld/map/ """ - request_rate = namedtuple('req_rate', 'requests seconds')(3, 15) + request_rate = urllib.robotparser.RequestRate(3, 15) crawl_delay = 1 good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html'] diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 4a72b4a0764bc1..f3cf43e50516c3 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -22,7 +22,7 @@ import zipfile from test import support -from test.support import TESTFN, android_not_root +from test.support import TESTFN TESTFN2 = TESTFN + "2" @@ -769,7 +769,6 @@ def test_copytree_winerror(self, mock_patch): @unittest.skipIf(os.name == 'nt', 'temporarily disabled on Windows') @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') - @unittest.skipIf(android_not_root, "hard links not allowed, non root user") def test_dont_copy_file_onto_link_to_itself(self): # bug 851123. os.mkdir(TESTFN) @@ -778,7 +777,10 @@ def test_dont_copy_file_onto_link_to_itself(self): try: with open(src, 'w') as f: f.write('cheddar') - os.link(src, dst) + try: + os.link(src, dst) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) self.assertRaises(shutil.SameFileError, shutil.copyfile, src, dst) with open(src, 'r') as f: self.assertEqual(f.read(), 'cheddar') @@ -822,9 +824,11 @@ def test_rmtree_on_symlink(self): # Issue #3002: copyfile and copytree block indefinitely on named pipes @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_copyfile_named_pipe(self): - os.mkfifo(TESTFN) + try: + os.mkfifo(TESTFN) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) try: self.assertRaises(shutil.SpecialFileError, shutil.copyfile, TESTFN, TESTFN2) @@ -833,7 +837,6 @@ def test_copyfile_named_pipe(self): finally: os.remove(TESTFN) - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') @support.skip_unless_symlink def test_copytree_named_pipe(self): @@ -842,7 +845,10 @@ def test_copytree_named_pipe(self): subdir = os.path.join(TESTFN, "subdir") os.mkdir(subdir) pipe = os.path.join(subdir, "mypipe") - os.mkfifo(pipe) + try: + os.mkfifo(pipe) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) try: shutil.copytree(TESTFN, TESTFN2) except shutil.Error as e: diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index fb16d091f90274..b97a7688a609c6 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5136,8 +5136,6 @@ def test_set_inheritable_cloexec(self): 0) - @unittest.skipUnless(hasattr(socket, "socketpair"), - "need socket.socketpair()") def test_socketpair(self): s1, s2 = socket.socketpair() self.addCleanup(s1.close) @@ -5575,6 +5573,9 @@ def create_alg(self, typ, name): else: return sock + # bpo-31705: On kernel older than 4.5, sendto() failed with ENOKEY, + # at least on ppc64le architecture + @support.requires_linux_version(4, 5) def test_sha256(self): expected = bytes.fromhex("ba7816bf8f01cfea414140de5dae2223b00361a396" "177a9cb410ff61f20015ad") diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index a23373fbdf39a1..6584ba5ba864e2 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -60,10 +60,14 @@ def simple_subprocess(testcase): if pid == 0: # Don't raise an exception; it would be caught by the test harness. os._exit(72) - yield None - pid2, status = os.waitpid(pid, 0) - testcase.assertEqual(pid2, pid) - testcase.assertEqual(72 << 8, status) + try: + yield None + except: + raise + finally: + pid2, status = os.waitpid(pid, 0) + testcase.assertEqual(pid2, pid) + testcase.assertEqual(72 << 8, status) class SocketServerTest(unittest.TestCase): @@ -108,7 +112,12 @@ def handle(self): self.wfile.write(line) if verbose: print("creating server") - server = MyServer(addr, MyHandler) + try: + server = MyServer(addr, MyHandler) + except PermissionError as e: + # Issue 29184: cannot bind() a Unix socket on Android. + self.skipTest('Cannot create server (%s, %s): %s' % + (svrcls, addr, e)) self.assertEqual(server.server_address, server.socket.getsockname()) return server diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index aa2429ac9827ae..c65290b945f6c9 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -512,10 +512,11 @@ def fail(cert, hostname): fail(cert, 'Xa.com') fail(cert, '.a.com') - # only match one left-most wildcard + # only match wildcards when they are the only thing + # in left-most segment cert = {'subject': ((('commonName', 'f*.com'),),)} - ok(cert, 'foo.com') - ok(cert, 'f.com') + fail(cert, 'foo.com') + fail(cert, 'f.com') fail(cert, 'bar.com') fail(cert, 'foo.a.com') fail(cert, 'bar.foo.com') @@ -552,8 +553,8 @@ def fail(cert, hostname): # are supported. idna = 'www*.pythön.org'.encode("idna").decode("ascii") cert = {'subject': ((('commonName', idna),),)} - ok(cert, 'www.pythön.org'.encode("idna").decode("ascii")) - ok(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www.pythön.org'.encode("idna").decode("ascii")) + fail(cert, 'www1.pythön.org'.encode("idna").decode("ascii")) fail(cert, 'ftp.pythön.org'.encode("idna").decode("ascii")) fail(cert, 'pythön.org'.encode("idna").decode("ascii")) @@ -637,7 +638,7 @@ def fail(cert, hostname): # Issue #17980: avoid denials of service by refusing more than one # wildcard per fragment. cert = {'subject': ((('commonName', 'a*b.com'),),)} - ok(cert, 'axxb.com') + fail(cert, 'axxb.com') cert = {'subject': ((('commonName', 'a*b.co*'),),)} fail(cert, 'axxb.com') cert = {'subject': ((('commonName', 'a*b*.com'),),)} diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index cd02a6ee37452b..73cd901bdbf5c1 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -1,7 +1,7 @@ import unittest import os import sys -from test.support import TESTFN, import_fresh_module, android_not_root +from test.support import TESTFN, import_fresh_module c_stat = import_fresh_module('stat', fresh=['_stat']) py_stat = import_fresh_module('stat', blocked=['_stat']) @@ -168,9 +168,11 @@ def test_link(self): self.assertS_IS("LNK", st_mode) @unittest.skipUnless(hasattr(os, 'mkfifo'), 'os.mkfifo not available') - @unittest.skipIf(android_not_root, "mkfifo not allowed, non root user") def test_fifo(self): - os.mkfifo(TESTFN, 0o700) + try: + os.mkfifo(TESTFN, 0o700) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) st_mode, modestr = self.get_mode() self.assertEqual(modestr, 'prwx------') self.assertS_IS("FIFO", st_mode) diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 72b1910c3838f2..ec305e54ff24f0 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -58,8 +58,10 @@ def setUp(self): import java java.util.Locale.setDefault(java.util.Locale.US) except ImportError: - import locale - locale.setlocale(locale.LC_TIME, 'C') + from locale import setlocale, LC_TIME + saved_locale = setlocale(LC_TIME) + setlocale(LC_TIME, 'C') + self.addCleanup(setlocale, LC_TIME, saved_locale) def test_strftime(self): now = time.time() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8632837780c9c7..e06f7b8e9952b8 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -6,6 +6,7 @@ import shutil import socket import stat +import subprocess import sys import tempfile import time @@ -426,6 +427,120 @@ def test_reap_children(self): # pending child process support.reap_children() + def check_options(self, args, func): + code = f'from test.support import {func}; print(repr({func}()))' + cmd = [sys.executable, *args, '-c', code] + env = {key: value for key, value in os.environ.items() + if not key.startswith('PYTHON')} + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + universal_newlines=True, + env=env) + self.assertEqual(proc.stdout.rstrip(), repr(args)) + self.assertEqual(proc.returncode, 0) + + def test_args_from_interpreter_flags(self): + # Test test.support.args_from_interpreter_flags() + for opts in ( + # no option + [], + # single option + ['-B'], + ['-s'], + ['-S'], + ['-E'], + ['-v'], + ['-b'], + ['-q'], + # same option multiple times + ['-bb'], + ['-vvv'], + # -W options + ['-Wignore'], + # -X options + ['-X', 'dev'], + ['-Wignore', '-X', 'dev'], + ['-X', 'faulthandler'], + ['-X', 'importtime'], + ['-X', 'showalloccount'], + ['-X', 'showrefcount'], + ['-X', 'tracemalloc'], + ['-X', 'tracemalloc=3'], + ): + with self.subTest(opts=opts): + self.check_options(opts, 'args_from_interpreter_flags') + + def test_optim_args_from_interpreter_flags(self): + # Test test.support.optim_args_from_interpreter_flags() + for opts in ( + # no option + [], + ['-O'], + ['-OO'], + ['-OOOO'], + ): + with self.subTest(opts=opts): + self.check_options(opts, 'optim_args_from_interpreter_flags') + + def test_match_test(self): + class Test: + def __init__(self, test_id): + self.test_id = test_id + + def id(self): + return self.test_id + + test_access = Test('test.test_os.FileTests.test_access') + test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests([]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match all using None + support.set_match_tests(None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests([test_access.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(['test_os']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(['test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(['filetests']) + self.assertFalse(support.match_test(test_access)) + support.set_match_tests(['FileTests']) + self.assertTrue(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(['*test_os.*.test_*']) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests([test_access.id(), test_chdir.id()]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + support.set_match_tests(['test_access', 'DONTMATCH']) + self.assertTrue(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # XXX -follows a list of untested API # make_legacy_pyc # is_resource_enabled @@ -447,7 +562,6 @@ def test_reap_children(self): # threading_cleanup # reap_threads # strip_python_stderr - # args_from_interpreter_flags # can_symlink # skip_unless_symlink # SuppressCrashReport diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 5d36581859b775..e161f56d30f214 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -125,19 +125,38 @@ From ast_for_call(): ->>> def f(it, *varargs): +>>> def f(it, *varargs, **kwargs): ... return list(it) >>> L = range(10) >>> f(x for x in L) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] >>> f(x for x in L, 1) Traceback (most recent call last): -SyntaxError: Generator expression must be parenthesized if not sole argument +SyntaxError: Generator expression must be parenthesized +>>> f(x for x in L, y=1) +Traceback (most recent call last): +SyntaxError: Generator expression must be parenthesized +>>> f(x for x in L, *[]) +Traceback (most recent call last): +SyntaxError: Generator expression must be parenthesized +>>> f(x for x in L, **{}) +Traceback (most recent call last): +SyntaxError: Generator expression must be parenthesized +>>> f(L, x for x in L) +Traceback (most recent call last): +SyntaxError: Generator expression must be parenthesized >>> f(x for x in L, y for y in L) Traceback (most recent call last): -SyntaxError: Generator expression must be parenthesized if not sole argument +SyntaxError: Generator expression must be parenthesized +>>> f(x for x in L,) +Traceback (most recent call last): +SyntaxError: Generator expression must be parenthesized >>> f((x for x in L), 1) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +>>> class C(x for x in L): +... pass +Traceback (most recent call last): +SyntaxError: invalid syntax >>> def g(*args, **kwargs): ... print(args, sorted(kwargs.items())) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 50eb1b7c5785d7..6346094ad08b4c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -526,10 +526,12 @@ def test_sys_flags(self): attrs = ("debug", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", - "bytes_warning", "quiet", "hash_randomization", "isolated") + "bytes_warning", "quiet", "hash_randomization", "isolated", + "dev_mode") for attr in attrs: self.assertTrue(hasattr(sys.flags, attr), attr) - self.assertEqual(type(getattr(sys.flags, attr)), int, attr) + attr_type = bool if attr == "dev_mode" else int + self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) self.assertEqual(len(sys.flags), len(attrs)) @@ -753,8 +755,15 @@ def test_debugmallocstats(self): @unittest.skipUnless(hasattr(sys, "getallocatedblocks"), "sys.getallocatedblocks unavailable on this build") def test_getallocatedblocks(self): + try: + import _testcapi + except ImportError: + with_pymalloc = support.with_pymalloc() + else: + alloc_name = _testcapi.pymem_getallocatorsname() + with_pymalloc = (alloc_name in ('pymalloc', 'pymalloc_debug')) + # Some sanity checks - with_pymalloc = sysconfig.get_config_var('WITH_PYMALLOC') a = sys.getallocatedblocks() self.assertIs(type(a), int) if with_pymalloc: @@ -808,6 +817,39 @@ def test_getandroidapilevel(self): self.assertIsInstance(level, int) self.assertGreater(level, 0) + def test_sys_tracebacklimit(self): + code = """if 1: + import sys + def f1(): + 1 / 0 + def f2(): + f1() + sys.tracebacklimit = %r + f2() + """ + def check(tracebacklimit, expected): + p = subprocess.Popen([sys.executable, '-c', code % tracebacklimit], + stderr=subprocess.PIPE) + out = p.communicate()[1] + self.assertEqual(out.splitlines(), expected) + + traceback = [ + b'Traceback (most recent call last):', + b' File "", line 8, in ', + b' File "", line 6, in f2', + b' File "", line 4, in f1', + b'ZeroDivisionError: division by zero' + ] + check(10, traceback) + check(3, traceback) + check(2, traceback[:1] + traceback[2:]) + check(1, traceback[:1] + traceback[3:]) + check(0, [traceback[-1]]) + check(-1, [traceback[-1]]) + check(1<<1000, traceback) + check(-1<<1000, [traceback[-1]]) + check(None, traceback) + @test.support.cpython_only class SizeofTest(unittest.TestCase): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index f0a5b21ab5a743..179cbc6dfffca7 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1147,7 +1147,10 @@ def test_link_size(self): target = os.path.join(TEMPDIR, "link_target") with open(target, "wb") as fobj: fobj.write(b"aaa") - os.link(target, link) + try: + os.link(target, link) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) try: tar = tarfile.open(tmpname, self.mode) try: @@ -1609,7 +1612,10 @@ def setUp(self): with open(self.foo, "wb") as fobj: fobj.write(b"foo") - os.link(self.foo, self.bar) + try: + os.link(self.foo, self.bar) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) self.tar = tarfile.open(tmpname, "w") self.tar.add(self.foo) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index b44646da709ffb..eda3885ad575f3 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -47,6 +47,12 @@ class _PyTime(enum.IntEnum): ) +def busy_wait(duration): + deadline = time.monotonic() + duration + while time.monotonic() < deadline: + pass + + class TimeTestCase(unittest.TestCase): def setUp(self): @@ -81,6 +87,10 @@ def check_ns(sec, ns): check_ns(time.process_time(), time.process_time_ns()) + if hasattr(time, 'thread_time'): + check_ns(time.thread_time(), + time.thread_time_ns()) + if hasattr(time, 'clock_gettime'): check_ns(time.clock_gettime(time.CLOCK_REALTIME), time.clock_gettime_ns(time.CLOCK_REALTIME)) @@ -486,10 +496,57 @@ def test_process_time(self): # on Windows self.assertLess(stop - start, 0.020) + # process_time() should include CPU time spent in any thread + start = time.process_time() + busy_wait(0.100) + stop = time.process_time() + self.assertGreaterEqual(stop - start, 0.020) # machine busy? + + t = threading.Thread(target=busy_wait, args=(0.100,)) + start = time.process_time() + t.start() + t.join() + stop = time.process_time() + self.assertGreaterEqual(stop - start, 0.020) # machine busy? + info = time.get_clock_info('process_time') self.assertTrue(info.monotonic) self.assertFalse(info.adjustable) + def test_thread_time(self): + if not hasattr(time, 'thread_time'): + if sys.platform.startswith(('linux', 'win')): + self.fail("time.thread_time() should be available on %r" + % (sys.platform,)) + else: + self.skipTest("need time.thread_time") + + # thread_time() should not include time spend during a sleep + start = time.thread_time() + time.sleep(0.100) + stop = time.thread_time() + # use 20 ms because thread_time() has usually a resolution of 15 ms + # on Windows + self.assertLess(stop - start, 0.020) + + # thread_time() should include CPU time spent in current thread... + start = time.thread_time() + busy_wait(0.100) + stop = time.thread_time() + self.assertGreaterEqual(stop - start, 0.020) # machine busy? + + # ...but not in other threads + t = threading.Thread(target=busy_wait, args=(0.100,)) + start = time.thread_time() + t.start() + t.join() + stop = time.thread_time() + self.assertLess(stop - start, 0.020) + + info = time.get_clock_info('thread_time') + self.assertTrue(info.monotonic) + self.assertFalse(info.adjustable) + @unittest.skipUnless(hasattr(time, 'clock_settime'), 'need time.clock_settime') def test_monotonic_settime(self): diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 780942a8061556..b0a0e1b2d7860a 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -171,6 +171,9 @@ def allocate_bytes4(size): traces = tracemalloc._get_traces() + obj1_traceback._frames = tuple(reversed(obj1_traceback._frames)) + obj2_traceback._frames = tuple(reversed(obj2_traceback._frames)) + trace1 = self.find_trace(traces, obj1_traceback) trace2 = self.find_trace(traces, obj2_traceback) domain1, size1, traceback1 = trace1 @@ -537,11 +540,11 @@ def test_snapshot_group_by_cumulative(self): def test_trace_format(self): snapshot, snapshot2 = create_snapshots() trace = snapshot.traces[0] - self.assertEqual(str(trace), 'a.py:2: 10 B') + self.assertEqual(str(trace), 'b.py:4: 10 B') traceback = trace.traceback - self.assertEqual(str(traceback), 'a.py:2') + self.assertEqual(str(traceback), 'b.py:4') frame = traceback[0] - self.assertEqual(str(frame), 'a.py:2') + self.assertEqual(str(frame), 'b.py:4') def test_statistic_format(self): snapshot, snapshot2 = create_snapshots() @@ -574,17 +577,32 @@ def getline(filename, lineno): side_effect=getline): tb = snapshot.traces[0].traceback self.assertEqual(tb.format(), + [' File "b.py", line 4', + ' ', + ' File "a.py", line 2', + ' ']) + + self.assertEqual(tb.format(limit=1), + [' File "a.py", line 2', + ' ']) + + self.assertEqual(tb.format(limit=-1), + [' File "b.py", line 4', + ' ']) + + self.assertEqual(tb.format(most_recent_first=True), [' File "a.py", line 2', ' ', ' File "b.py", line 4', ' ']) - self.assertEqual(tb.format(limit=1), + self.assertEqual(tb.format(limit=1, most_recent_first=True), [' File "a.py", line 2', ' ']) - self.assertEqual(tb.format(limit=-1), - []) + self.assertEqual(tb.format(limit=-1, most_recent_first=True), + [' File "b.py", line 4', + ' ']) class TestFilters(unittest.TestCase): @@ -829,16 +847,23 @@ def test_env_limit(self): stdout = stdout.rstrip() self.assertEqual(stdout, b'10') + def check_env_var_invalid(self, nframe): + with support.SuppressCrashReport(): + ok, stdout, stderr = assert_python_failure( + '-c', 'pass', + PYTHONTRACEMALLOC=str(nframe)) + + if b'ValueError: the number of frames must be in range' in stderr: + return + if b'PYTHONTRACEMALLOC: invalid number of frames' in stderr: + return + self.fail(f"unexpeced output: {stderr!a}") + + def test_env_var_invalid(self): for nframe in (-1, 0, 2**30): with self.subTest(nframe=nframe): - with support.SuppressCrashReport(): - ok, stdout, stderr = assert_python_failure( - '-c', 'pass', - PYTHONTRACEMALLOC=str(nframe)) - self.assertIn(b'PYTHONTRACEMALLOC: invalid ' - b'number of frames', - stderr) + self.check_env_var_invalid(nframe) def test_sys_xoptions(self): for xoptions, nframe in ( @@ -852,15 +877,21 @@ def test_sys_xoptions(self): stdout = stdout.rstrip() self.assertEqual(stdout, str(nframe).encode('ascii')) + def check_sys_xoptions_invalid(self, nframe): + args = ('-X', 'tracemalloc=%s' % nframe, '-c', 'pass') + with support.SuppressCrashReport(): + ok, stdout, stderr = assert_python_failure(*args) + + if b'ValueError: the number of frames must be in range' in stderr: + return + if b'-X tracemalloc=NFRAME: invalid number of frames' in stderr: + return + self.fail(f"unexpeced output: {stderr!a}") + def test_sys_xoptions_invalid(self): for nframe in (-1, 0, 2**30): with self.subTest(nframe=nframe): - with support.SuppressCrashReport(): - args = ('-X', 'tracemalloc=%s' % nframe, '-c', 'pass') - ok, stdout, stderr = assert_python_failure(*args) - self.assertIn(b'-X tracemalloc=NFRAME: invalid ' - b'number of frames', - stderr) + self.check_sys_xoptions_invalid(nframe) @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 28133a3560f3c5..e822ff7bb5eeb1 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -844,6 +844,38 @@ def func(ns): self.assertEqual(C.y, 1) self.assertEqual(C.z, 2) + def test_new_class_with_mro_entry(self): + class A: pass + class C: + def __mro_entries__(self, bases): + return (A,) + c = C() + D = types.new_class('D', (c,), {}) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + + def test_new_class_with_mro_entry_none(self): + class A: pass + class B: pass + class C: + def __mro_entries__(self, bases): + return () + c = C() + D = types.new_class('D', (A, c, B), {}) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + + def test_new_class_with_mro_entry_error(self): + class A: pass + class C: + def __mro_entries__(self, bases): + return A + c = C() + with self.assertRaises(TypeError): + types.new_class('D', (c,), {}) + # Many of the following tests are derived from test_descr.py def test_prepare_class(self): # Basic test of metaclass derivation @@ -886,6 +918,28 @@ def __prepare__(*args): class Bar(metaclass=BadMeta()): pass + def test_resolve_bases(self): + class A: pass + class B: pass + class C: + def __mro_entries__(self, bases): + if A in bases: + return () + return (A,) + c = C() + self.assertEqual(types.resolve_bases(()), ()) + self.assertEqual(types.resolve_bases((c,)), (A,)) + self.assertEqual(types.resolve_bases((C,)), (C,)) + self.assertEqual(types.resolve_bases((A, C)), (A, C)) + self.assertEqual(types.resolve_bases((c, A)), (A,)) + self.assertEqual(types.resolve_bases((A, c)), (A,)) + x = (A,) + y = (C,) + z = (A, C) + t = (A, C, B) + for bases in [x, y, z, t]: + self.assertIs(types.resolve_bases(bases), bases) + def test_metaclass_derivation(self): # issue1294232: correct metaclass calculation new_calls = [] # to check the order of __new__ calls diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a3b6eb933935e2..4ae1ebf05f324b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -37,6 +37,9 @@ from test import mod_generics_cache +PY36 = sys.version_info[:2] >= (3, 6) + + class BaseTestCase(TestCase): def assertIsSubclass(self, cls, class_or_tuple, msg=None): @@ -633,6 +636,27 @@ def test_init(self): with self.assertRaises(TypeError): Generic[T, S, T] + @skipUnless(PY36, "__init_subclass__ support required") + def test_init_subclass(self): + class X(typing.Generic[T]): + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.attr = 42 + class Y(X): + pass + self.assertEqual(Y.attr, 42) + with self.assertRaises(AttributeError): + X.attr + X.attr = 1 + Y.attr = 2 + class Z(Y): + pass + class W(X[int]): + pass + self.assertEqual(Y.attr, 2) + self.assertEqual(Z.attr, 42) + self.assertEqual(W.attr, 42) + def test_repr(self): self.assertEqual(repr(SimpleMapping), __name__ + '.' + 'SimpleMapping') @@ -1080,6 +1104,30 @@ class Node(Generic[T]): ... self.assertTrue(t is copy(t)) self.assertTrue(t is deepcopy(t)) + def test_copy_generic_instances(self): + T = TypeVar('T') + class C(Generic[T]): + def __init__(self, attr: T) -> None: + self.attr = attr + + c = C(42) + self.assertEqual(copy(c).attr, 42) + self.assertEqual(deepcopy(c).attr, 42) + self.assertIsNot(copy(c), c) + self.assertIsNot(deepcopy(c), c) + c.attr = 1 + self.assertEqual(copy(c).attr, 1) + self.assertEqual(deepcopy(c).attr, 1) + ci = C[int](42) + self.assertEqual(copy(ci).attr, 42) + self.assertEqual(deepcopy(ci).attr, 42) + self.assertIsNot(copy(ci), ci) + self.assertIsNot(deepcopy(ci), ci) + ci.attr = 1 + self.assertEqual(copy(ci).attr, 1) + self.assertEqual(deepcopy(ci).attr, 1) + self.assertEqual(ci.__orig_class__, C[int]) + def test_weakref_all(self): T = TypeVar('T') things = [Any, Union[T, int], Callable[..., T], Tuple[Any, Any], @@ -1580,8 +1628,6 @@ async def __aexit__(self, etype, eval, tb): asyncio = None AwaitableWrapper = AsyncIteratorWrapper = ACM = object -PY36 = sys.version_info[:2] >= (3, 6) - PY36_TESTS = """ from test import ann_module, ann_module2, ann_module3 from typing import AsyncContextManager diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 341007b6507d52..2b77863e52e490 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2068,11 +2068,14 @@ def test_codecs_errors(self): # Error handling (wrong arguments) self.assertRaises(TypeError, "hello".encode, 42, 42, 42) - # Error handling (lone surrogate in PyUnicode_TransformDecimalToASCII()) - self.assertRaises(UnicodeError, float, "\ud800") - self.assertRaises(UnicodeError, float, "\udf00") - self.assertRaises(UnicodeError, complex, "\ud800") - self.assertRaises(UnicodeError, complex, "\udf00") + # Error handling (lone surrogate in + # _PyUnicode_TransformDecimalAndSpaceToASCII()) + self.assertRaises(ValueError, int, "\ud800") + self.assertRaises(ValueError, int, "\udf00") + self.assertRaises(ValueError, float, "\ud800") + self.assertRaises(ValueError, float, "\udf00") + self.assertRaises(ValueError, complex, "\ud800") + self.assertRaises(ValueError, complex, "\udf00") def test_codecs(self): # Encoding diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 083c2aa8aab54d..f21bd6dfa15235 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -512,61 +512,62 @@ def test_find_mac(self): self.assertEqual(mac, 0x1234567890ab) - def check_node(self, node, requires=None, network=False): + def check_node(self, node, requires=None): if requires and node is None: self.skipTest('requires ' + requires) hex = '%012x' % node if support.verbose >= 2: print(hex, end=' ') - if network: - # 47 bit will never be set in IEEE 802 addresses obtained - # from network cards. - self.assertFalse(node & 0x010000000000, hex) self.assertTrue(0 < node < (1 << 48), "%s is not an RFC 4122 node ID" % hex) @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): node = self.uuid._ifconfig_getnode() - self.check_node(node, 'ifconfig', True) + self.check_node(node, 'ifconfig') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ip_getnode(self): node = self.uuid._ip_getnode() - self.check_node(node, 'ip', True) + self.check_node(node, 'ip') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_arp_getnode(self): node = self.uuid._arp_getnode() - self.check_node(node, 'arp', True) + self.check_node(node, 'arp') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_lanscan_getnode(self): node = self.uuid._lanscan_getnode() - self.check_node(node, 'lanscan', True) + self.check_node(node, 'lanscan') @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_netstat_getnode(self): node = self.uuid._netstat_getnode() - self.check_node(node, 'netstat', True) + self.check_node(node, 'netstat') @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_ipconfig_getnode(self): node = self.uuid._ipconfig_getnode() - self.check_node(node, 'ipconfig', True) + self.check_node(node, 'ipconfig') @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') @unittest.skipUnless(importable('netbios'), 'requires netbios') def test_netbios_getnode(self): node = self.uuid._netbios_getnode() - self.check_node(node, network=True) + self.check_node(node) def test_random_getnode(self): node = self.uuid._random_getnode() - # Least significant bit of first octet must be set. - self.assertTrue(node & 0x010000000000, '%012x' % node) + # The multicast bit, i.e. the least significant bit of first octet, + # must be set for randomly generated MAC addresses. See RFC 4122, + # $4.1.6. + self.assertTrue(node & (1 << 40), '%012x' % node) self.check_node(node) + node2 = self.uuid._random_getnode() + self.assertNotEqual(node2, node, '%012x' % node) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_unix_getnode(self): if not importable('_uuid') and not importable('ctypes'): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 2d76e65271b0b7..c5504267706770 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -369,7 +369,9 @@ def do_test_with_pip(self, system_site_packages): self.fail(msg.format(exc, details)) # Ensure pip is available in the virtual environment envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) - cmd = [envpy, '-Im', 'pip', '--version'] + # Ignore DeprecationWarning since pip code is not part of Python + cmd = [envpy, '-W', 'ignore::DeprecationWarning', '-I', + '-m', 'pip', '--version'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() @@ -386,7 +388,8 @@ def do_test_with_pip(self, system_site_packages): # http://bugs.python.org/issue19728 # Check the private uninstall command provided for the Windows # installers works (at least in a virtual environment) - cmd = [envpy, '-Im', 'ensurepip._uninstall'] + cmd = [envpy, '-W', 'ignore::DeprecationWarning', '-I', + '-m', 'ensurepip._uninstall'] with EnvironmentVarGuard() as envvars: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e007dc7e39e030..e60bc4d27f6678 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -125,6 +125,7 @@ def test_ignore(self): self.module.filterwarnings("ignore", category=UserWarning) self.module.warn("FilterTests.test_ignore", UserWarning) self.assertEqual(len(w), 0) + self.assertEqual(list(__warningregistry__), ['version']) def test_ignore_after_default(self): with original_warnings.catch_warnings(record=True, @@ -940,11 +941,11 @@ def func(): expected = textwrap.dedent(''' {fname}:5: ResourceWarning: unclosed file <...> f = None - Object allocated at (most recent call first): - File "{fname}", lineno 3 - f = open(__file__) + Object allocated at (most recent call last): File "{fname}", lineno 7 func() + File "{fname}", lineno 3 + f = open(__file__) ''') expected = expected.format(fname=support.TESTFN).strip() self.assertEqual(stderr, expected) diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index d1da838ac715f9..ce21c3df814037 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -418,7 +418,7 @@ def g2(v = None): "Yielded g2 spam", "Yielded g2 more spam", "Finishing g2", - "g2 returned StopIteration(3,)", + "g2 returned StopIteration(3)", "Yielded g1 eggs", "Finishing g1", ]) @@ -696,15 +696,15 @@ def g(r): "g starting", "f resuming g", "g returning 1", - "f caught StopIteration(1,)", + "f caught StopIteration(1)", "g starting", "f resuming g", "g returning (2,)", - "f caught StopIteration((2,),)", + "f caught StopIteration((2,))", "g starting", "f resuming g", - "g returning StopIteration(3,)", - "f caught StopIteration(StopIteration(3,),)", + "g returning StopIteration(3)", + "f caught StopIteration(StopIteration(3))", ]) def test_send_and_return_with_value(self): @@ -741,17 +741,17 @@ def g(r): "f sending spam to g", "g received 'spam'", "g returning 1", - 'f caught StopIteration(1,)', + 'f caught StopIteration(1)', 'g starting', 'f sending spam to g', "g received 'spam'", 'g returning (2,)', - 'f caught StopIteration((2,),)', + 'f caught StopIteration((2,))', 'g starting', 'f sending spam to g', "g received 'spam'", - 'g returning StopIteration(3,)', - 'f caught StopIteration(StopIteration(3,),)' + 'g returning StopIteration(3)', + 'f caught StopIteration(StopIteration(3))' ]) def test_catching_exception_from_subgen_and_returning(self): diff --git a/Lib/tracemalloc.py b/Lib/tracemalloc.py index 597a2978afe0cd..2c1ac3b39b0784 100644 --- a/Lib/tracemalloc.py +++ b/Lib/tracemalloc.py @@ -171,16 +171,18 @@ def __repr__(self): @total_ordering class Traceback(Sequence): """ - Sequence of Frame instances sorted from the most recent frame - to the oldest frame. + Sequence of Frame instances sorted from the oldest frame + to the most recent frame. """ __slots__ = ("_frames",) def __init__(self, frames): Sequence.__init__(self) # frames is a tuple of frame tuples: see Frame constructor for the - # format of a frame tuple - self._frames = frames + # format of a frame tuple; it is reversed, because _tracemalloc + # returns frames sorted from most recent to oldest, but the + # Python API expects oldest to most recent + self._frames = tuple(reversed(frames)) def __len__(self): return len(self._frames) @@ -209,11 +211,19 @@ def __str__(self): def __repr__(self): return "" % (tuple(self),) - def format(self, limit=None): + def format(self, limit=None, most_recent_first=False): lines = [] - if limit is not None and limit < 0: - return lines - for frame in self[:limit]: + if limit is not None: + if limit > 0: + frame_slice = self[-limit:] + else: + frame_slice = self[:limit] + else: + frame_slice = self + + if most_recent_first: + frame_slice = reversed(frame_slice) + for frame in frame_slice: lines.append(' File "%s", line %s' % (frame.filename, frame.lineno)) line = linecache.getline(frame.filename, frame.lineno).strip() diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index 0a58332a666400..6daf694427d4cf 100644 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -136,7 +136,7 @@ def __init__(self, filename=None): import subprocess # Make sure we are the currently activated OS X application # so that our menu bar appears. - p = subprocess.Popen( + subprocess.run( [ 'osascript', '-e', 'tell application "System Events"', diff --git a/Lib/types.py b/Lib/types.py index 336918fea09d4a..08aabd6343db60 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -60,10 +60,32 @@ def _m(self): pass # Provide a PEP 3115 compliant mechanism for class creation def new_class(name, bases=(), kwds=None, exec_body=None): """Create a class object dynamically using the appropriate metaclass.""" - meta, ns, kwds = prepare_class(name, bases, kwds) + resolved_bases = resolve_bases(bases) + meta, ns, kwds = prepare_class(name, resolved_bases, kwds) if exec_body is not None: exec_body(ns) - return meta(name, bases, ns, **kwds) + if resolved_bases is not bases: + ns['__orig_bases__'] = bases + return meta(name, resolved_bases, ns, **kwds) + +def resolve_bases(bases): + """Resolve MRO entries dynamically as specified by PEP 560.""" + new_bases = list(bases) + updated = False + for i, base in enumerate(bases): + if isinstance(base, type): + continue + if not hasattr(base, "__mro_entries__"): + continue + new_base = base.__mro_entries__(bases) + updated = True + if not isinstance(new_base, tuple): + raise TypeError("__mro_entries__ must return a tuple") + else: + new_bases[i:i+1] = new_base + if not updated: + return bases + return tuple(new_bases) def prepare_class(name, bases=(), kwds=None): """Call the __prepare__ method of the appropriate metaclass. diff --git a/Lib/typing.py b/Lib/typing.py index c00a3a10e1fae1..b5564cc29a2d83 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -973,7 +973,8 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) - namespace.update({'__origin__': origin, '__extra__': extra}) + namespace.update({'__origin__': origin, '__extra__': extra, + '_gorg': None if not origin else origin._gorg}) self = super().__new__(cls, name, bases, namespace, _root=True) super(GenericMeta, self).__setattr__('_gorg', self if not origin else origin._gorg) @@ -1160,17 +1161,12 @@ def __instancecheck__(self, instance): # classes are supposed to be rare anyways. return issubclass(instance.__class__, self) - def __copy__(self): - return self.__class__(self.__name__, self.__bases__, - _no_slots_copy(self.__dict__), - self.__parameters__, self.__args__, self.__origin__, - self.__extra__, self.__orig_bases__) - def __setattr__(self, attr, value): # We consider all the subscripted generics as proxies for original class if ( attr.startswith('__') and attr.endswith('__') or - attr.startswith('_abc_') + attr.startswith('_abc_') or + self._gorg is None # The class is not fully created, see #typing/506 ): super(GenericMeta, self).__setattr__(attr, value) else: diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index e860debc0f3920..d936a96e73fbc7 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -8,7 +8,7 @@ import functools import warnings -from fnmatch import fnmatch +from fnmatch import fnmatch, fnmatchcase from . import case, suite, util @@ -70,6 +70,7 @@ class TestLoader(object): """ testMethodPrefix = 'test' sortTestMethodsUsing = staticmethod(util.three_way_cmp) + testNamePatterns = None suiteClass = suite.TestSuite _top_level_dir = None @@ -222,11 +223,16 @@ def loadTestsFromNames(self, names, module=None): def getTestCaseNames(self, testCaseClass): """Return a sorted sequence of method names found within testCaseClass """ - def isTestMethod(attrname, testCaseClass=testCaseClass, - prefix=self.testMethodPrefix): - return attrname.startswith(prefix) and \ - callable(getattr(testCaseClass, attrname)) - testFnNames = list(filter(isTestMethod, dir(testCaseClass))) + def shouldIncludeMethod(attrname): + if not attrname.startswith(self.testMethodPrefix): + return False + testFunc = getattr(testCaseClass, attrname) + if not callable(testFunc): + return False + fullName = '%s.%s' % (testCaseClass.__module__, testFunc.__qualname__) + return self.testNamePatterns is None or \ + any(fnmatchcase(fullName, pattern) for pattern in self.testNamePatterns) + testFnNames = list(filter(shouldIncludeMethod, dir(testCaseClass))) if self.sortTestMethodsUsing: testFnNames.sort(key=functools.cmp_to_key(self.sortTestMethodsUsing)) return testFnNames @@ -486,16 +492,17 @@ def _find_test_path(self, full_path, pattern, namespace=False): defaultTestLoader = TestLoader() -def _makeLoader(prefix, sortUsing, suiteClass=None): +def _makeLoader(prefix, sortUsing, suiteClass=None, testNamePatterns=None): loader = TestLoader() loader.sortTestMethodsUsing = sortUsing loader.testMethodPrefix = prefix + loader.testNamePatterns = testNamePatterns if suiteClass: loader.suiteClass = suiteClass return loader -def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp): - return _makeLoader(prefix, sortUsing).getTestCaseNames(testCaseClass) +def getTestCaseNames(testCaseClass, prefix, sortUsing=util.three_way_cmp, testNamePatterns=None): + return _makeLoader(prefix, sortUsing, testNamePatterns=testNamePatterns).getTestCaseNames(testCaseClass) def makeSuite(testCaseClass, prefix='test', sortUsing=util.three_way_cmp, suiteClass=suite.TestSuite): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 807604f08dfd14..e62469aa2a170f 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -46,6 +46,12 @@ def _convert_names(names): return [_convert_name(name) for name in names] +def _convert_select_pattern(pattern): + if not '*' in pattern: + pattern = '*%s*' % pattern + return pattern + + class TestProgram(object): """A command-line program that runs a set of tests; this is primarily for making test modules conveniently executable. @@ -53,7 +59,7 @@ class TestProgram(object): # defaults for testing module=None verbosity = 1 - failfast = catchbreak = buffer = progName = warnings = None + failfast = catchbreak = buffer = progName = warnings = testNamePatterns = None _discovery_parser = None def __init__(self, module='__main__', defaultTest=None, argv=None, @@ -140,8 +146,13 @@ def parseArgs(self, argv): self.testNames = list(self.defaultTest) self.createTests() - def createTests(self): - if self.testNames is None: + def createTests(self, from_discovery=False, Loader=None): + if self.testNamePatterns: + self.testLoader.testNamePatterns = self.testNamePatterns + if from_discovery: + loader = self.testLoader if Loader is None else Loader() + self.test = loader.discover(self.start, self.pattern, self.top) + elif self.testNames is None: self.test = self.testLoader.loadTestsFromModule(self.module) else: self.test = self.testLoader.loadTestsFromNames(self.testNames, @@ -179,6 +190,11 @@ def _getParentArgParser(self): action='store_true', help='Buffer stdout and stderr during tests') self.buffer = False + if self.testNamePatterns is None: + parser.add_argument('-k', dest='testNamePatterns', + action='append', type=_convert_select_pattern, + help='Only run tests which match the given substring') + self.testNamePatterns = [] return parser @@ -225,8 +241,7 @@ def _do_discovery(self, argv, Loader=None): self._initArgParsers() self._discovery_parser.parse_args(argv, self) - loader = self.testLoader if Loader is None else Loader() - self.test = loader.discover(self.start, self.pattern, self.top) + self.createTests(from_discovery=True, Loader=Loader) def runTests(self): if self.catchbreak: diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index 1131a755eaa3f2..bfd722940b5683 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -1226,6 +1226,56 @@ def test_3(self): pass names = ['test_1', 'test_2', 'test_3'] self.assertEqual(loader.getTestCaseNames(TestC), names) + # "Return a sorted sequence of method names found within testCaseClass" + # + # If TestLoader.testNamePatterns is set, only tests that match one of these + # patterns should be included. + def test_getTestCaseNames__testNamePatterns(self): + class MyTest(unittest.TestCase): + def test_1(self): pass + def test_2(self): pass + def foobar(self): pass + + loader = unittest.TestLoader() + + loader.testNamePatterns = [] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + + loader.testNamePatterns = ['*1'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1']) + + loader.testNamePatterns = ['*1', '*2'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1', 'test_2']) + + loader.testNamePatterns = ['*My*'] + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1', 'test_2']) + + loader.testNamePatterns = ['*my*'] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + + # "Return a sorted sequence of method names found within testCaseClass" + # + # If TestLoader.testNamePatterns is set, only tests that match one of these + # patterns should be included. + # + # For backwards compatibility reasons (see bpo-32071), the check may only + # touch a TestCase's attribute if it starts with the test method prefix. + def test_getTestCaseNames__testNamePatterns__attribute_access_regression(self): + class Trap: + def __get__(*ignored): + self.fail('Non-test attribute accessed') + + class MyTest(unittest.TestCase): + def test_1(self): pass + foobar = Trap() + + loader = unittest.TestLoader() + self.assertEqual(loader.getTestCaseNames(MyTest), ['test_1']) + + loader = unittest.TestLoader() + loader.testNamePatterns = [] + self.assertEqual(loader.getTestCaseNames(MyTest), []) + ################################################################ ### /Tests for TestLoader.getTestCaseNames() diff --git a/Lib/unittest/test/test_program.py b/Lib/unittest/test/test_program.py index 1cfc17959e074a..4a62ae1b11306e 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/unittest/test/test_program.py @@ -2,6 +2,7 @@ import os import sys +import subprocess from test import support import unittest import unittest.test @@ -409,6 +410,33 @@ def testParseArgsAbsolutePathsThatCannotBeConverted(self): # for invalid filenames should we raise a useful error rather than # leaving the current error message (import of filename fails) in place? + def testParseArgsSelectedTestNames(self): + program = self.program + argv = ['progname', '-k', 'foo', '-k', 'bar', '-k', '*pat*'] + + program.createTests = lambda: None + program.parseArgs(argv) + + self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) + + def testSelectedTestNamesFunctionalTest(self): + def run_unittest(args): + p = subprocess.Popen([sys.executable, '-m', 'unittest'] + args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, cwd=os.path.dirname(__file__)) + with p: + _, stderr = p.communicate() + return stderr.decode() + + t = '_test_warnings' + self.assertIn('Ran 7 tests', run_unittest([t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', 'TestWarnings', t])) + self.assertIn('Ran 7 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) + self.assertIn('Ran 2 tests', run_unittest(['-k', 'f', t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', 't', t])) + self.assertIn('Ran 3 tests', run_unittest(['-k', '*t', t])) + self.assertIn('Ran 7 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) + self.assertIn('Ran 1 test', run_unittest(['-k', '*test_warnings.*warning*', t])) + if __name__ == '__main__': unittest.main() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 9dab4c1c3a8880..daac29c68dc36d 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -16,6 +16,9 @@ __all__ = ["RobotFileParser"] +RequestRate = collections.namedtuple("RequestRate", "requests seconds") + + class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -136,11 +139,7 @@ def parse(self, lines): # check if all values are sane if (len(numbers) == 2 and numbers[0].strip().isdigit() and numbers[1].strip().isdigit()): - req_rate = collections.namedtuple('req_rate', - 'requests seconds') - entry.req_rate = req_rate - entry.req_rate.requests = int(numbers[0]) - entry.req_rate.seconds = int(numbers[1]) + entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 if state == 2: self._add_entry(entry) diff --git a/Lib/uuid.py b/Lib/uuid.py index 020c6e73c863d4..be06a6eff3f015 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -342,11 +342,30 @@ def _popen(command, *args): env=env) return proc +# For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant +# bit of the first octet signifies whether the MAC address is universally (0) +# or locally (1) administered. Network cards from hardware manufacturers will +# always be universally administered to guarantee global uniqueness of the MAC +# address, but any particular machine may have other interfaces which are +# locally administered. An example of the latter is the bridge interface to +# the Touch Bar on MacBook Pros. +# +# This bit works out to be the 42nd bit counting from 1 being the least +# significant, or 1<<41. We'll prefer universally administered MAC addresses +# over locally administered ones since the former are globally unique, but +# we'll return the first of the latter found if that's all the machine has. +# +# See https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local + +def _is_universal(mac): + return not (mac & (1 << 41)) + def _find_mac(command, args, hw_identifiers, get_index): + first_local_mac = None try: proc = _popen(command, *args.split()) if not proc: - return + return None with proc: for line in proc.stdout: words = line.lower().rstrip().split() @@ -355,8 +374,9 @@ def _find_mac(command, args, hw_identifiers, get_index): try: word = words[get_index(i)] mac = int(word.replace(b':', b''), 16) - if mac: + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): # Virtual interfaces, such as those provided by # VPNs, do not have a colon-delimited MAC address @@ -366,6 +386,7 @@ def _find_mac(command, args, hw_identifiers, get_index): pass except OSError: pass + return first_local_mac or None def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" @@ -375,6 +396,7 @@ def _ifconfig_getnode(): mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) if mac: return mac + return None def _ip_getnode(): """Get the hardware address on Unix by running ip.""" @@ -382,6 +404,7 @@ def _ip_getnode(): mac = _find_mac('ip', 'link list', [b'link/ether'], lambda i: i+1) if mac: return mac + return None def _arp_getnode(): """Get the hardware address on Unix by running arp.""" @@ -404,8 +427,10 @@ def _arp_getnode(): # This works on Linux, FreeBSD and NetBSD mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) + # Return None instead of 0. if mac: return mac + return None def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" @@ -415,32 +440,36 @@ def _lanscan_getnode(): def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" # This might work on AIX, Tru64 UNIX. + first_local_mac = None try: proc = _popen('netstat', '-ia') if not proc: - return + return None with proc: words = proc.stdout.readline().rstrip().split() try: i = words.index(b'Address') except ValueError: - return + return None for line in proc.stdout: try: words = line.rstrip().split() word = words[i] if len(word) == 17 and word.count(b':') == 5: mac = int(word.replace(b':', b''), 16) - if mac: + if _is_universal(mac): return mac + first_local_mac = first_local_mac or mac except (ValueError, IndexError): pass except OSError: pass + return first_local_mac or None def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" import os, re + first_local_mac = None dirs = ['', r'c:\windows\system32', r'c:\winnt\system32'] try: import ctypes @@ -458,18 +487,23 @@ def _ipconfig_getnode(): for line in pipe: value = line.split(':')[-1].strip().lower() if re.match('([0-9a-f][0-9a-f]-){5}[0-9a-f][0-9a-f]', value): - return int(value.replace('-', ''), 16) + mac = int(value.replace('-', ''), 16) + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + return first_local_mac or None def _netbios_getnode(): """Get the hardware address on Windows using NetBIOS calls. See http://support.microsoft.com/kb/118623 for details.""" import win32wnet, netbios + first_local_mac = None ncb = netbios.NCB() ncb.Command = netbios.NCBENUM ncb.Buffer = adapters = netbios.LANA_ENUM() adapters._pack() if win32wnet.Netbios(ncb) != 0: - return + return None adapters._unpack() for i in range(adapters.length): ncb.Reset() @@ -488,7 +522,11 @@ def _netbios_getnode(): bytes = status.adapter_address[:6] if len(bytes) != 6: continue - return int.from_bytes(bytes, 'big') + mac = int.from_bytes(bytes, 'big') + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + return first_local_mac or None _generate_time_safe = _UuidCreate = None @@ -601,9 +639,19 @@ def _windll_getnode(): return UUID(bytes=bytes_(_buffer.raw)).node def _random_getnode(): - """Get a random node ID, with eighth bit set as suggested by RFC 4122.""" + """Get a random node ID.""" + # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or + # pseudo-randomly generated value may be used; see Section 4.5. The + # multicast bit must be set in such addresses, in order that they will + # never conflict with addresses obtained from network cards." + # + # The "multicast bit" of a MAC address is defined to be "the least + # significant bit of the first octet". This works out to be the 41st bit + # counting from 1 being the least significant bit, or 1<<40. + # + # See https://en.wikipedia.org/wiki/MAC_address#Unicast_vs._multicast import random - return random.getrandbits(48) | 0x010000000000 + return random.getrandbits(48) | (1 << 40) _node = None @@ -633,6 +681,7 @@ def getnode(): continue if _node is not None: return _node + assert False, '_random_getnode() returned None' _last_timestamp = None diff --git a/Lib/warnings.py b/Lib/warnings.py index a1f774637a24f9..4e7241fe6ca450 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -62,7 +62,7 @@ def _formatwarnmsg_impl(msg): tb = None if tb is not None: - s += 'Object allocated at (most recent call first):\n' + s += 'Object allocated at (most recent call last):\n' for frame in tb: s += (' File "%s", lineno %s\n' % (frame.filename, frame.lineno)) @@ -128,7 +128,6 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - import re assert action in ("error", "ignore", "always", "default", "module", "once"), "invalid action: %r" % (action,) assert isinstance(message, str), "message must be a string" @@ -137,8 +136,20 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, assert isinstance(module, str), "module must be a string" assert isinstance(lineno, int) and lineno >= 0, \ "lineno must be an int >= 0" - _add_filter(action, re.compile(message, re.I), category, - re.compile(module), lineno, append=append) + + if message or module: + import re + + if message: + message = re.compile(message, re.I) + else: + message = None + if module: + module = re.compile(module) + else: + module = None + + _add_filter(action, message, category, module, lineno, append=append) def simplefilter(action, category=Warning, lineno=0, append=False): """Insert a simple entry into the list of warnings filters (at the front). @@ -353,7 +364,6 @@ def warn_explicit(message, category, filename, lineno, action = defaultaction # Early exit actions if action == "ignore": - registry[key] = 1 return # Prime the linecache for formatting, in case the @@ -486,7 +496,6 @@ def __exit__(self, *exc_info): # - a compiled regex that must match the module that is being warned # - a line number for the line being warning, or 0 to mean any line # If either if the compiled regexs are None, match anything. -_warnings_defaults = False try: from _warnings import (filters, _defaultaction, _onceregistry, warn, warn_explicit, _filters_mutated) @@ -504,14 +513,21 @@ def _filters_mutated(): global _filters_version _filters_version += 1 + _warnings_defaults = False + # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: - silence = [ImportWarning, PendingDeprecationWarning] - silence.append(DeprecationWarning) - for cls in silence: - simplefilter("ignore", category=cls) + dev_mode = ('dev' in getattr(sys, '_xoptions', {})) + py_debug = hasattr(sys, 'gettotalrefcount') + + if not(dev_mode or py_debug): + silence = [ImportWarning, PendingDeprecationWarning] + silence.append(DeprecationWarning) + for cls in silence: + simplefilter("ignore", category=cls) + bytes_warning = sys.flags.bytes_warning if bytes_warning > 1: bytes_action = "error" @@ -520,11 +536,17 @@ def _filters_mutated(): else: bytes_action = "ignore" simplefilter(bytes_action, category=BytesWarning, append=1) + # resource usage warnings are enabled by default in pydebug mode - if hasattr(sys, 'gettotalrefcount'): - resource_action = "always" + if dev_mode or py_debug: + resource_action = "default" else: resource_action = "ignore" simplefilter(resource_action, category=ResourceWarning, append=1) + if dev_mode: + simplefilter("default", category=Warning, append=1) + + del py_debug, dev_mode + del _warnings_defaults diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 4cb7e3214cd40a..cc6241f817a271 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -213,9 +213,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 1.0.2k", - url="https://www.openssl.org/source/openssl-1.0.2k.tar.gz", - checksum='f965fc0bf01bf882b31314b61391ae65', + name="OpenSSL 1.0.2m", + url="https://www.openssl.org/source/openssl-1.0.2m.tar.gz", + checksum='10e9e37f492094b9ef296f68f24a7666', patches=[ "openssl_sdk_makedepend.patch", ], @@ -270,9 +270,9 @@ def library_recipes(): if PYTHON_3: result.extend([ dict( - name="XZ 5.2.2", - url="http://tukaani.org/xz/xz-5.2.2.tar.gz", - checksum='7cf6a8544a7dae8e8106fdf7addfa28c', + name="XZ 5.2.3", + url="http://tukaani.org/xz/xz-5.2.3.tar.gz", + checksum='ef68674fb47a8b8e741b34e429d86e9d', configure_pre=[ '--disable-dependency-tracking', ] @@ -315,13 +315,14 @@ def library_recipes(): ), ), dict( - name="SQLite 3.14.2", - url="https://www.sqlite.org/2016/sqlite-autoconf-3140200.tar.gz", - checksum='90c53cacb811db27f990b8292bd96159', + name="SQLite 3.21.0", + url="https://www.sqlite.org/2017/sqlite-autoconf-3210000.tar.gz", + checksum='7913de4c3126ba3c24689cb7a199ea31', extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' '-DSQLITE_ENABLE_FTS3_PARENTHESIS ' + '-DSQLITE_ENABLE_JSON1 ' '-DSQLITE_ENABLE_RTREE ' '-DSQLITE_TCL=0 ' '%s' % ('','-DSQLITE_WITHOUT_ZONEMALLOC ')[LT_10_5]), @@ -844,7 +845,6 @@ def build_openssl_arch(archbase, arch): "enable-tlsext", "no-ssl2", "no-ssl3", - "no-ssl3-method", # "enable-unit-test", "shared", "--install_prefix=%s"%shellQuote(archbase), diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h index 7b74a9b24d2dd0..3fadb7d614af11 100644 --- a/Mac/PythonLauncher/FileSettings.h +++ b/Mac/PythonLauncher/FileSettings.h @@ -24,18 +24,18 @@ @interface FileSettings : NSObject { - NSString *interpreter; // The pathname of the interpreter to use - NSArray *interpreters; // List of known interpreters - BOOL honourhashbang; // #! line overrides interpreter - BOOL debug; // -d option: debug parser - BOOL verbose; // -v option: verbose import - BOOL inspect; // -i option: interactive mode after script - BOOL optimize; // -O option: optimize bytecode - BOOL nosite; // -S option: don't import site.py - BOOL tabs; // -t option: warn about inconsistent tabs - NSString *others; // other options - NSString *scriptargs; // script arguments (not for preferences) - BOOL with_terminal; // Run in terminal window + NSString *interpreter; // The pathname of the interpreter to use + NSArray *interpreters; // List of known interpreters + BOOL honourhashbang; // #! line overrides interpreter + BOOL debug; // -d option: debug parser + BOOL verbose; // -v option: verbose import + BOOL inspect; // -i option: interactive mode after script + BOOL optimize; // -O option: optimize bytecode + BOOL nosite; // -S option: don't import site.py + BOOL tabs; // -t option: warn about inconsistent tabs + NSString *others; // other options + NSString *scriptargs; // script arguments (not for preferences) + BOOL with_terminal; // Run in terminal window FileSettings *origsource; NSString *prefskey; diff --git a/Mac/PythonLauncher/MyAppDelegate.h b/Mac/PythonLauncher/MyAppDelegate.h index 097b54177c881d..7252072aa95605 100644 --- a/Mac/PythonLauncher/MyAppDelegate.h +++ b/Mac/PythonLauncher/MyAppDelegate.h @@ -4,8 +4,8 @@ @interface MyAppDelegate : NSObject { - BOOL initial_action_done; - BOOL should_terminate; + BOOL initial_action_done; + BOOL should_terminate; } - (id)init; - (IBAction)showPreferences:(id)sender; diff --git a/Makefile.pre.in b/Makefile.pre.in index 566afba2538ac0..14f6f8abc54e8a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -337,8 +337,9 @@ PYTHON_OBJS= \ Python/importdl.o \ Python/marshal.o \ Python/modsupport.o \ - Python/mystrtoul.o \ Python/mysnprintf.o \ + Python/mystrtoul.o \ + Python/pathconfig.o \ Python/peephole.o \ Python/pyarena.o \ Python/pyctype.o \ @@ -1015,7 +1016,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/ceval.h \ $(srcdir)/Include/internal/gil.h \ $(srcdir)/Include/internal/mem.h \ - $(srcdir)/Include/internal/pymalloc.h \ $(srcdir)/Include/internal/pystate.h \ $(srcdir)/Include/internal/warnings.h \ $(DTRACE_HEADERS) @@ -1606,10 +1606,9 @@ autoconf: # Create a tags file for vi tags:: - cd $(srcdir); \ - ctags -w Include/*.h Include/internal/*.h; \ - for i in $(SRCDIRS); do ctags -f tags -w -a $$i/*.[ch]; \ - done; \ + ctags -w $(srcdir)/Include/*.h $(srcdir)/Include/internal/*.h + for i in $(SRCDIRS); do ctags -f tags -w -a $(srcdir)/$$i/*.[ch]; done + ctags -f tags -w -a $(srcdir)/Modules/_ctypes/*.[ch] LC_ALL=C sort -o tags tags # Create a tags file for GNU Emacs diff --git a/Misc/ACKS b/Misc/ACKS index 05113903f06de7..54d8d62b633f70 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1031,6 +1031,7 @@ Bill van Melle Lucas Prado Melo Ezio Melotti Doug Mennella +Dimitri Merejkowsky Brian Merrell Alexis Métaireau Luke Mewburn @@ -1466,6 +1467,7 @@ Nathan Paul Simons Guilherme Simões Adam Simpkins Ravi Sinha +Mandeep Singh Janne Sinkkonen Ng Pheng Siong Yann Sionneau diff --git a/Misc/NEWS.d/3.5.2rc1.rst b/Misc/NEWS.d/3.5.2rc1.rst index c33dcb2309f636..caed84a06f78d5 100644 --- a/Misc/NEWS.d/3.5.2rc1.rst +++ b/Misc/NEWS.d/3.5.2rc1.rst @@ -420,7 +420,7 @@ patch by ingrid. .. section: Library A new version of typing.py provides several new classes and features: -@overload outside stubs, Reversible, DefaultDict, Text, ContextManager, +@overload outside stubs, DefaultDict, Text, ContextManager, Type[], NewType(), TYPE_CHECKING, and numerous bug fixes (note that some of the new features are not yet implemented in mypy or other static analyzers). Also classes for PEP 492 (Awaitable, AsyncIterable, AsyncIterator) have been diff --git a/Misc/NEWS.d/next/Build/2017-11-02-20-13-46.bpo-28791.STt3jL.rst b/Misc/NEWS.d/next/Build/2017-11-02-20-13-46.bpo-28791.STt3jL.rst new file mode 100644 index 00000000000000..ef9bc0f6cccf9d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-02-20-13-46.bpo-28791.STt3jL.rst @@ -0,0 +1 @@ +Update OS X installer to use SQLite 3.21.0. diff --git a/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst b/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst new file mode 100644 index 00000000000000..a071bd8b325048 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-18-11-19-28.bpo-32059.a0Hxgp.rst @@ -0,0 +1,2 @@ +``detect_modules()`` in ``setup.py`` now also searches the sysroot paths +when cross-compiling. diff --git a/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst b/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst new file mode 100644 index 00000000000000..60f05db8d4f043 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-16-56-24.bpo-29040.14lCSr.rst @@ -0,0 +1,2 @@ +Support building Android with Unified Headers. The first NDK release to +support Unified Headers is android-ndk-r14. diff --git a/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst b/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst new file mode 100644 index 00000000000000..0c57e33c0a5221 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-17-12-24.bpo-28762.R6uu8w.rst @@ -0,0 +1 @@ +Revert the last commit, the F_LOCK macro is defined by Android Unified Headers. diff --git a/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst b/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst new file mode 100644 index 00000000000000..db435b008f724c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-21-17-27-59.bpo-28538.DsNBS7.rst @@ -0,0 +1,2 @@ +Revert the previous changes, the if_nameindex structure is defined by +Unified Headers. diff --git a/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst b/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst new file mode 100644 index 00000000000000..d71c66415d3e00 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-11-24-21-25-43.bpo-32125.K8zWgn.rst @@ -0,0 +1,2 @@ +The ``Py_UseClassExceptionsFlag`` flag has been removed. It was deprecated +and wasn't used anymore since Python 2.0. diff --git a/Misc/NEWS.d/next/C API/2017-11-30-18-13-45.bpo-20891.wBnMdF.rst b/Misc/NEWS.d/next/C API/2017-11-30-18-13-45.bpo-20891.wBnMdF.rst new file mode 100644 index 00000000000000..e89cf1292afa8a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2017-11-30-18-13-45.bpo-20891.wBnMdF.rst @@ -0,0 +1,3 @@ +Fix PyGILState_Ensure(). When PyGILState_Ensure() is called in a non-Python +thread before PyEval_InitThreads(), only call PyEval_InitThreads() after +calling PyThreadState_New() to fix a crash. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-12-22-21-01.bpo-30399.45f1gv.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-12-22-21-01.bpo-30399.45f1gv.rst new file mode 100644 index 00000000000000..ccd1575ec949a7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-12-22-21-01.bpo-30399.45f1gv.rst @@ -0,0 +1,2 @@ +Standard repr() of BaseException with a single argument no longer contains +redundant trailing comma. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-28-22-06-03.bpo-30696.lhC3HE.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-28-22-06-03.bpo-30696.lhC3HE.rst new file mode 100644 index 00000000000000..76bc683488017d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-28-22-06-03.bpo-30696.lhC3HE.rst @@ -0,0 +1 @@ +Fix the interactive interpreter looping endlessly when no memory. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-05-16-11-07.bpo-31949.2yNC_z.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-05-16-11-07.bpo-31949.2yNC_z.rst new file mode 100644 index 00000000000000..029cb57939ff03 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-05-16-11-07.bpo-31949.2yNC_z.rst @@ -0,0 +1,9 @@ +Fixed several issues in printing tracebacks (PyTraceBack_Print()). + +* Setting sys.tracebacklimit to 0 or less now suppresses printing tracebacks. +* Setting sys.tracebacklimit to None now causes using the default limit. +* Setting sys.tracebacklimit to an integer larger than LONG_MAX now means using + the limit LONG_MAX rather than the default limit. +* Fixed integer overflows in the case of more than 2**31 traceback items on + Windows. +* Fixed output errors handling. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-12-11-44-22.bpo-28180.HQX000.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-12-11-44-22.bpo-28180.HQX000.rst new file mode 100644 index 00000000000000..edf4581c6114d8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-12-11-44-22.bpo-28180.HQX000.rst @@ -0,0 +1,4 @@ +A new internal ``_Py_SetLocaleFromEnv(category)`` helper function has been +added in order to improve the consistency of behaviour across different +``libc`` implementations (e.g. Android doesn't support setting the locale from +the environment by default). diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-13-00-37-11.bpo-32012.Kprjqe.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-13-00-37-11.bpo-32012.Kprjqe.rst new file mode 100644 index 00000000000000..776a261013478e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-13-00-37-11.bpo-32012.Kprjqe.rst @@ -0,0 +1,4 @@ +SyntaxError is now correctly raised when a generator expression without +parenthesis is passed as an argument, but followed by a trailing comma. +A generator expression always needs to be directly inside a set of parentheses +and cannot have a comma on either side. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-15-10-49-35.bpo-32023.XnCGT5.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-15-10-49-35.bpo-32023.XnCGT5.rst new file mode 100644 index 00000000000000..d4a84380521958 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-15-10-49-35.bpo-32023.XnCGT5.rst @@ -0,0 +1,3 @@ +SyntaxError is now correctly raised when a generator expression without +parenthesis is used instead of an inheritance list in a class definition. +The duplication of the parentheses can be omitted only on calls. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst new file mode 100644 index 00000000000000..21d59e0d61edac --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst @@ -0,0 +1,2 @@ +Add a new "developer mode": new "-X dev" command line option to enable debug +checks at runtime. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst new file mode 100644 index 00000000000000..d2a770b9375fa0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-24-01-13-58.bpo-32096.CQTHXJ.rst @@ -0,0 +1,4 @@ +Revert memory allocator changes in the C API: move structures back from +_PyRuntime to Objects/obmalloc.c. The memory allocators are once again initialized +statically, and so PyMem_RawMalloc() and Py_DecodeLocale() can be +called before _PyRuntime_Initialize(). diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-14-36-30.bpo-32137.Stj5nL.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-14-36-30.bpo-32137.Stj5nL.rst new file mode 100644 index 00000000000000..f8f4ab93c9e2e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-14-36-30.bpo-32137.Stj5nL.rst @@ -0,0 +1,2 @@ +The repr of deeply nested dict now raises a RecursionError instead of +crashing due to a stack overflow. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst new file mode 100644 index 00000000000000..555c94ebaee23a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst @@ -0,0 +1,3 @@ +Yield expressions are now deprecated in comprehensions and generator +expressions. They are still permitted in the definition of the outermost +iterable, as that is evaluated directly in the enclosing scope. diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-02-21-37-22.bpo-32176.Wt25-N.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-02-21-37-22.bpo-32176.Wt25-N.rst new file mode 100644 index 00000000000000..9d567112866f8f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-02-21-37-22.bpo-32176.Wt25-N.rst @@ -0,0 +1,5 @@ +co_flags.CO_NOFREE is now always set correctly by the code object +constructor based on freevars and cellvars, rather than needing to be set +correctly by the caller. This ensures it will be cleared automatically when +additional cell references are injected into a modified code object and +function. diff --git a/Misc/NEWS.d/next/Documentation/2017-11-21-10-54-16.bpo-32105.91mhWm.rst b/Misc/NEWS.d/next/Documentation/2017-11-21-10-54-16.bpo-32105.91mhWm.rst new file mode 100644 index 00000000000000..6f95b1e9da18cd --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2017-11-21-10-54-16.bpo-32105.91mhWm.rst @@ -0,0 +1 @@ +Added asyncio.BaseEventLoop.connect_accepted_socket versionaddded marker. diff --git a/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst b/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst new file mode 100644 index 00000000000000..c5ee6736a84565 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-11-21-08-26-08.bpo-32100.P43qx2.rst @@ -0,0 +1,2 @@ +IDLE: Fix old and new bugs in pathbrowser; improve tests. +Patch mostly by Cheryl Sabella. diff --git a/Misc/NEWS.d/next/IDLE/2017-11-28-21-47-15.bpo-32164.2T2Na8.rst b/Misc/NEWS.d/next/IDLE/2017-11-28-21-47-15.bpo-32164.2T2Na8.rst new file mode 100644 index 00000000000000..1db575c3c6d3e4 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-11-28-21-47-15.bpo-32164.2T2Na8.rst @@ -0,0 +1,2 @@ +Delete unused file idlelib/tabbedpages.py. Use of TabbedPageSet in +configdialog was replaced by ttk.Notebook. diff --git a/Misc/NEWS.d/next/IDLE/2017-12-04-15-04-43.bpo-32207.IzyAJo.rst b/Misc/NEWS.d/next/IDLE/2017-12-04-15-04-43.bpo-32207.IzyAJo.rst new file mode 100644 index 00000000000000..e521c9b0bbdc81 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-12-04-15-04-43.bpo-32207.IzyAJo.rst @@ -0,0 +1,6 @@ +Improve tk event exception tracebacks in IDLE. +When tk event handling is driven by IDLE's run loop, a confusing +and distracting queue.EMPTY traceback context is no longer added +to tk event exception tracebacks. The traceback is now the same +as when event handling is driven by user code. Patch based on a +suggestion by Serhiy Storchaka. diff --git a/Misc/NEWS.d/next/Library/2017-10-05-12-45-29.bpo-30349.6zKJsF.rst b/Misc/NEWS.d/next/Library/2017-10-05-12-45-29.bpo-30349.6zKJsF.rst new file mode 100644 index 00000000000000..6862e02502ab80 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-05-12-45-29.bpo-30349.6zKJsF.rst @@ -0,0 +1,3 @@ +FutureWarning is now emitted if a regular expression contains character set +constructs that will change semantically in the future (nested sets and set +operations). diff --git a/Misc/NEWS.d/next/Library/2017-10-12-19-05-54.bpo-30143.25_hU1.rst b/Misc/NEWS.d/next/Library/2017-10-12-19-05-54.bpo-30143.25_hU1.rst new file mode 100644 index 00000000000000..a1f83128542532 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-12-19-05-54.bpo-30143.25_hU1.rst @@ -0,0 +1,2 @@ +2to3 now generates a code that uses abstract collection classes from +collections.abc rather than collections. diff --git a/Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst b/Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst new file mode 100644 index 00000000000000..b1014827af6221 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-23-12-05-33.bpo-28416.Ldnw8X.rst @@ -0,0 +1,3 @@ +Instances of pickle.Pickler subclass with the persistent_id() method and +pickle.Unpickler subclass with the persistent_load() method no longer create +reference cycles. diff --git a/Misc/NEWS.d/next/Library/2017-10-24-21-10-44.bpo-31702.SfwJDI.rst b/Misc/NEWS.d/next/Library/2017-10-24-21-10-44.bpo-31702.SfwJDI.rst new file mode 100644 index 00000000000000..3505cbd78c7ca0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-10-24-21-10-44.bpo-31702.SfwJDI.rst @@ -0,0 +1,2 @@ +crypt.mksalt() now allows to specify the number of rounds for SHA-256 and +SHA-512 hashing. diff --git a/Misc/NEWS.d/next/Library/2017-11-10-16-27-26.bpo-28369.IS74nd.rst b/Misc/NEWS.d/next/Library/2017-11-10-16-27-26.bpo-28369.IS74nd.rst new file mode 100644 index 00000000000000..111c9612f6c6f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-10-16-27-26.bpo-28369.IS74nd.rst @@ -0,0 +1,4 @@ +Enhance add_reader/writer check that socket is not used by some transport. +Before, only cases when add_reader/writer were called with an int FD were +supported. Now the check is implemented correctly for all file-like +objects. diff --git a/Misc/NEWS.d/next/Library/2017-11-12-20-47-59.bpo-32011.NzVDdZ.rst b/Misc/NEWS.d/next/Library/2017-11-12-20-47-59.bpo-32011.NzVDdZ.rst new file mode 100644 index 00000000000000..e2ba502c6571bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-12-20-47-59.bpo-32011.NzVDdZ.rst @@ -0,0 +1,2 @@ +Restored support of loading marshal files with the TYPE_INT64 code. These +files can be produced in Python 2.7. diff --git a/Misc/NEWS.d/next/Library/2017-11-13-17-48-33.bpo-32015.4nqRTD.rst b/Misc/NEWS.d/next/Library/2017-11-13-17-48-33.bpo-32015.4nqRTD.rst new file mode 100644 index 00000000000000..6117e5625d7edb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-13-17-48-33.bpo-32015.4nqRTD.rst @@ -0,0 +1,2 @@ +Fixed the looping of asyncio in the case of reconnection the socket during +waiting async read/write from/to the socket. diff --git a/Misc/NEWS.d/next/Library/2017-11-15-13-44-28.bpo-32034.uHAOmu.rst b/Misc/NEWS.d/next/Library/2017-11-15-13-44-28.bpo-32034.uHAOmu.rst new file mode 100644 index 00000000000000..828e8cde571383 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-15-13-44-28.bpo-32034.uHAOmu.rst @@ -0,0 +1 @@ +Make asyncio.IncompleteReadError and LimitOverrunError pickleable. diff --git a/Misc/NEWS.d/next/Library/2017-11-15-19-04-22.bpo-32037.r8-5Nk.rst b/Misc/NEWS.d/next/Library/2017-11-15-19-04-22.bpo-32037.r8-5Nk.rst new file mode 100644 index 00000000000000..d077d0e21105dd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-15-19-04-22.bpo-32037.r8-5Nk.rst @@ -0,0 +1,4 @@ +Integers that fit in a signed 32-bit integer will be now pickled with +protocol 0 using the INT opcode. This will decrease the size of a pickle, +speed up pickling and unpickling, and make these integers be unpickled as +int instances in Python 2. diff --git a/Misc/NEWS.d/next/Library/2017-11-15-20-03-45.bpo-32025.lnIKYT.rst b/Misc/NEWS.d/next/Library/2017-11-15-20-03-45.bpo-32025.lnIKYT.rst new file mode 100644 index 00000000000000..f3fe3b5a96d134 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-15-20-03-45.bpo-32025.lnIKYT.rst @@ -0,0 +1 @@ +Add time.thread_time() and time.thread_time_ns() diff --git a/Misc/NEWS.d/next/Library/2017-11-16-02-32-41.bpo-32018.YMQ7Q2.rst b/Misc/NEWS.d/next/Library/2017-11-16-02-32-41.bpo-32018.YMQ7Q2.rst new file mode 100644 index 00000000000000..aa8a47cb382e8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-16-02-32-41.bpo-32018.YMQ7Q2.rst @@ -0,0 +1,2 @@ +inspect.signature should follow PEP 8, if the parameter has an annotation and a +default value. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2017-11-16-20-09-45.bpo-32046.9sGDtw.rst b/Misc/NEWS.d/next/Library/2017-11-16-20-09-45.bpo-32046.9sGDtw.rst new file mode 100644 index 00000000000000..a6fc3c46fc36c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-16-20-09-45.bpo-32046.9sGDtw.rst @@ -0,0 +1,2 @@ +Updates 2to3 to convert from operator.isCallable(obj) to callable(obj). +Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Library/2017-11-17-18-28-53.bpo-32066.OMQFLH.rst b/Misc/NEWS.d/next/Library/2017-11-17-18-28-53.bpo-32066.OMQFLH.rst new file mode 100644 index 00000000000000..cbe07054246364 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-17-18-28-53.bpo-32066.OMQFLH.rst @@ -0,0 +1,2 @@ +asyncio: Support pathlib.Path in create_unix_connection; sock arg should be +optional diff --git a/Misc/NEWS.d/next/Library/2017-11-18-17-09-01.bpo-32069.S0wyy4.rst b/Misc/NEWS.d/next/Library/2017-11-18-17-09-01.bpo-32069.S0wyy4.rst new file mode 100644 index 00000000000000..2d695391e201c8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-18-17-09-01.bpo-32069.S0wyy4.rst @@ -0,0 +1 @@ +Drop legacy SSL transport from asyncio, ssl.MemoryBIO is always used anyway. diff --git a/Misc/NEWS.d/next/Library/2017-11-18-21-13-52.bpo-32072.nwDV8L.rst b/Misc/NEWS.d/next/Library/2017-11-18-21-13-52.bpo-32072.nwDV8L.rst new file mode 100644 index 00000000000000..6da5bb427d4cb9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-18-21-13-52.bpo-32072.nwDV8L.rst @@ -0,0 +1,6 @@ +Fixed issues with binary plists: + +* Fixed saving bytearrays. +* Identical objects will be saved only once. +* Equal references will be load as identical objects. +* Added support for saving and loading recursive data structures. diff --git a/Misc/NEWS.d/next/Library/2017-11-20-01-01-01.bpo-25054.rOlRV6.rst b/Misc/NEWS.d/next/Library/2017-11-20-01-01-01.bpo-25054.rOlRV6.rst new file mode 100644 index 00000000000000..d30bdbeeb7d862 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-20-01-01-01.bpo-25054.rOlRV6.rst @@ -0,0 +1 @@ +Added support of splitting on a pattern that could match an empty string. diff --git a/Misc/NEWS.d/next/Library/2017-11-20-01-29-46.bpo-1647489.-ZNNkh.rst b/Misc/NEWS.d/next/Library/2017-11-20-01-29-46.bpo-1647489.-ZNNkh.rst new file mode 100644 index 00000000000000..7c741ad0762684 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-20-01-29-46.bpo-1647489.-ZNNkh.rst @@ -0,0 +1,3 @@ +Fixed searching regular expression patterns that could match an empty +string. Non-empty string can now be correctly found after matching an empty +string. diff --git a/Misc/NEWS.d/next/Library/2017-11-20-15-28-31.bpo-32088.mV-4Nu.rst b/Misc/NEWS.d/next/Library/2017-11-20-15-28-31.bpo-32088.mV-4Nu.rst new file mode 100644 index 00000000000000..fda75d7745e16c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-20-15-28-31.bpo-32088.mV-4Nu.rst @@ -0,0 +1,3 @@ +warnings: When Python is build is debug mode (``Py_DEBUG``), +:exc:`DeprecationWarning`, :exc:`PendingDeprecationWarning` and +:exc:`ImportWarning` warnings are now displayed by default. diff --git a/Misc/NEWS.d/next/Library/2017-11-21-16-05-35.bpo-27535.JLhcNz.rst b/Misc/NEWS.d/next/Library/2017-11-21-16-05-35.bpo-27535.JLhcNz.rst new file mode 100644 index 00000000000000..51bcfb7d76953b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-21-16-05-35.bpo-27535.JLhcNz.rst @@ -0,0 +1,4 @@ +The warnings module doesn't leak memory anymore in the hidden warnings +registry for the "ignore" action of warnings filters. warn_explicit() +function doesn't add the warning key to the registry anymore for the +"ignore" action. diff --git a/Misc/NEWS.d/next/Library/2017-11-22-09-44-15.bpo-32110.VJa9bo.rst b/Misc/NEWS.d/next/Library/2017-11-22-09-44-15.bpo-32110.VJa9bo.rst new file mode 100644 index 00000000000000..b57ff1acafffc0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-09-44-15.bpo-32110.VJa9bo.rst @@ -0,0 +1,3 @@ +``codecs.StreamReader.read(n)`` now returns not more than *n* +characters/bytes for non-negative *n*. This makes it compatible with +``read()`` methods of other file-like objects. diff --git a/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst b/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst new file mode 100644 index 00000000000000..9d8e4da822f377 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-12-54-46.bpo-28684.NLiDKZ.rst @@ -0,0 +1,5 @@ +The new test.support.skip_unless_bind_unix_socket() decorator is used here to +skip asyncio tests that fail because the platform lacks a functional bind() +function for unix domain sockets (as it is the case for non root users on the +recent Android versions that run now SELinux in enforcing mode). + diff --git a/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst b/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst new file mode 100644 index 00000000000000..b6153c235d0cc4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-17-21-01.bpo-10049.ttsBqb.rst @@ -0,0 +1,3 @@ +Added *nullcontext* no-op context manager to contextlib. This provides a +simpler and faster alternative to ExitStack() when handling optional context +managers. diff --git a/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst b/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst new file mode 100644 index 00000000000000..2f0f54041cbf6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-22-19-52-17.bpo-32071.4WNhUH.rst @@ -0,0 +1,2 @@ +Added the ``-k`` command-line option to ``python -m unittest`` to run only +tests that match the given pattern(s). diff --git a/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst b/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst new file mode 100644 index 00000000000000..514740b433f38d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-16-15-55.bpo-19610.Dlca2P.rst @@ -0,0 +1,4 @@ +``setup()`` now warns about invalid types for some fields. + +The ``distutils.dist.Distribution`` class now warns when ``classifiers``, +``keywords`` and ``platforms`` fields are not specified as a list or a string. diff --git a/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst b/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst new file mode 100644 index 00000000000000..d9b54259cc6ce3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-21-47-36.bpo-12382.xWT9k0.rst @@ -0,0 +1,2 @@ +:func:`msilib.OpenDatabase` now raises a better exception message when it +couldn't open or create an MSI file. Initial patch by William Tisäter. diff --git a/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst b/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst new file mode 100644 index 00000000000000..89a193c9ef5b8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-23-22-12-11.bpo-31325.8jAUxN.rst @@ -0,0 +1,5 @@ +Fix wrong usage of :func:`collections.namedtuple` in +the :meth:`RobotFileParser.parse() ` +method. + +Initial patch by Robin Wellner. diff --git a/Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst b/Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst new file mode 100644 index 00000000000000..7701c86aa47710 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-00-59-12.bpo-32121.ePbmwC.rst @@ -0,0 +1,6 @@ +Made ``tracemalloc.Traceback`` behave more like the traceback module, +sorting the frames from oldest to most recent. ``Traceback.format()`` +now accepts negative *limit*, truncating the result to the ``abs(limit)`` +oldest frames. To get the old behaviour, one can use the new +*most_recent_first* argument to ``Traceback.format()``. +(Patch by Jesse Bakker.) diff --git a/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst b/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst new file mode 100644 index 00000000000000..074036b1d31e36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-11-50-41.bpo-28334.3gGGlt.rst @@ -0,0 +1,3 @@ +Use :func:`os.path.expanduser` to find the ``~/.netrc`` file in +:class:`netrc.netrc`. If it does not exist, :exc:`FileNotFoundError` +is raised. Patch by Dimitri Merejkowsky. diff --git a/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst b/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst new file mode 100644 index 00000000000000..20d94779e4453a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-24-14-07-55.bpo-12239.Nj3A0x.rst @@ -0,0 +1,2 @@ +Make :meth:`msilib.SummaryInformation.GetProperty` return ``None`` when the +value of property is ``VT_EMPTY``. Initial patch by Mark Mc Mahon. diff --git a/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst b/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst new file mode 100644 index 00000000000000..cecc10aebb9912 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-26-17-00-52.bpo-23033.YGXRWT.rst @@ -0,0 +1,3 @@ +Wildcard is now supported in hostname when it is one and only character in +the left most segment of hostname in second argument of +:meth:`ssl.match_hostname`. Patch by Mandeep Singh. diff --git a/Misc/NEWS.d/next/Library/2017-11-26-18-48-17.bpo-32107.h2ph2K.rst b/Misc/NEWS.d/next/Library/2017-11-26-18-48-17.bpo-32107.h2ph2K.rst new file mode 100644 index 00000000000000..b26daa7b1b3bfa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-26-18-48-17.bpo-32107.h2ph2K.rst @@ -0,0 +1,5 @@ +``uuid.getnode()`` now preferentially returns universally administered MAC +addresses if available, over locally administered MAC addresses. This makes a +better guarantee for global uniqueness of UUIDs returned from +``uuid.uuid1()``. If only locally administered MAC addresses are available, +the first such one found is returned. diff --git a/Misc/NEWS.d/next/Library/2017-11-27-11-29-34.bpo-32089.6ydDYv.rst b/Misc/NEWS.d/next/Library/2017-11-27-11-29-34.bpo-32089.6ydDYv.rst new file mode 100644 index 00000000000000..02d87536e6a6f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-27-11-29-34.bpo-32089.6ydDYv.rst @@ -0,0 +1,3 @@ +warnings: In development (-X dev) and debug mode (pydebug build), use the +"default" action for ResourceWarning, rather than the "always" action, in +the default warnings filters. diff --git a/Misc/NEWS.d/next/Library/2017-11-28-15-27-10.bpo-32154.kDox7L.rst b/Misc/NEWS.d/next/Library/2017-11-28-15-27-10.bpo-32154.kDox7L.rst new file mode 100644 index 00000000000000..67c63062a69c13 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-28-15-27-10.bpo-32154.kDox7L.rst @@ -0,0 +1,5 @@ +The ``asyncio.windows_utils.socketpair()`` function has been removed: use +directly :func:`socket.socketpair` which is available on all platforms since +Python 3.5 (before, it wasn't available on Windows). +``asyncio.windows_utils.socketpair()`` was just an alias to +``socket.socketpair`` on Python 3.5 and newer. diff --git a/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst b/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst new file mode 100644 index 00000000000000..715a269a5c0a00 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-29-00-42-47.bpo-321010.-axD5l.rst @@ -0,0 +1 @@ +Add :attr:`sys.flags.dev_mode` flag diff --git a/Misc/NEWS.d/next/Library/2017-11-30-20-38-16.bpo-32186.O42bVe.rst b/Misc/NEWS.d/next/Library/2017-11-30-20-38-16.bpo-32186.O42bVe.rst new file mode 100644 index 00000000000000..ea696c6098a1d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-30-20-38-16.bpo-32186.O42bVe.rst @@ -0,0 +1,3 @@ +io.FileIO.readall() and io.FileIO.read() now release the GIL when +getting the file size. Fixed hang of all threads with inaccessible NFS +server. Patch by Nir Soffer. diff --git a/Misc/NEWS.d/next/Library/2017-12-02-16-06-00.bpo-27240.Kji34M.rst b/Misc/NEWS.d/next/Library/2017-12-02-16-06-00.bpo-27240.Kji34M.rst new file mode 100644 index 00000000000000..c933ee7d91687c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-02-16-06-00.bpo-27240.Kji34M.rst @@ -0,0 +1,3 @@ +The header folding algorithm for the new email policies has been rewritten, +which also fixes bpo-30788, bpo-31831, and bpo-32182. In particular, RFC2231 +folding is now done correctly. diff --git a/Misc/NEWS.d/next/Library/2017-12-04-15-51-57.bpo-32214.uozdNj.rst b/Misc/NEWS.d/next/Library/2017-12-04-15-51-57.bpo-32214.uozdNj.rst new file mode 100644 index 00000000000000..fd9c4d4eed83b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-04-15-51-57.bpo-32214.uozdNj.rst @@ -0,0 +1,2 @@ +PEP 557, Data Classes. Provides a decorator which adds boilerplate methods +to classes which use type annotations so specify fields. diff --git a/Misc/NEWS.d/next/Library/2017-12-05-02-03-07.bpo-28556.9Z_PsJ.rst b/Misc/NEWS.d/next/Library/2017-12-05-02-03-07.bpo-28556.9Z_PsJ.rst new file mode 100644 index 00000000000000..8f3a89549704a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-05-02-03-07.bpo-28556.9Z_PsJ.rst @@ -0,0 +1,3 @@ +Two minor fixes for ``typing`` module: allow shallow copying instances of +generic classes, improve interaction of ``__init_subclass__`` with generics. +Original PRs by Ivan Levkivskyi. diff --git a/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst b/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst new file mode 100644 index 00000000000000..b5ba9d574e35b4 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-24-18-15-12.bpo-32126.PLmNLn.rst @@ -0,0 +1,2 @@ +Skip test_get_event_loop_new_process in test.test_asyncio.test_events when +sem_open() is not functional. diff --git a/Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst b/Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst new file mode 100644 index 00000000000000..e80516d984e8a9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-25-14-53-29.bpo-28668.Y1G6pA.rst @@ -0,0 +1,3 @@ +test.support.requires_multiprocessing_queue is removed. Skip tests with +test.support.import_module('multiprocessing.synchronize') instead when the +semaphore implementation is broken or missing. diff --git a/Misc/NEWS.d/next/Tests/2017-11-26-17-11-27.bpo-32136.Y11luJ.rst b/Misc/NEWS.d/next/Tests/2017-11-26-17-11-27.bpo-32136.Y11luJ.rst new file mode 100644 index 00000000000000..b7138c4bba52b1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-26-17-11-27.bpo-32136.Y11luJ.rst @@ -0,0 +1,3 @@ +The runtime embedding tests have been split out from +``Lib/test/test_capi.py`` into a new ``Lib/test/test_embed.py`` +file. diff --git a/Misc/NEWS.d/next/Tests/2017-11-27-16-18-58.bpo-32138.QsTvf-.rst b/Misc/NEWS.d/next/Tests/2017-11-27-16-18-58.bpo-32138.QsTvf-.rst new file mode 100644 index 00000000000000..2430aa7f106b5f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-27-16-18-58.bpo-32138.QsTvf-.rst @@ -0,0 +1,2 @@ +Skip on Android test_faulthandler tests that raise SIGSEGV and remove the +test.support.requires_android_level decorator. diff --git a/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst b/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst new file mode 100644 index 00000000000000..aa2d30c3d6beb6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-11-30-12-27-10.bpo-31705.yULW7O.rst @@ -0,0 +1,3 @@ +Skip test_socket.test_sha256() on Linux kernel older than 4.5. The test +fails with ENOKEY on kernel 3.10 (on ppc64le). A fix was merged into the +kernel 4.5. diff --git a/Misc/NEWS.d/next/Tests/2017-12-04-23-19-16.bpo-31380.VlMmHW.rst b/Misc/NEWS.d/next/Tests/2017-12-04-23-19-16.bpo-31380.VlMmHW.rst new file mode 100644 index 00000000000000..2baecf5adcf7d5 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2017-12-04-23-19-16.bpo-31380.VlMmHW.rst @@ -0,0 +1 @@ +Skip test_httpservers test_undecodable_file on macOS: fails on APFS. diff --git a/Misc/NEWS.d/next/Tools-Demos/2017-11-28-21-24-41.bpo-32159.RSl4QK.rst b/Misc/NEWS.d/next/Tools-Demos/2017-11-28-21-24-41.bpo-32159.RSl4QK.rst new file mode 100644 index 00000000000000..29b3545739189c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2017-11-28-21-24-41.bpo-32159.RSl4QK.rst @@ -0,0 +1,3 @@ +Remove CVS and Subversion tools: remove svneol.py and treesync.py scripts. +CPython migrated from CVS to Subversion, to Mercurial, and then to Git. CVS +and Subversion are no longer used to develop CPython. diff --git a/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst b/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst new file mode 100644 index 00000000000000..6a6618e9692558 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2017-11-19-09-46-27.bpo-1102.NY-g1F.rst @@ -0,0 +1,4 @@ +Return ``None`` when ``View.Fetch()`` returns ``ERROR_NO_MORE_ITEMS`` +instead of raising ``MSIError``. + +Initial patch by Anthony Tuininga. diff --git a/Misc/NEWS.d/next/macOS/2017-12-04-21-57-43.bpo-31392.f8huBC.rst b/Misc/NEWS.d/next/macOS/2017-12-04-21-57-43.bpo-31392.f8huBC.rst new file mode 100644 index 00000000000000..555b3812c6a113 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2017-12-04-21-57-43.bpo-31392.f8huBC.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 1.0.2m diff --git a/Misc/README b/Misc/README index ddb8f3f975904a..8e8acbc5b3cec7 100644 --- a/Misc/README +++ b/Misc/README @@ -22,6 +22,7 @@ README.AIX Information about using Python on AIX README.coverity Information about running Coverity's Prevent on Python README.valgrind Information for Valgrind users, see valgrind-python.supp SpecialBuilds.txt Describes extra symbols you can set for debug builds -svnmap.txt Map of old SVN revs and branches to hg changeset ids +svnmap.txt Map of old SVN revs and branches to hg changeset ids, + help history-digging valgrind-python.supp Valgrind suppression file, see README.valgrind vgrindefs Python configuration for vgrind (a generic pretty printer) diff --git a/Misc/python.man b/Misc/python.man index 9f71d69dfaf260..b4110295536606 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -124,7 +124,7 @@ This terminates the option list (following options are passed as arguments to the command). .TP .B \-d -Turn on parser debugging output (for wizards only, depending on +Turn on parser debugging output (for expert only, depending on compilation options). .TP .B \-E diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 3a6ad86e87e79d..bdc372811598e5 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1336,7 +1336,7 @@ static PyObject *py_dl_open(PyObject *self, PyObject *args) handle = ctypes_dlopen(name_str, mode); Py_XDECREF(name2); if (!handle) { - char *errmsg = ctypes_dlerror(); + const char *errmsg = ctypes_dlerror(); if (!errmsg) errmsg = "dlopen() error"; PyErr_SetString(PyExc_OSError, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index ae1905b8a903e0..42f4a85e356c44 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -96,7 +96,7 @@ /* Release Number */ -char *PyCursesVersion = "2.2"; +static const char PyCursesVersion[] = "2.2"; /* Includes */ @@ -2562,7 +2562,7 @@ PyCurses_setupterm(PyObject* self, PyObject *args, PyObject* keywds) } if (!initialised_setupterm && setupterm(termstr,fd,&err) == ERR) { - char* s = "setupterm: unknown error"; + const char* s = "setupterm: unknown error"; if (err == 0) { s = "setupterm: could not find terminal"; diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b6755b851dca8e..269142cfee5242 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -683,10 +683,12 @@ _io_FileIO_readall_impl(fileio *self) Py_ssize_t bytes_read = 0; Py_ssize_t n; size_t bufsize; + int fstat_result; if (self->fd < 0) return err_closed(); + Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH #ifdef MS_WINDOWS pos = _lseeki64(self->fd, 0L, SEEK_CUR); @@ -694,8 +696,10 @@ _io_FileIO_readall_impl(fileio *self) pos = lseek(self->fd, 0L, SEEK_CUR); #endif _Py_END_SUPPRESS_IPH + fstat_result = _Py_fstat_noraise(self->fd, &status); + Py_END_ALLOW_THREADS - if (_Py_fstat_noraise(self->fd, &status) == 0) + if (fstat_result == 0) end = status.st_size; else end = (Py_off_t)-1; diff --git a/Modules/_json.c b/Modules/_json.c index 13218a6ecce587..5a9464e34fb7f3 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1650,8 +1650,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, continue; } else { - /* TODO: include repr of key */ - PyErr_SetString(PyExc_TypeError, "keys must be a string"); + PyErr_Format(PyExc_TypeError, + "keys must be str, int, float, bool or None, " + "not %.100s", key->ob_type->tp_name); goto bail; } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index b71fb9350e62b3..da915efd831595 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -360,6 +360,69 @@ _Pickle_FastCall(PyObject *func, PyObject *obj) /*************************************************************************/ +/* Retrieve and deconstruct a method for avoiding a reference cycle + (pickler -> bound method of pickler -> pickler) */ +static int +init_method_ref(PyObject *self, _Py_Identifier *name, + PyObject **method_func, PyObject **method_self) +{ + PyObject *func, *func2; + + /* *method_func and *method_self should be consistent. All refcount decrements + should be occurred after setting *method_self and *method_func. */ + func = _PyObject_GetAttrId(self, name); + if (func == NULL) { + *method_self = NULL; + Py_CLEAR(*method_func); + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return -1; + } + PyErr_Clear(); + return 0; + } + + if (PyMethod_Check(func) && PyMethod_GET_SELF(func) == self) { + /* Deconstruct a bound Python method */ + func2 = PyMethod_GET_FUNCTION(func); + Py_INCREF(func2); + *method_self = self; /* borrowed */ + Py_XSETREF(*method_func, func2); + Py_DECREF(func); + return 0; + } + else { + *method_self = NULL; + Py_XSETREF(*method_func, func); + return 0; + } +} + +/* Bind a method if it was deconstructed */ +static PyObject * +reconstruct_method(PyObject *func, PyObject *self) +{ + if (self) { + return PyMethod_New(func, self); + } + else { + Py_INCREF(func); + return func; + } +} + +static PyObject * +call_method(PyObject *func, PyObject *self, PyObject *obj) +{ + if (self) { + return PyObject_CallFunctionObjArgs(func, self, obj, NULL); + } + else { + return PyObject_CallFunctionObjArgs(func, obj, NULL); + } +} + +/*************************************************************************/ + /* Internal data type used as the unpickling stack. */ typedef struct { PyObject_VAR_HEAD @@ -552,6 +615,8 @@ typedef struct PicklerObject { objects to support self-referential objects pickling. */ PyObject *pers_func; /* persistent_id() method, can be NULL */ + PyObject *pers_func_self; /* borrowed reference to self if pers_func + is an unbound method, NULL otherwise */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ PyObject *write; /* write() method of the output stream. */ @@ -590,6 +655,8 @@ typedef struct UnpicklerObject { Py_ssize_t memo_len; /* Number of objects in the memo */ PyObject *pers_func; /* persistent_load() method, can be NULL. */ + PyObject *pers_func_self; /* borrowed reference to self if pers_func + is an unbound method, NULL otherwise */ Py_buffer buffer; char *input_buffer; @@ -1777,8 +1844,10 @@ fast_save_enter(PicklerObject *self, PyObject *obj) } } key = PyLong_FromVoidPtr(obj); - if (key == NULL) + if (key == NULL) { + self->fast_nesting = -1; return 0; + } if (PyDict_GetItemWithError(self->fast_memo, key)) { Py_DECREF(key); PyErr_Format(PyExc_ValueError, @@ -1789,6 +1858,8 @@ fast_save_enter(PicklerObject *self, PyObject *obj) return 0; } if (PyErr_Occurred()) { + Py_DECREF(key); + self->fast_nesting = -1; return 0; } if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) { @@ -1854,18 +1925,13 @@ save_long(PicklerObject *self, PyObject *obj) PyObject *repr = NULL; Py_ssize_t size; long val; + int overflow; int status = 0; - const char long_op = LONG; - - val= PyLong_AsLong(obj); - if (val == -1 && PyErr_Occurred()) { - /* out of range for int pickling */ - PyErr_Clear(); - } - else if (self->bin && - (sizeof(long) <= 4 || - (val <= 0x7fffffffL && val >= (-0x7fffffffL - 1)))) { + val= PyLong_AsLongAndOverflow(obj, &overflow); + if (!overflow && (sizeof(long) <= 4 || + (val <= 0x7fffffffL && val >= (-0x7fffffffL - 1)))) + { /* result fits in a signed 4-byte integer. Note: we can't use -0x80000000L in the above condition because some @@ -1878,31 +1944,35 @@ save_long(PicklerObject *self, PyObject *obj) char pdata[32]; Py_ssize_t len = 0; - pdata[1] = (unsigned char)(val & 0xff); - pdata[2] = (unsigned char)((val >> 8) & 0xff); - pdata[3] = (unsigned char)((val >> 16) & 0xff); - pdata[4] = (unsigned char)((val >> 24) & 0xff); - - if ((pdata[4] == 0) && (pdata[3] == 0)) { - if (pdata[2] == 0) { - pdata[0] = BININT1; - len = 2; + if (self->bin) { + pdata[1] = (unsigned char)(val & 0xff); + pdata[2] = (unsigned char)((val >> 8) & 0xff); + pdata[3] = (unsigned char)((val >> 16) & 0xff); + pdata[4] = (unsigned char)((val >> 24) & 0xff); + + if ((pdata[4] != 0) || (pdata[3] != 0)) { + pdata[0] = BININT; + len = 5; } - else { + else if (pdata[2] != 0) { pdata[0] = BININT2; len = 3; } + else { + pdata[0] = BININT1; + len = 2; + } } else { - pdata[0] = BININT; - len = 5; + sprintf(pdata, "%c%ld\n", INT, val); + len = strlen(pdata); } - if (_Pickler_Write(self, pdata, len) < 0) return -1; return 0; } + assert(!PyErr_Occurred()); if (self->proto >= 2) { /* Linear-time pickling. */ @@ -1982,6 +2052,7 @@ save_long(PicklerObject *self, PyObject *obj) goto error; } else { + const char long_op = LONG; const char *string; /* proto < 2: write the repr and newline. This is quadratic-time (in @@ -3440,7 +3511,7 @@ save_type(PicklerObject *self, PyObject *obj) } static int -save_pers(PicklerObject *self, PyObject *obj, PyObject *func) +save_pers(PicklerObject *self, PyObject *obj) { PyObject *pid = NULL; int status = 0; @@ -3448,8 +3519,7 @@ save_pers(PicklerObject *self, PyObject *obj, PyObject *func) const char persid_op = PERSID; const char binpersid_op = BINPERSID; - Py_INCREF(obj); - pid = _Pickle_FastCall(func, obj); + pid = call_method(self->pers_func, self->pers_func_self, obj); if (pid == NULL) return -1; @@ -3827,7 +3897,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) 0 if it did nothing successfully; 1 if a persistent id was saved. */ - if ((status = save_pers(self, obj, self->pers_func)) != 0) + if ((status = save_pers(self, obj)) != 0) goto done; } @@ -4242,13 +4312,10 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, self->fast_nesting = 0; self->fast_memo = NULL; - self->pers_func = _PyObject_GetAttrId((PyObject *)self, - &PyId_persistent_id); - if (self->pers_func == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - return -1; - } - PyErr_Clear(); + if (init_method_ref((PyObject *)self, &PyId_persistent_id, + &self->pers_func, &self->pers_func_self) < 0) + { + return -1; } self->dispatch_table = _PyObject_GetAttrId((PyObject *)self, @@ -4515,11 +4582,11 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj) static PyObject * Pickler_get_persid(PicklerObject *self) { - if (self->pers_func == NULL) + if (self->pers_func == NULL) { PyErr_SetString(PyExc_AttributeError, "persistent_id"); - else - Py_INCREF(self->pers_func); - return self->pers_func; + return NULL; + } + return reconstruct_method(self->pers_func, self->pers_func_self); } static int @@ -4536,6 +4603,7 @@ Pickler_set_persid(PicklerObject *self, PyObject *value) return -1; } + self->pers_func_self = NULL; Py_INCREF(value); Py_XSETREF(self->pers_func, value); @@ -5485,7 +5553,7 @@ load_stack_global(UnpicklerObject *self) static int load_persid(UnpicklerObject *self) { - PyObject *pid; + PyObject *pid, *obj; Py_ssize_t len; char *s; @@ -5505,13 +5573,12 @@ load_persid(UnpicklerObject *self) return -1; } - /* This does not leak since _Pickle_FastCall() steals the reference - to pid first. */ - pid = _Pickle_FastCall(self->pers_func, pid); - if (pid == NULL) + obj = call_method(self->pers_func, self->pers_func_self, pid); + Py_DECREF(pid); + if (obj == NULL) return -1; - PDATA_PUSH(self->stack, pid, -1); + PDATA_PUSH(self->stack, obj, -1); return 0; } else { @@ -5526,20 +5593,19 @@ load_persid(UnpicklerObject *self) static int load_binpersid(UnpicklerObject *self) { - PyObject *pid; + PyObject *pid, *obj; if (self->pers_func) { PDATA_POP(self->stack, pid); if (pid == NULL) return -1; - /* This does not leak since _Pickle_FastCall() steals the - reference to pid first. */ - pid = _Pickle_FastCall(self->pers_func, pid); - if (pid == NULL) + obj = call_method(self->pers_func, self->pers_func_self, pid); + Py_DECREF(pid); + if (obj == NULL) return -1; - PDATA_PUSH(self->stack, pid, -1); + PDATA_PUSH(self->stack, obj, -1); return 0; } else { @@ -6686,13 +6752,10 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, self->fix_imports = fix_imports; - self->pers_func = _PyObject_GetAttrId((PyObject *)self, - &PyId_persistent_load); - if (self->pers_func == NULL) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - return -1; - } - PyErr_Clear(); + if (init_method_ref((PyObject *)self, &PyId_persistent_load, + &self->pers_func, &self->pers_func_self) < 0) + { + return -1; } self->stack = (Pdata *)Pdata_New(); @@ -6979,11 +7042,11 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj) static PyObject * Unpickler_get_persload(UnpicklerObject *self) { - if (self->pers_func == NULL) + if (self->pers_func == NULL) { PyErr_SetString(PyExc_AttributeError, "persistent_load"); - else - Py_INCREF(self->pers_func); - return self->pers_func; + return NULL; + } + return reconstruct_method(self->pers_func, self->pers_func_self); } static int @@ -7001,6 +7064,7 @@ Unpickler_set_persload(UnpicklerObject *self, PyObject *value) return -1; } + self->pers_func_self = NULL; Py_INCREF(value); Py_XSETREF(self->pers_func, value); diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 57eee2d32df465..3e83fb662bba78 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -1550,7 +1550,7 @@ static PyObject * pysqlite_connection_exit(pysqlite_Connection* self, PyObject* args) { PyObject* exc_type, *exc_value, *exc_tb; - char* method_name; + const char* method_name; PyObject* result; if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) { diff --git a/Modules/_sre.c b/Modules/_sre.c index a9b6b50e84e69a..68fc523c251b38 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -446,6 +446,8 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->isbytes = isbytes; state->charsize = charsize; + state->match_all = 0; + state->must_advance = 0; state->beginning = ptr; @@ -559,14 +561,14 @@ pattern_dealloc(PatternObject* self) } LOCAL(Py_ssize_t) -sre_match(SRE_STATE* state, SRE_CODE* pattern, int match_all) +sre_match(SRE_STATE* state, SRE_CODE* pattern) { if (state->charsize == 1) - return sre_ucs1_match(state, pattern, match_all); + return sre_ucs1_match(state, pattern, 1); if (state->charsize == 2) - return sre_ucs2_match(state, pattern, match_all); + return sre_ucs2_match(state, pattern, 1); assert(state->charsize == 4); - return sre_ucs4_match(state, pattern, match_all); + return sre_ucs4_match(state, pattern, 1); } LOCAL(Py_ssize_t) @@ -606,7 +608,7 @@ _sre_SRE_Pattern_match_impl(PatternObject *self, PyObject *string, TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr)); - status = sre_match(&state, PatternObject_GetCode(self), 0); + status = sre_match(&state, PatternObject_GetCode(self)); TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); if (PyErr_Occurred()) { @@ -645,7 +647,8 @@ _sre_SRE_Pattern_fullmatch_impl(PatternObject *self, PyObject *string, TRACE(("|%p|%p|FULLMATCH\n", PatternObject_GetCode(self), state.ptr)); - status = sre_match(&state, PatternObject_GetCode(self), 1); + state.match_all = 1; + status = sre_match(&state, PatternObject_GetCode(self)); TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); if (PyErr_Occurred()) { @@ -808,11 +811,8 @@ _sre_SRE_Pattern_findall_impl(PatternObject *self, PyObject *string, if (status < 0) goto error; - if (state.ptr == state.start) - state.start = (void*) ((char*) state.ptr + state.charsize); - else - state.start = state.ptr; - + state.must_advance = (state.ptr == state.start); + state.start = state.ptr; } state_fini(&state); @@ -901,17 +901,6 @@ _sre_SRE_Pattern_split_impl(PatternObject *self, PyObject *string, void* last; assert(self->codesize != 0); - if (self->code[0] != SRE_OP_INFO || self->code[3] == 0) { - if (self->code[0] == SRE_OP_INFO && self->code[4] == 0) { - PyErr_SetString(PyExc_ValueError, - "split() requires a non-empty pattern match."); - return NULL; - } - if (PyErr_WarnEx(PyExc_FutureWarning, - "split() requires a non-empty pattern match.", - 1) < 0) - return NULL; - } if (!state_init(&state, self, string, 0, PY_SSIZE_T_MAX)) return NULL; @@ -942,14 +931,6 @@ _sre_SRE_Pattern_split_impl(PatternObject *self, PyObject *string, goto error; } - if (state.start == state.ptr) { - if (last == state.end || state.ptr == state.end) - break; - /* skip one character */ - state.start = (void*) ((char*) state.ptr + state.charsize); - continue; - } - /* get segment before this match */ item = getslice(state.isbytes, state.beginning, string, STATE_OFFSET(&state, last), @@ -974,7 +955,7 @@ _sre_SRE_Pattern_split_impl(PatternObject *self, PyObject *string, } n = n + 1; - + state.must_advance = 1; last = state.start = state.ptr; } @@ -1101,9 +1082,7 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string, if (status < 0) goto error; - } else if (i == b && i == e && n > 0) - /* ignore empty match on latest position */ - goto next; + } if (filter_is_callable) { /* pass match object through filter */ @@ -1130,16 +1109,8 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string, i = e; n = n + 1; - -next: - /* move on */ - if (state.ptr == state.end) - break; - if (state.ptr == state.start) - state.start = (void*) ((char*) state.ptr + state.charsize); - else - state.start = state.ptr; - + state.must_advance = 1; + state.start = state.ptr; } /* get segment following last match */ @@ -2450,7 +2421,7 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self) state->ptr = state->start; - status = sre_match(state, PatternObject_GetCode(self->pattern), 0); + status = sre_match(state, PatternObject_GetCode(self->pattern)); if (PyErr_Occurred()) return NULL; @@ -2459,12 +2430,10 @@ _sre_SRE_Scanner_match_impl(ScannerObject *self) if (status == 0) state->start = NULL; - else if (state->ptr != state->start) + else { + state->must_advance = (state->ptr == state->start); state->start = state->ptr; - else if (state->ptr != state->end) - state->start = (void*) ((char*) state->ptr + state->charsize); - else - state->start = NULL; + } return match; } @@ -2499,12 +2468,10 @@ _sre_SRE_Scanner_search_impl(ScannerObject *self) if (status == 0) state->start = NULL; - else if (state->ptr != state->start) + else { + state->must_advance = (state->ptr == state->start); state->start = state->ptr; - else if (state->ptr != state->end) - state->start = (void*) ((char*) state->ptr + state->charsize); - else - state->start = NULL; + } return match; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 52108091f567cd..4bb3e82d1dcfee 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2413,7 +2413,7 @@ test_with_docstring(PyObject *self) static PyObject * test_string_to_double(PyObject *self) { double result; - char *msg; + const char *msg; #define CHECK_STRING(STR, expected) \ result = PyOS_string_to_double(STR, NULL, NULL); \ @@ -4104,6 +4104,19 @@ pymem_malloc_without_gil(PyObject *self, PyObject *args) Py_RETURN_NONE; } + +static PyObject* +test_pymem_getallocatorsname(PyObject *self, PyObject *args) +{ + const char *name = _PyMem_GetAllocatorsName(); + if (name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "cannot get allocators name"); + return NULL; + } + return PyUnicode_FromString(name); +} + + static PyObject* pyobject_malloc_without_gil(PyObject *self, PyObject *args) { @@ -4624,6 +4637,7 @@ static PyMethodDef TestMethods[] = { {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS}, + {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS}, {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS}, @@ -5115,6 +5129,11 @@ PyInit__testcapi(void) PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); +#ifdef WITH_PYMALLOC + PyModule_AddObject(m, "WITH_PYMALLOC", Py_True); +#else + PyModule_AddObject(m, "WITH_PYMALLOC", Py_False); +#endif TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index aaa13da890b19f..c9171f52e385cc 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1363,7 +1363,7 @@ PyInit__thread(void) if (m == NULL) return NULL; - timeout_max = (double)PY_TIMEOUT_MAX * 1e-6; + timeout_max = (_PyTime_t)PY_TIMEOUT_MAX * 1e-6; time_max = _PyTime_AsSecondsDouble(_PyTime_MAX); timeout_max = Py_MIN(timeout_max, time_max); /* Round towards minus infinity */ diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index af2a2fa4d52f24..e07022cce2bc9c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1066,8 +1066,16 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; - if (tracemalloc_init() < 0) + if (max_nframe < 1 || max_nframe > MAX_NFRAME) { + PyErr_Format(PyExc_ValueError, + "the number of frames must be in range [1; %i]", + (int)MAX_NFRAME); return -1; + } + + if (tracemalloc_init() < 0) { + return -1; + } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ @@ -1500,7 +1508,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) /*[clinic input] _tracemalloc.start - nframe: Py_ssize_t = 1 + nframe: int = 1 / Start tracing Python memory allocations. @@ -1510,22 +1518,12 @@ trace to nframe. [clinic start generated code]*/ static PyObject * -_tracemalloc_start_impl(PyObject *module, Py_ssize_t nframe) -/*[clinic end generated code: output=0f558d2079511553 input=997841629cc441cb]*/ +_tracemalloc_start_impl(PyObject *module, int nframe) +/*[clinic end generated code: output=caae05c23c159d3c input=40d849b5b29d1933]*/ { - int nframe_int; - - if (nframe < 1 || nframe > MAX_NFRAME) { - PyErr_Format(PyExc_ValueError, - "the number of frames must be in range [1; %i]", - (int)MAX_NFRAME); + if (tracemalloc_start(nframe) < 0) { return NULL; } - nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - - if (tracemalloc_start(nframe_int) < 0) - return NULL; - Py_RETURN_NONE; } @@ -1658,87 +1656,13 @@ PyInit__tracemalloc(void) } -static int -parse_sys_xoptions(PyObject *value) -{ - PyObject *valuelong; - long nframe; - - if (value == Py_True) - return 1; - - assert(PyUnicode_Check(value)); - if (PyUnicode_GetLength(value) == 0) - return -1; - - valuelong = PyLong_FromUnicodeObject(value, 10); - if (valuelong == NULL) - return -1; - - nframe = PyLong_AsLong(valuelong); - Py_DECREF(valuelong); - if (nframe == -1 && PyErr_Occurred()) - return -1; - - if (nframe < 1 || nframe > MAX_NFRAME) - return -1; - - return Py_SAFE_DOWNCAST(nframe, long, int); -} - - int -_PyTraceMalloc_Init(void) +_PyTraceMalloc_Init(int nframe) { - char *p; - int nframe; - assert(PyGILState_Check()); - - if ((p = Py_GETENV("PYTHONTRACEMALLOC")) && *p != '\0') { - char *endptr = p; - long value; - - errno = 0; - value = strtol(p, &endptr, 10); - if (*endptr != '\0' - || value < 1 - || value > MAX_NFRAME - || errno == ERANGE) - { - Py_FatalError("PYTHONTRACEMALLOC: invalid number of frames"); - return -1; - } - - nframe = (int)value; - } - else { - PyObject *xoptions, *key, *value; - - xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return -1; - - key = PyUnicode_FromString("tracemalloc"); - if (key == NULL) - return -1; - - value = PyDict_GetItemWithError(xoptions, key); /* borrowed */ - Py_DECREF(key); - if (value == NULL) { - if (PyErr_Occurred()) - return -1; - - /* -X tracemalloc is not used */ - return 0; - } - - nframe = parse_sys_xoptions(value); - if (nframe < 0) { - Py_FatalError("-X tracemalloc=NFRAME: invalid number of frames"); - } + if (nframe == 0) { + return 0; } - return tracemalloc_start(nframe); } diff --git a/Modules/_weakref.c b/Modules/_weakref.c index f9c68d6a640f34..c1238e00d35f4a 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -138,15 +138,15 @@ weakref_functions[] = { static struct PyModuleDef weakrefmodule = { - PyModuleDef_HEAD_INIT, - "_weakref", - "Weak-reference support module.", - -1, - weakref_functions, - NULL, - NULL, - NULL, - NULL + PyModuleDef_HEAD_INIT, + "_weakref", + "Weak-reference support module.", + -1, + weakref_functions, + NULL, + NULL, + NULL, + NULL }; PyMODINIT_FUNC diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 4f778a2dea3ec3..8c3f0a1c6c6465 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1928,8 +1928,10 @@ make_array(PyTypeObject *arraytype, char typecode, PyObject *items) return NULL; new_args = PyTuple_New(2); - if (new_args == NULL) + if (new_args == NULL) { + Py_DECREF(typecode_obj); return NULL; + } Py_INCREF(items); PyTuple_SET_ITEM(new_args, 0, typecode_obj); PyTuple_SET_ITEM(new_args, 1, items); diff --git a/Modules/clinic/_tracemalloc.c.h b/Modules/clinic/_tracemalloc.c.h index df7b750155e48e..206dea748be0dd 100644 --- a/Modules/clinic/_tracemalloc.c.h +++ b/Modules/clinic/_tracemalloc.c.h @@ -87,15 +87,15 @@ PyDoc_STRVAR(_tracemalloc_start__doc__, {"start", (PyCFunction)_tracemalloc_start, METH_FASTCALL, _tracemalloc_start__doc__}, static PyObject * -_tracemalloc_start_impl(PyObject *module, Py_ssize_t nframe); +_tracemalloc_start_impl(PyObject *module, int nframe); static PyObject * _tracemalloc_start(PyObject *module, PyObject **args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nframe = 1; + int nframe = 1; - if (!_PyArg_ParseStack(args, nargs, "|n:start", + if (!_PyArg_ParseStack(args, nargs, "|i:start", &nframe)) { goto exit; } @@ -185,4 +185,4 @@ _tracemalloc_get_traced_memory(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _tracemalloc_get_traced_memory_impl(module); } -/*[clinic end generated code: output=c9a0111391b3ec45 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db4f909464c186e2 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 33edc0547e6a74..0e85cce132f5e1 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1281,47 +1281,26 @@ PyInit_faulthandler(void) return m; } -/* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable - is defined, or if sys._xoptions has a 'faulthandler' key. */ - static int -faulthandler_env_options(void) +faulthandler_init_enable(void) { - PyObject *xoptions, *key, *module, *res; - char *p; - - if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) { - /* PYTHONFAULTHANDLER environment variable is missing - or an empty string */ - int has_key; - - xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return -1; - - key = PyUnicode_FromString("faulthandler"); - if (key == NULL) - return -1; - - has_key = PyDict_Contains(xoptions, key); - Py_DECREF(key); - if (has_key <= 0) - return has_key; - } - - module = PyImport_ImportModule("faulthandler"); + PyObject *module = PyImport_ImportModule("faulthandler"); if (module == NULL) { return -1; } - res = _PyObject_CallMethodId(module, &PyId_enable, NULL); + + PyObject *res = _PyObject_CallMethodId(module, &PyId_enable, NULL); Py_DECREF(module); - if (res == NULL) + if (res == NULL) { return -1; + } Py_DECREF(res); + return 0; } -int _PyFaulthandler_Init(void) +_PyInitError +_PyFaulthandler_Init(int enable) { #ifdef HAVE_SIGALTSTACK int err; @@ -1345,14 +1324,17 @@ int _PyFaulthandler_Init(void) thread.cancel_event = PyThread_allocate_lock(); thread.running = PyThread_allocate_lock(); if (!thread.cancel_event || !thread.running) { - PyErr_SetString(PyExc_RuntimeError, - "could not allocate locks for faulthandler"); - return -1; + return _Py_INIT_ERR("failed to allocate locks for faulthandler"); } PyThread_acquire_lock(thread.cancel_event, 1); #endif - return faulthandler_env_options(); + if (enable) { + if (faulthandler_init_enable() < 0) { + return _Py_INIT_ERR("failed to enable faulthandler"); + } + } + return _Py_INIT_OK(); } void _PyFaulthandler_Fini(void) diff --git a/Modules/fpectlmodule.c b/Modules/fpectlmodule.c index 404f69269aa4b8..42ef0f6072e30c 100644 --- a/Modules/fpectlmodule.c +++ b/Modules/fpectlmodule.c @@ -125,7 +125,8 @@ static void fpe_reset(Sigfunc *handler) extern long ieee_handler(const char*, const char*, sigfpe_handler_type); #endif - char *mode="exception", *in="all", *out; + const char *mode="exception", *in="all"; + char *out; (void) nonstandard_arithmetic(); (void) ieee_flags("clearall",mode,in,&out); (void) ieee_handler("set","common",(sigfpe_handler_type)handler); diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 6e26c7a68f6037..121eb46012cdcc 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1608,7 +1608,7 @@ _PyGC_DumpShutdownStats(void) { if (!(_PyRuntime.gc.debug & DEBUG_SAVEALL) && _PyRuntime.gc.garbage != NULL && PyList_GET_SIZE(_PyRuntime.gc.garbage) > 0) { - char *message; + const char *message; if (_PyRuntime.gc.debug & DEBUG_UNCOLLECTABLE) message = "gc: %zd uncollectable objects at " \ "shutdown"; diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index b6fb53cb3d8d97..06e87bfc8ced81 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -342,7 +342,7 @@ getaddrinfo(const char*hostname, const char*servname, port = htons((u_short)atoi(servname)); } else { struct servent *sp; - char *proto; + const char *proto; proto = NULL; switch (pai->ai_socktype) { diff --git a/Modules/getpath.c b/Modules/getpath.c index dd3387a9d77f98..fc2b5442ce26b8 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -1,13 +1,14 @@ /* Return the initial module search path. */ #include "Python.h" +#include "internal/pystate.h" #include "osdefs.h" #include #include #ifdef __APPLE__ -#include +# include #endif /* Search in some common locations for the associated Python libraries. @@ -97,7 +98,7 @@ */ #ifdef __cplusplus - extern "C" { +extern "C" { #endif @@ -109,13 +110,31 @@ #define LANDMARK L"os.py" #endif -static wchar_t prefix[MAXPATHLEN+1]; -static wchar_t exec_prefix[MAXPATHLEN+1]; -static wchar_t progpath[MAXPATHLEN+1]; -static wchar_t *module_search_path = NULL; +#define DECODE_LOCALE_ERR(NAME, LEN) \ + ((LEN) == (size_t)-2) \ + ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ + : _Py_INIT_NO_MEMORY() + +typedef struct { + wchar_t *path_env; /* PATH environment variable */ + + wchar_t *pythonpath; /* PYTHONPATH define */ + wchar_t *prefix; /* PREFIX define */ + wchar_t *exec_prefix; /* EXEC_PREFIX define */ + + wchar_t *lib_python; /* "lib/pythonX.Y" */ + wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t zip_path[MAXPATHLEN+1]; /* ".../lib/pythonXY.zip" */ + + int prefix_found; /* found platform independent libraries? */ + int exec_prefix_found; /* found the platform dependent libraries? */ +} PyCalculatePath; + +static const wchar_t delimiter[2] = {DELIM, '\0'}; +static const wchar_t separator[2] = {SEP, '\0'}; -/* Get file status. Encode the path to the locale encoding. */ +/* Get file status. Encode the path to the locale encoding. */ static int _Py_wstat(const wchar_t* path, struct stat *buf) { @@ -131,6 +150,7 @@ _Py_wstat(const wchar_t* path, struct stat *buf) return err; } + static void reduce(wchar_t *dir) { @@ -140,14 +160,17 @@ reduce(wchar_t *dir) dir[i] = '\0'; } + static int isfile(wchar_t *filename) /* Is file, not directory */ { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISREG(buf.st_mode)) + } + if (!S_ISREG(buf.st_mode)) { return 0; + } return 1; } @@ -155,41 +178,50 @@ isfile(wchar_t *filename) /* Is file, not directory */ static int ismodule(wchar_t *filename) /* Is module -- check for .pyc too */ { - if (isfile(filename)) + if (isfile(filename)) { return 1; + } /* Check for the compiled version of prefix. */ if (wcslen(filename) < MAXPATHLEN) { wcscat(filename, L"c"); - if (isfile(filename)) + if (isfile(filename)) { return 1; + } } return 0; } +/* Is executable file */ static int -isxfile(wchar_t *filename) /* Is executable file */ +isxfile(wchar_t *filename) { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISREG(buf.st_mode)) + } + if (!S_ISREG(buf.st_mode)) { return 0; - if ((buf.st_mode & 0111) == 0) + } + if ((buf.st_mode & 0111) == 0) { return 0; + } return 1; } +/* Is directory */ static int -isdir(wchar_t *filename) /* Is directory */ +isdir(wchar_t *filename) { struct stat buf; - if (_Py_wstat(filename, &buf) != 0) + if (_Py_wstat(filename, &buf) != 0) { return 0; - if (!S_ISDIR(buf.st_mode)) + } + if (!S_ISDIR(buf.st_mode)) { return 0; + } return 1; } @@ -207,58 +239,67 @@ static void joinpath(wchar_t *buffer, wchar_t *stuff) { size_t n, k; - if (stuff[0] == SEP) + if (stuff[0] == SEP) { n = 0; + } else { n = wcslen(buffer); - if (n > 0 && buffer[n-1] != SEP && n < MAXPATHLEN) + if (n > 0 && buffer[n-1] != SEP && n < MAXPATHLEN) { buffer[n++] = SEP; + } } - if (n > MAXPATHLEN) + if (n > MAXPATHLEN) { Py_FatalError("buffer overflow in getpath.c's joinpath()"); + } k = wcslen(stuff); - if (n + k > MAXPATHLEN) + if (n + k > MAXPATHLEN) { k = MAXPATHLEN - n; + } wcsncpy(buffer+n, stuff, k); buffer[n+k] = '\0'; } + /* copy_absolute requires that path be allocated at least MAXPATHLEN + 1 bytes and that p be no more than MAXPATHLEN bytes. */ static void copy_absolute(wchar_t *path, wchar_t *p, size_t pathlen) { - if (p[0] == SEP) + if (p[0] == SEP) { wcscpy(path, p); + } else { if (!_Py_wgetcwd(path, pathlen)) { /* unable to get the current directory */ wcscpy(path, p); return; } - if (p[0] == '.' && p[1] == SEP) + if (p[0] == '.' && p[1] == SEP) { p += 2; + } joinpath(path, p); } } + /* absolutize() requires that path be allocated at least MAXPATHLEN+1 bytes. */ static void absolutize(wchar_t *path) { wchar_t buffer[MAXPATHLEN+1]; - if (path[0] == SEP) + if (path[0] == SEP) { return; + } copy_absolute(buffer, path, MAXPATHLEN+1); wcscpy(path, buffer); } + /* search for a prefix value in an environment file. If found, copy it to the provided buffer, which is expected to be no more than MAXPATHLEN bytes long. */ - static int find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) { @@ -272,15 +313,18 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) PyObject * decoded; int n; - if (p == NULL) + if (p == NULL) { break; + } n = strlen(p); if (p[n - 1] != '\n') { /* line has overflowed - bail */ break; } - if (p[0] == '#') /* Comment - skip */ + if (p[0] == '#') { + /* Comment - skip */ continue; + } decoded = PyUnicode_DecodeUTF8(buffer, n, "surrogateescape"); if (decoded != NULL) { Py_ssize_t k; @@ -307,92 +351,139 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) return result; } + /* search_for_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ static int -search_for_prefix(wchar_t *argv0_path, wchar_t *home, wchar_t *_prefix, - wchar_t *lib_python) +search_for_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *prefix) { size_t n; wchar_t *vpath; /* If PYTHONHOME is set, we believe it unconditionally */ - if (home) { - wchar_t *delim; - wcsncpy(prefix, home, MAXPATHLEN); + if (main_config->home) { + wcsncpy(prefix, main_config->home, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; - delim = wcschr(prefix, DELIM); - if (delim) + wchar_t *delim = wcschr(prefix, DELIM); + if (delim) { *delim = L'\0'; - joinpath(prefix, lib_python); + } + joinpath(prefix, calculate->lib_python); joinpath(prefix, LANDMARK); return 1; } /* Check to see if argv[0] is in the build directory */ - wcsncpy(prefix, argv0_path, MAXPATHLEN); + wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; joinpath(prefix, L"Modules/Setup"); if (isfile(prefix)) { /* Check VPATH to see if argv0_path is in the build directory. */ vpath = Py_DecodeLocale(VPATH, NULL); if (vpath != NULL) { - wcsncpy(prefix, argv0_path, MAXPATHLEN); + wcsncpy(prefix, calculate->argv0_path, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; joinpath(prefix, vpath); PyMem_RawFree(vpath); joinpath(prefix, L"Lib"); joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + if (ismodule(prefix)) { return -1; + } } } /* Search from argv0_path, until root is found */ - copy_absolute(prefix, argv0_path, MAXPATHLEN+1); + copy_absolute(prefix, calculate->argv0_path, MAXPATHLEN+1); do { n = wcslen(prefix); - joinpath(prefix, lib_python); + joinpath(prefix, calculate->lib_python); joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + if (ismodule(prefix)) { return 1; + } prefix[n] = L'\0'; reduce(prefix); } while (prefix[0]); /* Look at configure's PREFIX */ - wcsncpy(prefix, _prefix, MAXPATHLEN); + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); prefix[MAXPATHLEN] = L'\0'; - joinpath(prefix, lib_python); + joinpath(prefix, calculate->lib_python); joinpath(prefix, LANDMARK); - if (ismodule(prefix)) + if (ismodule(prefix)) { return 1; + } /* Fail */ return 0; } +static void +calculate_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *prefix) +{ + calculate->prefix_found = search_for_prefix(main_config, calculate, prefix); + if (!calculate->prefix_found) { + if (!Py_FrozenFlag) { + fprintf(stderr, + "Could not find platform independent libraries \n"); + } + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); + joinpath(prefix, calculate->lib_python); + } + else { + reduce(prefix); + } +} + + +static void +calculate_reduce_prefix(PyCalculatePath *calculate, wchar_t *prefix) +{ + /* Reduce prefix and exec_prefix to their essence, + * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. + * If we're loading relative to the build directory, + * return the compiled-in defaults instead. + */ + if (calculate->prefix_found > 0) { + reduce(prefix); + reduce(prefix); + /* The prefix is the root directory, but reduce() chopped + * off the "/". */ + if (!prefix[0]) { + wcscpy(prefix, separator); + } + } + else { + wcsncpy(prefix, calculate->prefix, MAXPATHLEN); + } +} + + /* search_for_exec_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ static int -search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, - wchar_t *_exec_prefix, wchar_t *lib_python) +search_for_exec_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *exec_prefix) { size_t n; /* If PYTHONHOME is set, we believe it unconditionally */ - if (home) { - wchar_t *delim; - delim = wcschr(home, DELIM); - if (delim) + if (main_config->home) { + wchar_t *delim = wcschr(main_config->home, DELIM); + if (delim) { wcsncpy(exec_prefix, delim+1, MAXPATHLEN); - else - wcsncpy(exec_prefix, home, MAXPATHLEN); + } + else { + wcsncpy(exec_prefix, main_config->home, MAXPATHLEN); + } exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, lib_python); + joinpath(exec_prefix, calculate->lib_python); joinpath(exec_prefix, L"lib-dynload"); return 1; } @@ -400,13 +491,14 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" is written by setup.py and contains the relative path to the location of shared library modules. */ - wcsncpy(exec_prefix, argv0_path, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->argv0_path, MAXPATHLEN); exec_prefix[MAXPATHLEN] = L'\0'; joinpath(exec_prefix, L"pybuilddir.txt"); if (isfile(exec_prefix)) { FILE *f = _Py_wfopen(exec_prefix, L"rb"); - if (f == NULL) + if (f == NULL) { errno = 0; + } else { char buf[MAXPATHLEN+1]; PyObject *decoded; @@ -422,7 +514,7 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, Py_DECREF(decoded); if (k >= 0) { rel_builddir_path[k] = L'\0'; - wcsncpy(exec_prefix, argv0_path, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->argv0_path, MAXPATHLEN); exec_prefix[MAXPATHLEN] = L'\0'; joinpath(exec_prefix, rel_builddir_path); return -1; @@ -432,54 +524,75 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home, } /* Search from argv0_path, until root is found */ - copy_absolute(exec_prefix, argv0_path, MAXPATHLEN+1); + copy_absolute(exec_prefix, calculate->argv0_path, MAXPATHLEN+1); do { n = wcslen(exec_prefix); - joinpath(exec_prefix, lib_python); + joinpath(exec_prefix, calculate->lib_python); joinpath(exec_prefix, L"lib-dynload"); - if (isdir(exec_prefix)) + if (isdir(exec_prefix)) { return 1; + } exec_prefix[n] = L'\0'; reduce(exec_prefix); } while (exec_prefix[0]); /* Look at configure's EXEC_PREFIX */ - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); exec_prefix[MAXPATHLEN] = L'\0'; - joinpath(exec_prefix, lib_python); + joinpath(exec_prefix, calculate->lib_python); joinpath(exec_prefix, L"lib-dynload"); - if (isdir(exec_prefix)) + if (isdir(exec_prefix)) { return 1; + } /* Fail */ return 0; } + static void -calculate_path(void) +calculate_exec_prefix(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, wchar_t *exec_prefix) { - extern wchar_t *Py_GetProgramName(void); - - static const wchar_t delimiter[2] = {DELIM, '\0'}; - static const wchar_t separator[2] = {SEP, '\0'}; - char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ - wchar_t *rtpypath = NULL; - wchar_t *home = Py_GetPythonHome(); - char *_path = getenv("PATH"); - wchar_t *path_buffer = NULL; - wchar_t *path = NULL; - wchar_t *prog = Py_GetProgramName(); - wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t zip_path[MAXPATHLEN+1]; - int pfound, efound; /* 1 if found; -1 if found build directory */ - wchar_t *buf; - size_t bufsz; - size_t prefixsz; - wchar_t *defpath; -#ifdef WITH_NEXT_FRAMEWORK - NSModule pythonModule; - const char* modPath; -#endif + calculate->exec_prefix_found = search_for_exec_prefix(main_config, + calculate, + exec_prefix); + if (!calculate->exec_prefix_found) { + if (!Py_FrozenFlag) { + fprintf(stderr, + "Could not find platform dependent libraries \n"); + } + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); + joinpath(exec_prefix, L"lib/lib-dynload"); + } + /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ +} + + +static void +calculate_reduce_exec_prefix(PyCalculatePath *calculate, wchar_t *exec_prefix) +{ + if (calculate->exec_prefix_found > 0) { + reduce(exec_prefix); + reduce(exec_prefix); + reduce(exec_prefix); + if (!exec_prefix[0]) { + wcscpy(exec_prefix, separator); + } + } + else { + wcsncpy(exec_prefix, calculate->exec_prefix, MAXPATHLEN); + } +} + + +static _PyInitError +calculate_program_full_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) +{ + wchar_t program_full_path[MAXPATHLEN+1]; + memset(program_full_path, 0, sizeof(program_full_path)); + #ifdef __APPLE__ #if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = MAXPATHLEN; @@ -488,32 +601,15 @@ calculate_path(void) #endif char execpath[MAXPATHLEN+1]; #endif - wchar_t *_pythonpath, *_prefix, *_exec_prefix; - wchar_t *lib_python; - - _pythonpath = Py_DecodeLocale(PYTHONPATH, NULL); - _prefix = Py_DecodeLocale(PREFIX, NULL); - _exec_prefix = Py_DecodeLocale(EXEC_PREFIX, NULL); - lib_python = Py_DecodeLocale("lib/python" VERSION, NULL); - - if (!_pythonpath || !_prefix || !_exec_prefix || !lib_python) { - Py_FatalError( - "Unable to decode path variables in getpath.c: " - "memory error"); - } - - if (_path) { - path_buffer = Py_DecodeLocale(_path, NULL); - path = path_buffer; - } /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no * other way to find a directory to start the search from. If * $PATH isn't exported, you lose. */ - if (wcschr(prog, SEP)) - wcsncpy(progpath, prog, MAXPATHLEN); + if (wcschr(main_config->program_name, SEP)) { + wcsncpy(program_full_path, main_config->program_name, MAXPATHLEN); + } #ifdef __APPLE__ /* On Mac OS X, if a script uses an interpreter of the form * "#!/opt/python2.3/bin/python", the kernel only passes "python" @@ -525,48 +621,71 @@ calculate_path(void) * will fail if a relative path was used. but in that case, * absolutize() should help us out below */ - else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && execpath[0] == SEP) { - size_t r = mbstowcs(progpath, execpath, MAXPATHLEN+1); - if (r == (size_t)-1 || r > MAXPATHLEN) { - /* Could not convert execpath, or it's too long. */ - progpath[0] = '\0'; + else if(0 == _NSGetExecutablePath(execpath, &nsexeclength) && + execpath[0] == SEP) + { + size_t len; + wchar_t *path = Py_DecodeLocale(execpath, &len); + if (path == NULL) { + return DECODE_LOCALE_ERR("executable path", len); } + wcsncpy(program_full_path, path, MAXPATHLEN); + PyMem_RawFree(path); } #endif /* __APPLE__ */ - else if (path) { + else if (calculate->path_env) { + wchar_t *path = calculate->path_env; while (1) { wchar_t *delim = wcschr(path, DELIM); if (delim) { size_t len = delim - path; - if (len > MAXPATHLEN) + if (len > MAXPATHLEN) { len = MAXPATHLEN; - wcsncpy(progpath, path, len); - *(progpath + len) = '\0'; + } + wcsncpy(program_full_path, path, len); + program_full_path[len] = '\0'; + } + else { + wcsncpy(program_full_path, path, MAXPATHLEN); } - else - wcsncpy(progpath, path, MAXPATHLEN); - joinpath(progpath, prog); - if (isxfile(progpath)) + joinpath(program_full_path, main_config->program_name); + if (isxfile(program_full_path)) { break; + } if (!delim) { - progpath[0] = L'\0'; + program_full_path[0] = L'\0'; break; } path = delim + 1; } } - else - progpath[0] = '\0'; - PyMem_RawFree(path_buffer); - if (progpath[0] != SEP && progpath[0] != '\0') - absolutize(progpath); - wcsncpy(argv0_path, progpath, MAXPATHLEN); - argv0_path[MAXPATHLEN] = '\0'; + else { + program_full_path[0] = '\0'; + } + if (program_full_path[0] != SEP && program_full_path[0] != '\0') { + absolutize(program_full_path); + } + + config->program_full_path = _PyMem_RawWcsdup(program_full_path); + if (config->program_full_path == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path) +{ + wcsncpy(calculate->argv0_path, program_full_path, MAXPATHLEN); + calculate->argv0_path[MAXPATHLEN] = '\0'; #ifdef WITH_NEXT_FRAMEWORK + NSModule pythonModule; + /* On Mac OS X we have a special case if we're running from a framework. ** This is because the python home should be set relative to the library, ** which is in the framework, not relative to the executable, which may @@ -574,7 +693,7 @@ calculate_path(void) */ pythonModule = NSModuleForSymbol(NSLookupAndBindSymbol("_Py_Initialize")); /* Use dylib functions to find out where the framework was loaded from */ - modPath = NSLibraryNameForModule(pythonModule); + const char* modPath = NSLibraryNameForModule(pythonModule); if (modPath != NULL) { /* We're in a framework. */ /* See if we might be in the build directory. The framework in the @@ -584,147 +703,142 @@ calculate_path(void) ** be running the interpreter in the build directory, so we use the ** build-directory-specific logic to find Lib and such. */ - wchar_t* wbuf = Py_DecodeLocale(modPath, NULL); + size_t len; + wchar_t* wbuf = Py_DecodeLocale(modPath, &len); if (wbuf == NULL) { - Py_FatalError("Cannot decode framework location"); + return DECODE_LOCALE_ERR("framework location", len); } - wcsncpy(argv0_path, wbuf, MAXPATHLEN); - reduce(argv0_path); - joinpath(argv0_path, lib_python); - joinpath(argv0_path, LANDMARK); - if (!ismodule(argv0_path)) { + wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); + reduce(calculate->argv0_path); + joinpath(calculate->argv0_path, calculate->lib_python); + joinpath(calculate->argv0_path, LANDMARK); + if (!ismodule(calculate->argv0_path)) { /* We are in the build directory so use the name of the executable - we know that the absolute path is passed */ - wcsncpy(argv0_path, progpath, MAXPATHLEN); + wcsncpy(calculate->argv0_path, program_full_path, MAXPATHLEN); } else { - /* Use the location of the library as the progpath */ - wcsncpy(argv0_path, wbuf, MAXPATHLEN); + /* Use the location of the library as the program_full_path */ + wcsncpy(calculate->argv0_path, wbuf, MAXPATHLEN); } PyMem_RawFree(wbuf); } #endif #if HAVE_READLINK - { - wchar_t tmpbuffer[MAXPATHLEN+1]; - int linklen = _Py_wreadlink(progpath, tmpbuffer, MAXPATHLEN); - while (linklen != -1) { - if (tmpbuffer[0] == SEP) - /* tmpbuffer should never be longer than MAXPATHLEN, - but extra check does not hurt */ - wcsncpy(argv0_path, tmpbuffer, MAXPATHLEN); - else { - /* Interpret relative to progpath */ - reduce(argv0_path); - joinpath(argv0_path, tmpbuffer); - } - linklen = _Py_wreadlink(argv0_path, tmpbuffer, MAXPATHLEN); + wchar_t tmpbuffer[MAXPATHLEN+1]; + int linklen = _Py_wreadlink(program_full_path, tmpbuffer, MAXPATHLEN); + while (linklen != -1) { + if (tmpbuffer[0] == SEP) { + /* tmpbuffer should never be longer than MAXPATHLEN, + but extra check does not hurt */ + wcsncpy(calculate->argv0_path, tmpbuffer, MAXPATHLEN); + } + else { + /* Interpret relative to program_full_path */ + reduce(calculate->argv0_path); + joinpath(calculate->argv0_path, tmpbuffer); } + linklen = _Py_wreadlink(calculate->argv0_path, tmpbuffer, MAXPATHLEN); } #endif /* HAVE_READLINK */ - reduce(argv0_path); + reduce(calculate->argv0_path); /* At this point, argv0_path is guaranteed to be less than - MAXPATHLEN bytes long. - */ + MAXPATHLEN bytes long. */ + return _Py_INIT_OK(); +} - /* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - */ - { - wchar_t tmpbuffer[MAXPATHLEN+1]; - wchar_t *env_cfg = L"pyvenv.cfg"; - FILE * env_file = NULL; +/* Search for an "pyvenv.cfg" environment configuration file, first in the + executable's directory and then in the parent directory. + If found, open it for use when searching for prefixes. +*/ +static void +calculate_read_pyenv(PyCalculatePath *calculate) +{ + wchar_t tmpbuffer[MAXPATHLEN+1]; + wchar_t *env_cfg = L"pyvenv.cfg"; + FILE *env_file; - wcscpy(tmpbuffer, argv0_path); + wcscpy(tmpbuffer, calculate->argv0_path); + joinpath(tmpbuffer, env_cfg); + env_file = _Py_wfopen(tmpbuffer, L"r"); + if (env_file == NULL) { + errno = 0; + + reduce(tmpbuffer); + reduce(tmpbuffer); joinpath(tmpbuffer, env_cfg); + env_file = _Py_wfopen(tmpbuffer, L"r"); if (env_file == NULL) { errno = 0; - reduce(tmpbuffer); - reduce(tmpbuffer); - joinpath(tmpbuffer, env_cfg); - env_file = _Py_wfopen(tmpbuffer, L"r"); - if (env_file == NULL) { - errno = 0; - } - } - if (env_file != NULL) { - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (find_env_config_value(env_file, L"home", tmpbuffer)) { - wcscpy(argv0_path, tmpbuffer); - } - fclose(env_file); - env_file = NULL; } } - pfound = search_for_prefix(argv0_path, home, _prefix, lib_python); - if (!pfound) { - if (!Py_FrozenFlag) - fprintf(stderr, - "Could not find platform independent libraries \n"); - wcsncpy(prefix, _prefix, MAXPATHLEN); - joinpath(prefix, lib_python); + if (env_file == NULL) { + return; } - else - reduce(prefix); - wcsncpy(zip_path, prefix, MAXPATHLEN); - zip_path[MAXPATHLEN] = L'\0'; - if (pfound > 0) { /* Use the reduced prefix returned by Py_GetPrefix() */ - reduce(zip_path); - reduce(zip_path); - } - else - wcsncpy(zip_path, _prefix, MAXPATHLEN); - joinpath(zip_path, L"lib/python00.zip"); - bufsz = wcslen(zip_path); /* Replace "00" with version */ - zip_path[bufsz - 6] = VERSION[0]; - zip_path[bufsz - 5] = VERSION[2]; - - efound = search_for_exec_prefix(argv0_path, home, - _exec_prefix, lib_python); - if (!efound) { - if (!Py_FrozenFlag) - fprintf(stderr, - "Could not find platform dependent libraries \n"); - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); - joinpath(exec_prefix, L"lib/lib-dynload"); + /* Look for a 'home' variable and set argv0_path to it, if found */ + if (find_env_config_value(env_file, L"home", tmpbuffer)) { + wcscpy(calculate->argv0_path, tmpbuffer); } - /* If we found EXEC_PREFIX do *not* reduce it! (Yet.) */ + fclose(env_file); +} - if ((!pfound || !efound) && !Py_FrozenFlag) - fprintf(stderr, - "Consider setting $PYTHONHOME to [:]\n"); - /* Calculate size of return buffer. - */ - bufsz = 0; +static void +calculate_zip_path(PyCalculatePath *calculate, const wchar_t *prefix) +{ + wcsncpy(calculate->zip_path, prefix, MAXPATHLEN); + calculate->zip_path[MAXPATHLEN] = L'\0'; + + if (calculate->prefix_found > 0) { + /* Use the reduced prefix returned by Py_GetPrefix() */ + reduce(calculate->zip_path); + reduce(calculate->zip_path); + } + else { + wcsncpy(calculate->zip_path, calculate->prefix, MAXPATHLEN); + } + joinpath(calculate->zip_path, L"lib/python00.zip"); + + /* Replace "00" with version */ + size_t bufsz = wcslen(calculate->zip_path); + calculate->zip_path[bufsz - 6] = VERSION[0]; + calculate->zip_path[bufsz - 5] = VERSION[2]; +} + - if (_rtpypath && _rtpypath[0] != '\0') { - size_t rtpypath_len; - rtpypath = Py_DecodeLocale(_rtpypath, &rtpypath_len); - if (rtpypath != NULL) - bufsz += rtpypath_len + 1; +static _PyInitError +calculate_module_search_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, + const wchar_t *prefix, const wchar_t *exec_prefix, + _PyPathConfig *config) +{ + /* Calculate size of return buffer */ + size_t bufsz = 0; + if (main_config->module_search_path_env != NULL) { + bufsz += wcslen(main_config->module_search_path_env) + 1; } - defpath = _pythonpath; - prefixsz = wcslen(prefix) + 1; + wchar_t *defpath = calculate->pythonpath; + size_t prefixsz = wcslen(prefix) + 1; while (1) { wchar_t *delim = wcschr(defpath, DELIM); - if (defpath[0] != SEP) + if (defpath[0] != SEP) { /* Paths are relative to prefix */ bufsz += prefixsz; + } - if (delim) + if (delim) { bufsz += delim - defpath + 1; + } else { bufsz += wcslen(defpath) + 1; break; @@ -732,38 +846,39 @@ calculate_path(void) defpath = delim + 1; } - bufsz += wcslen(zip_path) + 1; + bufsz += wcslen(calculate->zip_path) + 1; bufsz += wcslen(exec_prefix) + 1; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + /* Allocate the buffer */ + wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { - Py_FatalError( - "Not enough memory for dynamic PYTHONPATH"); + return _Py_INIT_NO_MEMORY(); } + buf[0] = '\0'; /* Run-time value of $PYTHONPATH goes first */ - if (rtpypath) { - wcscpy(buf, rtpypath); + if (main_config->module_search_path_env) { + wcscpy(buf, main_config->module_search_path_env); wcscat(buf, delimiter); } - else - buf[0] = '\0'; /* Next is the default zip path */ - wcscat(buf, zip_path); + wcscat(buf, calculate->zip_path); wcscat(buf, delimiter); /* Next goes merge of compile-time $PYTHONPATH with * dynamically located prefix. */ - defpath = _pythonpath; + defpath = calculate->pythonpath; while (1) { wchar_t *delim = wcschr(defpath, DELIM); if (defpath[0] != SEP) { wcscat(buf, prefix); if (prefixsz >= 2 && prefix[prefixsz - 2] != SEP && - defpath[0] != (delim ? DELIM : L'\0')) { /* not empty */ + defpath[0] != (delim ? DELIM : L'\0')) + { + /* not empty */ wcscat(buf, separator); } } @@ -772,7 +887,7 @@ calculate_path(void) size_t len = delim - defpath + 1; size_t end = wcslen(buf) + len; wcsncat(buf, defpath, len); - *(buf + end) = '\0'; + buf[end] = '\0'; } else { wcscat(buf, defpath); @@ -785,94 +900,137 @@ calculate_path(void) /* Finally, on goes the directory for dynamic-load modules */ wcscat(buf, exec_prefix); - /* And publish the results */ - module_search_path = buf; + config->module_search_path = buf; + return _Py_INIT_OK(); +} - /* Reduce prefix and exec_prefix to their essence, - * e.g. /usr/local/lib/python1.5 is reduced to /usr/local. - * If we're loading relative to the build directory, - * return the compiled-in defaults instead. - */ - if (pfound > 0) { - reduce(prefix); - reduce(prefix); - /* The prefix is the root directory, but reduce() chopped - * off the "/". */ - if (!prefix[0]) - wcscpy(prefix, separator); - } - else - wcsncpy(prefix, _prefix, MAXPATHLEN); - if (efound > 0) { - reduce(exec_prefix); - reduce(exec_prefix); - reduce(exec_prefix); - if (!exec_prefix[0]) - wcscpy(exec_prefix, separator); +static _PyInitError +calculate_init(PyCalculatePath *calculate, + const _PyMainInterpreterConfig *main_config) +{ + size_t len; + char *path = getenv("PATH"); + if (path) { + calculate->path_env = Py_DecodeLocale(path, &len); + if (!calculate->path_env) { + return DECODE_LOCALE_ERR("PATH environment variable", len); + } } - else - wcsncpy(exec_prefix, _exec_prefix, MAXPATHLEN); - PyMem_RawFree(_pythonpath); - PyMem_RawFree(_prefix); - PyMem_RawFree(_exec_prefix); - PyMem_RawFree(lib_python); - PyMem_RawFree(rtpypath); + calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); + if (!calculate->pythonpath) { + return DECODE_LOCALE_ERR("PYTHONPATH define", len); + } + calculate->prefix = Py_DecodeLocale(PREFIX, &len); + if (!calculate->prefix) { + return DECODE_LOCALE_ERR("PREFIX define", len); + } + calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len); + if (!calculate->prefix) { + return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); + } + calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len); + if (!calculate->lib_python) { + return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); + } + return _Py_INIT_OK(); } -/* External interface */ -void -Py_SetPath(const wchar_t *path) +static void +calculate_free(PyCalculatePath *calculate) { - if (module_search_path != NULL) { - PyMem_RawFree(module_search_path); - module_search_path = NULL; - } - if (path != NULL) { - extern wchar_t *Py_GetProgramName(void); - wchar_t *prog = Py_GetProgramName(); - wcsncpy(progpath, prog, MAXPATHLEN); - exec_prefix[0] = prefix[0] = L'\0'; - module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (module_search_path != NULL) - wcscpy(module_search_path, path); - } + PyMem_RawFree(calculate->pythonpath); + PyMem_RawFree(calculate->prefix); + PyMem_RawFree(calculate->exec_prefix); + PyMem_RawFree(calculate->lib_python); + PyMem_RawFree(calculate->path_env); } -wchar_t * -Py_GetPath(void) -{ - if (!module_search_path) - calculate_path(); - return module_search_path; -} -wchar_t * -Py_GetPrefix(void) +static _PyInitError +calculate_path_impl(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) { - if (!module_search_path) - calculate_path(); - return prefix; -} + _PyInitError err; -wchar_t * -Py_GetExecPrefix(void) -{ - if (!module_search_path) - calculate_path(); - return exec_prefix; + err = calculate_program_full_path(main_config, calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = calculate_argv0_path(calculate, config->program_full_path); + if (_Py_INIT_FAILED(err)) { + return err; + } + + calculate_read_pyenv(calculate); + + wchar_t prefix[MAXPATHLEN+1]; + memset(prefix, 0, sizeof(prefix)); + calculate_prefix(main_config, calculate, prefix); + + calculate_zip_path(calculate, prefix); + + wchar_t exec_prefix[MAXPATHLEN+1]; + memset(exec_prefix, 0, sizeof(exec_prefix)); + calculate_exec_prefix(main_config, calculate, exec_prefix); + + if ((!calculate->prefix_found || !calculate->exec_prefix_found) && + !Py_FrozenFlag) + { + fprintf(stderr, + "Consider setting $PYTHONHOME to [:]\n"); + } + + err = calculate_module_search_path(main_config, calculate, + prefix, exec_prefix, config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + calculate_reduce_prefix(calculate, prefix); + + config->prefix = _PyMem_RawWcsdup(prefix); + if (config->prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + calculate_reduce_exec_prefix(calculate, exec_prefix); + + config->exec_prefix = _PyMem_RawWcsdup(exec_prefix); + if (config->exec_prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); } -wchar_t * -Py_GetProgramFullPath(void) + +_PyInitError +_PyPathConfig_Calculate(_PyPathConfig *config, + const _PyMainInterpreterConfig *main_config) { - if (!module_search_path) - calculate_path(); - return progpath; -} + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + _PyInitError err = calculate_init(&calculate, main_config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = calculate_path_impl(main_config, &calculate, config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _Py_INIT_OK(); + +done: + calculate_free(&calculate); + return err; +} #ifdef __cplusplus } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 91902363649882..985915f09d5683 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -810,7 +810,7 @@ static PyObject * tee(PyObject *self, PyObject *args) { Py_ssize_t i, n=2; - PyObject *it, *iterable, *copyable, *result; + PyObject *it, *iterable, *copyable, *copyfunc, *result; _Py_IDENTIFIER(__copy__); if (!PyArg_ParseTuple(args, "O|n", &iterable, &n)) @@ -829,25 +829,43 @@ tee(PyObject *self, PyObject *args) Py_DECREF(result); return NULL; } - if (!_PyObject_HasAttrId(it, &PyId___copy__)) { + + copyfunc = _PyObject_GetAttrId(it, &PyId___copy__); + if (copyfunc != NULL) { + copyable = it; + } + else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(it); + Py_DECREF(result); + return NULL; + } + else { + PyErr_Clear(); copyable = tee_fromiterable(it); Py_DECREF(it); if (copyable == NULL) { Py_DECREF(result); return NULL; } - } else - copyable = it; - PyTuple_SET_ITEM(result, 0, copyable); - for (i=1 ; i @@ -20,9 +21,9 @@ #endif #if defined(MS_WINDOWS) -#define PYTHONHOMEHELP "\\lib" +#define PYTHONHOMEHELP "\\python{major}{minor}" #else -#define PYTHONHOMEHELP "/pythonX.X" +#define PYTHONHOMEHELP "/lib/pythonX.X" #endif #include "pygetopt.h" @@ -35,9 +36,25 @@ extern "C" { #endif +#define DECODE_LOCALE_ERR(NAME, LEN) \ + (((LEN) == -2) \ + ? _Py_INIT_USER_ERR("cannot decode " #NAME) \ + : _Py_INIT_NO_MEMORY()) + + +#define SET_DECODE_ERROR(NAME, LEN) \ + do { \ + if ((LEN) == (size_t)-2) { \ + pymain->err = _Py_INIT_USER_ERR("cannot decode " #NAME); \ + } \ + else { \ + pymain->err = _Py_INIT_NO_MEMORY(); \ + } \ + } while (0) + /* For Py_GetArgcArgv(); set by main() */ static wchar_t **orig_argv; -static int orig_argc; +static int orig_argc; /* command line options */ #define BASE_OPTS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" @@ -107,15 +124,16 @@ static const char usage_6[] = " hooks.\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" " coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n" -" locale coercion and locale compatibility warnings on stderr.\n"; +" locale coercion and locale compatibility warnings on stderr.\n" +"PYTHONDEVMODE: enable the development mode.\n"; -static int -usage(int exitcode, const wchar_t* program) +static void +pymain_usage(int error, const wchar_t* program) { - FILE *f = exitcode ? stderr : stdout; + FILE *f = error ? stderr : stdout; fprintf(f, usage_line, program); - if (exitcode) + if (error) fprintf(f, "Try `python -h' for more information.\n"); else { fputs(usage_1, f); @@ -125,50 +143,71 @@ usage(int exitcode, const wchar_t* program) fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP); fputs(usage_6, f); } - return exitcode; } -static void RunStartupFile(PyCompilerFlags *cf) + +static char* +pymain_get_env_var(const char *name) +{ + char *var = Py_GETENV(name); + if (var && var[0] != '\0') { + return var; + } + else { + return NULL; + } +} + + +static void +pymain_run_startup(PyCompilerFlags *cf) { char *startup = Py_GETENV("PYTHONSTARTUP"); - if (startup != NULL && startup[0] != '\0') { - FILE *fp = _Py_fopen(startup, "r"); - if (fp != NULL) { - (void) PyRun_SimpleFileExFlags(fp, startup, 0, cf); - PyErr_Clear(); - fclose(fp); - } else { - int save_errno; - - save_errno = errno; - PySys_WriteStderr("Could not open PYTHONSTARTUP\n"); - errno = save_errno; - PyErr_SetFromErrnoWithFilename(PyExc_OSError, - startup); - PyErr_Print(); - PyErr_Clear(); - } + if (startup == NULL || startup[0] == '\0') { + return; + } + + FILE *fp = _Py_fopen(startup, "r"); + if (fp == NULL) { + int save_errno = errno; + PySys_WriteStderr("Could not open PYTHONSTARTUP\n"); + errno = save_errno; + + PyErr_SetFromErrnoWithFilename(PyExc_OSError, + startup); + PyErr_Print(); + PyErr_Clear(); + return; } + + (void) PyRun_SimpleFileExFlags(fp, startup, 0, cf); + PyErr_Clear(); + fclose(fp); } -static void RunInteractiveHook(void) +static void +pymain_run_interactive_hook(void) { PyObject *sys, *hook, *result; sys = PyImport_ImportModule("sys"); - if (sys == NULL) + if (sys == NULL) { goto error; + } + hook = PyObject_GetAttrString(sys, "__interactivehook__"); Py_DECREF(sys); - if (hook == NULL) + if (hook == NULL) { PyErr_Clear(); - else { - result = _PyObject_CallNoArg(hook); - Py_DECREF(hook); - if (result == NULL) - goto error; - else - Py_DECREF(result); + return; + } + + result = _PyObject_CallNoArg(hook); + Py_DECREF(hook); + if (result == NULL) { + goto error; } + Py_DECREF(result); + return; error: @@ -178,7 +217,8 @@ static void RunInteractiveHook(void) } -static int RunModule(wchar_t *modname, int set_argv0) +static int +pymain_run_module(wchar_t *modname, int set_argv0) { PyObject *module, *runpy, *runmodule, *runargs, *result; runpy = PyImport_ImportModule("runpy"); @@ -228,23 +268,26 @@ static int RunModule(wchar_t *modname, int set_argv0) } static PyObject * -AsImportPathEntry(wchar_t *filename) +pymain_get_importer(wchar_t *filename) { PyObject *sys_path0 = NULL, *importer; sys_path0 = PyUnicode_FromWideChar(filename, wcslen(filename)); - if (sys_path0 == NULL) + if (sys_path0 == NULL) { goto error; + } importer = PyImport_GetImporter(sys_path0); - if (importer == NULL) + if (importer == NULL) { goto error; + } if (importer == Py_None) { Py_DECREF(sys_path0); Py_DECREF(importer); return NULL; } + Py_DECREF(importer); return sys_path0; @@ -258,49 +301,25 @@ AsImportPathEntry(wchar_t *filename) static int -RunMainFromImporter(PyObject *sys_path0) -{ - PyObject *sys_path; - int sts; - - /* Assume sys_path0 has already been checked by AsImportPathEntry, - * so put it in sys.path[0] and import __main__ */ - sys_path = PySys_GetObject("path"); - if (sys_path == NULL) { - PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); - goto error; - } - sts = PyList_Insert(sys_path, 0, sys_path0); - if (sts) { - sys_path0 = NULL; - goto error; - } - - sts = RunModule(L"__main__", 0); - return sts != 0; - -error: - Py_XDECREF(sys_path0); - PyErr_Print(); - return 1; -} - -static int -run_command(wchar_t *command, PyCompilerFlags *cf) +pymain_run_command(wchar_t *command, PyCompilerFlags *cf) { PyObject *unicode, *bytes; int ret; unicode = PyUnicode_FromWideChar(command, -1); - if (unicode == NULL) + if (unicode == NULL) { goto error; + } + bytes = PyUnicode_AsUTF8String(unicode); Py_DECREF(unicode); - if (bytes == NULL) + if (bytes == NULL) { goto error; + } + ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), cf); Py_DECREF(bytes); - return ret != 0; + return (ret != 0); error: PySys_WriteStderr("Unable to decode the command from the command line:\n"); @@ -308,11 +327,12 @@ run_command(wchar_t *command, PyCompilerFlags *cf) return 1; } + static int -run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf) +pymain_run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf) { PyObject *unicode, *bytes = NULL; - char *filename_str; + const char *filename_str; int run; /* call pending calls like signal handlers (SIGINT) */ @@ -327,15 +347,17 @@ run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf) bytes = PyUnicode_EncodeFSDefault(unicode); Py_DECREF(unicode); } - if (bytes != NULL) + if (bytes != NULL) { filename_str = PyBytes_AsString(bytes); + } else { PyErr_Clear(); filename_str = ""; } } - else + else { filename_str = ""; + } run = PyRun_AnyFileExFlags(fp, filename_str, filename != NULL, p_cf); Py_XDECREF(bytes); @@ -345,13 +367,16 @@ run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf) /* Main program */ -/*TODO: Add arg processing to PEP 432 as a new configuration setup API - */ +typedef struct { + size_t len; + wchar_t **options; +} _Py_OptList; + typedef struct { wchar_t *filename; /* Trailing arg without -c or -m */ wchar_t *command; /* -c argument */ wchar_t *module; /* -m argument */ - PyObject *warning_options; /* -W options */ + _Py_OptList warning_options; /* -W options */ PyObject *extra_options; /* -X options */ int print_help; /* -h, -? options */ int print_version; /* -V option */ @@ -368,36 +393,186 @@ typedef struct { int verbosity; /* Py_VerboseFlag */ int quiet_flag; /* Py_QuietFlag */ int skip_first_line; /* -x option */ + _Py_OptList xoptions; /* -X options */ } _Py_CommandLineDetails; -#define _Py_CommandLineDetails_INIT \ - {NULL, NULL, NULL, NULL, NULL, \ - 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0, 0, 0, 0} +/* Structure used by Py_Main() to pass data to subfunctions */ +typedef struct { + /* Exit status ("exit code") */ + int status; + PyCompilerFlags cf; + /* non-zero is stdin is a TTY or if -i option is used */ + int stdin_is_interactive; + _PyCoreConfig core_config; + _PyMainInterpreterConfig config; + _Py_CommandLineDetails cmdline; + PyObject *main_importer_path; + /* non-zero if filename, command (-c) or module (-m) is set + on the command line */ + int run_code; + /* Error message if a function failed */ + _PyInitError err; + /* PYTHONWARNINGS env var */ + _Py_OptList env_warning_options; + int argc; + wchar_t **argv; +} _PyMain; + +/* .cmdline is initialized to zeros */ +#define _PyMain_INIT \ + {.status = 0, \ + .cf = {.cf_flags = 0}, \ + .core_config = _PyCoreConfig_INIT, \ + .config = _PyMainInterpreterConfig_INIT, \ + .main_importer_path = NULL, \ + .run_code = -1, \ + .err = _Py_INIT_OK(), \ + .env_warning_options = {0, NULL}} + + +static void +pymain_optlist_clear(_Py_OptList *list) +{ + for (size_t i=0; i < list->len; i++) { + PyMem_RawFree(list->options[i]); + } + PyMem_RawFree(list->options); + list->len = 0; + list->options = NULL; +} + +static void +pymain_free_impl(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + pymain_optlist_clear(&cmdline->warning_options); + pymain_optlist_clear(&cmdline->xoptions); + PyMem_RawFree(cmdline->command); + + pymain_optlist_clear(&pymain->env_warning_options); + Py_CLEAR(pymain->main_importer_path); + + _PyMainInterpreterConfig_Clear(&pymain->config); + +#ifdef __INSURE__ + /* Insure++ is a memory analysis tool that aids in discovering + * memory leaks and other memory problems. On Python exit, the + * interned string dictionaries are flagged as being in use at exit + * (which it is). Under normal circumstances, this is fine because + * the memory will be automatically reclaimed by the system. Under + * memory debugging, it's a huge source of useless noise, so we + * trade off slower shutdown for less distraction in the memory + * reports. -baw + */ + _Py_ReleaseInternedUnicodeStrings(); +#endif /* __INSURE__ */ +} + +static void +pymain_free(_PyMain *pymain) +{ + /* Force the allocator used by pymain_parse_cmdline_envvars() */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + pymain_free_impl(pymain); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +static int +pymain_run_main_from_importer(_PyMain *pymain) +{ + PyObject *sys_path0 = pymain->main_importer_path; + PyObject *sys_path; + int sts; + + /* Assume sys_path0 has already been checked by pymain_get_importer(), + * so put it in sys.path[0] and import __main__ */ + sys_path = PySys_GetObject("path"); + if (sys_path == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path"); + goto error; + } + + sts = PyList_Insert(sys_path, 0, sys_path0); + if (sts) { + sys_path0 = NULL; + goto error; + } + + sts = pymain_run_module(L"__main__", 0); + return sts != 0; + +error: + Py_CLEAR(pymain->main_importer_path); + PyErr_Print(); + return 1; +} + + +static wchar_t* +pymain_wstrdup(_PyMain *pymain, wchar_t *str) +{ + wchar_t *str2 = _PyMem_RawWcsdup(str); + if (str2 == NULL) { + pymain->err = _Py_INIT_NO_MEMORY(); + return NULL; + } + return str2; +} + + +static int +pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str) +{ + wchar_t *str2 = pymain_wstrdup(pymain, str); + if (str2 == NULL) { + return -1; + } + + size_t size = (list->len + 1) * sizeof(list[0]); + wchar_t **options2 = (wchar_t **)PyMem_RawRealloc(list->options, size); + if (options2 == NULL) { + PyMem_RawFree(str2); + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } + options2[list->len] = str2; + list->options = options2; + list->len++; + return 0; +} + +/* Parse the command line arguments + Return 0 on success. + Return 1 if parsing failed. + Set pymain->err and return -1 on other errors. */ static int -read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) +pymain_parse_cmdline_impl(_PyMain *pymain) { - PyObject *warning_option = NULL; - wchar_t *command = NULL; - wchar_t *module = NULL; - int c; + _Py_CommandLineDetails *cmdline = &pymain->cmdline; _PyOS_ResetGetOpt(); + do { + int c = _PyOS_GetOpt(pymain->argc, pymain->argv, PROGRAM_OPTS); + if (c == EOF) { + break; + } - while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) { if (c == 'c') { - size_t len; /* -c is the last option; following arguments that look like options are left for the command to interpret. */ - - len = wcslen(_PyOS_optarg) + 1 + 1; - command = (wchar_t *)PyMem_RawMalloc(sizeof(wchar_t) * len); - if (command == NULL) - Py_FatalError( - "not enough memory to copy -c argument"); - wcscpy(command, _PyOS_optarg); + size_t len = wcslen(_PyOS_optarg) + 1 + 1; + wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); + if (command == NULL) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } + memcpy(command, _PyOS_optarg, len * sizeof(wchar_t)); command[len - 2] = '\n'; command[len - 1] = 0; cmdline->command = command; @@ -408,8 +583,7 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) /* -m is the last option; following arguments that look like options are left for the module to interpret. */ - module = _PyOS_optarg; - cmdline->module = module; + cmdline->module = _PyOS_optarg; break; } @@ -428,6 +602,7 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) break; case 'I': + pymain->core_config.ignore_environment++; cmdline->isolated++; cmdline->no_user_site_directory++; break; @@ -451,7 +626,7 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) break; case 'E': - /* Handled prior to core initialization */ + pymain->core_config.ignore_environment++; break; case 't': @@ -480,21 +655,17 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) break; case 'W': - if (cmdline->warning_options == NULL) - cmdline->warning_options = PyList_New(0); - if (cmdline->warning_options == NULL) - Py_FatalError("failure in handling of -W argument"); - warning_option = PyUnicode_FromWideChar(_PyOS_optarg, -1); - if (warning_option == NULL) - Py_FatalError("failure in handling of -W argument"); - if (PyList_Append(cmdline->warning_options, warning_option) == -1) - Py_FatalError("failure in handling of -W argument"); - Py_DECREF(warning_option); + if (pymain_optlist_append(pymain, &cmdline->warning_options, + _PyOS_optarg) < 0) { + return -1; + } break; case 'X': - /* TODO: Delay addition of X options to sys module */ - PySys_AddXOption(_PyOS_optarg); + if (pymain_optlist_append(pymain, &cmdline->xoptions, + _PyOS_optarg) < 0) { + return -1; + } break; case 'q': @@ -508,20 +679,22 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) /* This space reserved for other options */ default: - return -1; - /*NOTREACHED*/ - + /* unknown argument: parsing failed */ + return 1; } - } + } while (1); - if (command == NULL && module == NULL && _PyOS_optind < argc && - wcscmp(argv[_PyOS_optind], L"-") != 0) + if (cmdline->command == NULL && cmdline->module == NULL + && _PyOS_optind < pymain->argc + && wcscmp(pymain->argv[_PyOS_optind], L"-") != 0) { - cmdline->filename = argv[_PyOS_optind]; + cmdline->filename = pymain->argv[_PyOS_optind]; } + return 0; } + static void maybe_set_flag(int *flag, int value) { @@ -534,146 +707,132 @@ maybe_set_flag(int *flag, int value) } } + static int -apply_command_line_and_environment(_Py_CommandLineDetails *cmdline) +pymain_add_xoptions(_PyMain *pymain) { - maybe_set_flag(&Py_BytesWarningFlag, cmdline->bytes_warning); - maybe_set_flag(&Py_DebugFlag, cmdline->debug); - maybe_set_flag(&Py_InspectFlag, cmdline->inspect); - maybe_set_flag(&Py_InteractiveFlag, cmdline->interactive); - maybe_set_flag(&Py_IsolatedFlag, cmdline->isolated); - maybe_set_flag(&Py_OptimizeFlag, cmdline->optimization_level); - maybe_set_flag(&Py_DontWriteBytecodeFlag, cmdline->dont_write_bytecode); - maybe_set_flag(&Py_NoUserSiteDirectory, cmdline->no_user_site_directory); - maybe_set_flag(&Py_NoSiteFlag, cmdline->no_site_import); - maybe_set_flag(&Py_UnbufferedStdioFlag, cmdline->use_unbuffered_io); - maybe_set_flag(&Py_VerboseFlag, cmdline->verbosity); - maybe_set_flag(&Py_QuietFlag, cmdline->quiet_flag); - - /* TODO: Apply PYTHONWARNINGS & -W options to sys module here */ - /* TODO: Apply -X options to sys module here */ + _Py_OptList *options = &pymain->cmdline.xoptions; + for (size_t i=0; i < options->len; i++) { + wchar_t *option = options->options[i]; + if (_PySys_AddXOptionWithError(option) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } + } return 0; } -int -Py_Main(int argc, wchar_t **argv) -{ - int c; - int sts; - FILE *fp = stdin; - char *p; -#ifdef MS_WINDOWS - wchar_t *wp; -#endif - int stdin_is_interactive = 0; - _Py_CommandLineDetails cmdline = _Py_CommandLineDetails_INIT; - _PyCoreConfig core_config = _PyCoreConfig_INIT; - PyCompilerFlags cf; - PyObject *main_importer_path = NULL; - - cf.cf_flags = 0; - orig_argc = argc; /* For Py_GetArgcArgv() */ - orig_argv = argv; - - /* Hash randomization needed early for all string operations - (including -W and -X options). */ - _PyOS_opterr = 0; /* prevent printing the error in 1st pass */ - while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) { - if (c == 'm' || c == 'c') { - /* -c / -m is the last option: following arguments are - not interpreter options. */ - break; +static int +pymain_add_warnings_optlist(_Py_OptList *warnings) +{ + for (size_t i = 0; i < warnings->len; i++) { + PyObject *option = PyUnicode_FromWideChar(warnings->options[i], -1); + if (option == NULL) { + return -1; } - if (c == 'E' || c == 'I') { - core_config.ignore_environment++; - break; + if (_PySys_AddWarnOptionWithError(option)) { + Py_DECREF(option); + return -1; } + Py_DECREF(option); } + return 0; +} - /* Initialize the core language runtime */ - Py_IgnoreEnvironmentFlag = core_config.ignore_environment; - core_config._disable_importlib = 0; - core_config.allocator = Py_GETENV("PYTHONMALLOC"); - _Py_InitializeCore(&core_config); +static int +pymain_add_warnings_options(_PyMain *pymain) +{ + PySys_ResetWarnOptions(); - /* Reprocess the command line with the language runtime available */ - if (read_command_line(argc, argv, &cmdline)) { - return usage(2, argv[0]); + if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; } - - if (cmdline.print_help) { - return usage(0, argv[0]); + if (pymain_add_warnings_optlist(&pymain->cmdline.warning_options) < 0) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; } + return 0; +} + - if (cmdline.print_version) { - printf("Python %s\n", cmdline.print_version >= 2 ? Py_GetVersion() : PY_VERSION); +/* Get warning options from PYTHONWARNINGS environment variable. + Return 0 on success. + Set pymain->err and return -1 on error. */ +static int +pymain_warnings_envvar(_PyMain *pymain) +{ + if (Py_IgnoreEnvironmentFlag) { return 0; } - PySys_ResetWarnOptions(); - apply_command_line_and_environment(&cmdline); - #ifdef MS_WINDOWS - if (!Py_IgnoreEnvironmentFlag && (wp = _wgetenv(L"PYTHONWARNINGS")) && - *wp != L'\0') { - wchar_t *buf, *warning, *context = NULL; - - buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t)); - if (buf == NULL) - Py_FatalError( - "not enough memory to copy PYTHONWARNINGS"); - wcscpy(buf, wp); + wchar_t *wp; + + if ((wp = _wgetenv(L"PYTHONWARNINGS")) && *wp != L'\0') { + wchar_t *warning, *context = NULL; + + wchar_t *buf = pymain_wstrdup(pymain, wp); + if (buf == NULL) { + return -1; + } for (warning = wcstok_s(buf, L",", &context); warning != NULL; warning = wcstok_s(NULL, L",", &context)) { - PySys_AddWarnOption(warning); + + if (pymain_optlist_append(pymain, &pymain->env_warning_options, + warning) < 0) { + PyMem_RawFree(buf); + return -1; + } } PyMem_RawFree(buf); } #else + char *p; + if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') { char *buf, *oldloc; - PyObject *unicode; /* settle for strtok here as there's no one standard C89 wcstok */ buf = (char *)PyMem_RawMalloc(strlen(p) + 1); - if (buf == NULL) - Py_FatalError( - "not enough memory to copy PYTHONWARNINGS"); + if (buf == NULL) { + pymain->err = _Py_INIT_NO_MEMORY(); + return -1; + } strcpy(buf, p); oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL)); setlocale(LC_ALL, ""); for (p = strtok(buf, ","); p != NULL; p = strtok(NULL, ",")) { -#ifdef __APPLE__ - /* Use utf-8 on Mac OS X */ - unicode = PyUnicode_FromString(p); -#else - unicode = PyUnicode_DecodeLocale(p, "surrogateescape"); -#endif - if (unicode == NULL) { - /* ignore errors */ - PyErr_Clear(); - continue; + size_t len; + wchar_t *warning = Py_DecodeLocale(p, &len); + if (warning == NULL) { + SET_DECODE_ERROR("PYTHONWARNINGS environment variable", len); + return -1; } - PySys_AddWarnOptionUnicode(unicode); - Py_DECREF(unicode); + if (pymain_optlist_append(pymain, &pymain->env_warning_options, + warning) < 0) { + PyMem_RawFree(warning); + return -1; + } + PyMem_RawFree(warning); } setlocale(LC_ALL, oldloc); PyMem_RawFree(oldloc); PyMem_RawFree(buf); } #endif - if (cmdline.warning_options != NULL) { - Py_ssize_t i; - for (i = 0; i < PyList_GET_SIZE(cmdline.warning_options); i++) { - PySys_AddWarnOptionUnicode(PyList_GET_ITEM(cmdline.warning_options, i)); - } - Py_DECREF(cmdline.warning_options); - } + return 0; +} + - stdin_is_interactive = Py_FdIsInteractive(stdin, (char *)0); +static void +pymain_init_stdio(_PyMain *pymain) +{ + pymain->stdin_is_interactive = (isatty(fileno(stdin)) + || Py_InteractiveFlag); #if defined(MS_WINDOWS) || defined(__CYGWIN__) /* don't translate newlines (\r\n <=> \n) */ @@ -706,8 +865,27 @@ Py_Main(int argc, wchar_t **argv) #endif /* !MS_WINDOWS */ /* Leave stderr alone - it should be unbuffered anyway. */ } +} + + +/* Get the program name: use PYTHONEXECUTABLE and __PYVENV_LAUNCHER__ + environment variables on macOS if available. */ +static _PyInitError +config_get_program_name(_PyMainInterpreterConfig *config) +{ + assert(config->program_name == NULL); + + /* If Py_SetProgramName() was called, use its value */ + wchar_t *program_name = _Py_path_config.program_name; + if (program_name != NULL) { + config->program_name = _PyMem_RawWcsdup(program_name); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } #ifdef __APPLE__ + char *p; /* On MacOS X, when the Python interpreter is embedded in an application bundle, it gets executed by a bootstrapping script that does os.execve() with an argv[0] that's different from the @@ -718,213 +896,798 @@ Py_Main(int argc, wchar_t **argv) See Lib/plat-mac/bundlebuiler.py for details about the bootstrap script. */ if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') { - wchar_t* buffer; - size_t len = strlen(p) + 1; - - buffer = PyMem_RawMalloc(len * sizeof(wchar_t)); - if (buffer == NULL) { - Py_FatalError( - "not enough memory to copy PYTHONEXECUTABLE"); + size_t len; + wchar_t* program_name = Py_DecodeLocale(p, &len); + if (program_name == NULL) { + return DECODE_LOCALE_ERR("PYTHONEXECUTABLE environment " + "variable", (Py_ssize_t)len); } - - mbstowcs(buffer, p, len); - Py_SetProgramName(buffer); - /* buffer is now handed off - do not free */ - } else { + config->program_name = program_name; + } #ifdef WITH_NEXT_FRAMEWORK + else { char* pyvenv_launcher = getenv("__PYVENV_LAUNCHER__"); - if (pyvenv_launcher && *pyvenv_launcher) { /* Used by Mac/Tools/pythonw.c to forward * the argv0 of the stub executable */ - wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, NULL); - - if (wbuf == NULL) { - Py_FatalError("Cannot decode __PYVENV_LAUNCHER__"); + size_t len; + wchar_t* program_name = Py_DecodeLocale(pyvenv_launcher, &len); + if (program_name == NULL) { + return DECODE_LOCALE_ERR("__PYVENV_LAUNCHER__ environment " + "variable", (Py_ssize_t)len); } - Py_SetProgramName(wbuf); - - /* Don't free wbuf, the argument to Py_SetProgramName - * must remain valid until Py_FinalizeEx is called. - */ - } else { - Py_SetProgramName(argv[0]); + config->program_name = program_name; } -#else - Py_SetProgramName(argv[0]); -#endif } -#else - Py_SetProgramName(argv[0]); -#endif - /* Replaces previous call to Py_Initialize() - * - * TODO: Move environment queries (etc) into Py_ReadConfig - */ - { - _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; +#endif /* WITH_NEXT_FRAMEWORK */ +#endif /* __APPLE__ */ + + return _Py_INIT_OK(); +} + - /* TODO: Moar config options! */ - config.install_signal_handlers = 1; - /* TODO: Print any exceptions raised by these operations */ - if (_Py_ReadMainInterpreterConfig(&config)) - Py_FatalError("Py_Main: Py_ReadMainInterpreterConfig failed"); - if (_Py_InitializeMainInterpreter(&config)) - Py_FatalError("Py_Main: Py_InitializeMainInterpreter failed"); +/* If config_get_program_name() found no program name: use argv[0] by default. + Return 0 on success. Set pymain->err and return -1 on error. */ +static int +pymain_get_program_name(_PyMain *pymain) +{ + if (pymain->config.program_name == NULL) { + /* Use argv[0] by default */ + pymain->config.program_name = pymain_wstrdup(pymain, pymain->argv[0]); + if (pymain->config.program_name == NULL) { + return -1; + } } + return 0; +} + + +/* Initialize the main interpreter. + * + * Replaces previous call to Py_Initialize() + * + * Return 0 on success. + * Set pymain->err and return -1 on error. + */ +static int +pymain_init_main_interpreter(_PyMain *pymain) +{ + _PyInitError err; + + err = _Py_InitializeMainInterpreter(&pymain->config); + if (_Py_INIT_FAILED(err)) { + pymain->err = err; + return -1; + } + return 0; +} + +static void +pymain_header(_PyMain *pymain) +{ /* TODO: Move this to _PyRun_PrepareMain */ - if (!Py_QuietFlag && (Py_VerboseFlag || - (cmdline.command == NULL && cmdline.filename == NULL && - cmdline.module == NULL && stdin_is_interactive))) { - fprintf(stderr, "Python %s on %s\n", - Py_GetVersion(), Py_GetPlatform()); - if (!Py_NoSiteFlag) - fprintf(stderr, "%s\n", COPYRIGHT); + if (Py_QuietFlag) { + return; + } + + if (!Py_VerboseFlag && (pymain->run_code || !pymain->stdin_is_interactive)) { + return; } + fprintf(stderr, "Python %s on %s\n", Py_GetVersion(), Py_GetPlatform()); + if (!Py_NoSiteFlag) { + fprintf(stderr, "%s\n", COPYRIGHT); + } +} + + +static void +pymain_set_argv(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + /* TODO: Move this to _Py_InitializeMainInterpreter */ - if (cmdline.command != NULL) { + if (cmdline->command != NULL) { /* Backup _PyOS_optind and force sys.argv[0] = '-c' */ _PyOS_optind--; - argv[_PyOS_optind] = L"-c"; + pymain->argv[_PyOS_optind] = L"-c"; } - if (cmdline.module != NULL) { + if (cmdline->module != NULL) { /* Backup _PyOS_optind and force sys.argv[0] = '-m'*/ _PyOS_optind--; - argv[_PyOS_optind] = L"-m"; - } - - if (cmdline.filename != NULL) { - main_importer_path = AsImportPathEntry(cmdline.filename); + pymain->argv[_PyOS_optind] = L"-m"; } - if (main_importer_path != NULL) { - /* Let RunMainFromImporter adjust sys.path[0] later */ - PySys_SetArgvEx(argc-_PyOS_optind, argv+_PyOS_optind, 0); + int update_path; + if (pymain->main_importer_path != NULL) { + /* Let pymain_run_main_from_importer() adjust sys.path[0] later */ + update_path = 0; } else { /* Use config settings to decide whether or not to update sys.path[0] */ - PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind); + update_path = (Py_IsolatedFlag == 0); } + PySys_SetArgvEx(pymain->argc - _PyOS_optind, + pymain->argv + _PyOS_optind, + update_path); +} - if ((Py_InspectFlag || (cmdline.command == NULL && - cmdline.filename == NULL && - cmdline.module == NULL)) && - isatty(fileno(stdin)) && - !Py_IsolatedFlag) { - PyObject *v; - v = PyImport_ImportModule("readline"); - if (v == NULL) - PyErr_Clear(); + +/* Set Py_XXX global configuration variables */ +static void +pymain_set_global_config(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + maybe_set_flag(&Py_BytesWarningFlag, cmdline->bytes_warning); + maybe_set_flag(&Py_DebugFlag, cmdline->debug); + maybe_set_flag(&Py_InspectFlag, cmdline->inspect); + maybe_set_flag(&Py_InteractiveFlag, cmdline->interactive); + maybe_set_flag(&Py_IsolatedFlag, cmdline->isolated); + maybe_set_flag(&Py_OptimizeFlag, cmdline->optimization_level); + maybe_set_flag(&Py_DontWriteBytecodeFlag, cmdline->dont_write_bytecode); + maybe_set_flag(&Py_NoUserSiteDirectory, cmdline->no_user_site_directory); + maybe_set_flag(&Py_NoSiteFlag, cmdline->no_site_import); + maybe_set_flag(&Py_UnbufferedStdioFlag, cmdline->use_unbuffered_io); + maybe_set_flag(&Py_VerboseFlag, cmdline->verbosity); + maybe_set_flag(&Py_QuietFlag, cmdline->quiet_flag); + + maybe_set_flag(&Py_IgnoreEnvironmentFlag, pymain->core_config.ignore_environment); +} + + +static void +pymain_import_readline(_PyMain *pymain) +{ + if (Py_IsolatedFlag) { + return; + } + if (!Py_InspectFlag && pymain->run_code) { + return; + } + if (!isatty(fileno(stdin))) { + return; + } + + PyObject *mod = PyImport_ImportModule("readline"); + if (mod == NULL) { + PyErr_Clear(); + } + else { + Py_DECREF(mod); + } +} + + +static FILE* +pymain_open_filename(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + FILE* fp; + + fp = _Py_wfopen(cmdline->filename, L"r"); + if (fp == NULL) { + char *cfilename_buffer; + const char *cfilename; + int err = errno; + cfilename_buffer = Py_EncodeLocale(cmdline->filename, NULL); + if (cfilename_buffer != NULL) + cfilename = cfilename_buffer; else - Py_DECREF(v); + cfilename = ""; + fprintf(stderr, "%ls: can't open file '%s': [Errno %d] %s\n", + pymain->argv[0], cfilename, err, strerror(err)); + PyMem_Free(cfilename_buffer); + pymain->status = 2; + return NULL; + } + + if (cmdline->skip_first_line) { + int ch; + /* Push back first newline so line numbers + remain the same */ + while ((ch = getc(fp)) != EOF) { + if (ch == '\n') { + (void)ungetc(ch, fp); + break; + } + } } - if (cmdline.command) { - sts = run_command(cmdline.command, &cf); - PyMem_RawFree(cmdline.command); - } else if (cmdline.module) { - sts = (RunModule(cmdline.module, 1) != 0); + struct _Py_stat_struct sb; + if (_Py_fstat_noraise(fileno(fp), &sb) == 0 && + S_ISDIR(sb.st_mode)) { + fprintf(stderr, + "%ls: '%ls' is a directory, cannot continue\n", + pymain->argv[0], cmdline->filename); + fclose(fp); + pymain->status = 1; + return NULL; + } + + return fp; +} + + +static void +pymain_run_filename(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + + if (cmdline->filename == NULL && pymain->stdin_is_interactive) { + Py_InspectFlag = 0; /* do exit on SystemExit */ + pymain_run_startup(&pymain->cf); + pymain_run_interactive_hook(); + } + + if (pymain->main_importer_path != NULL) { + pymain->status = pymain_run_main_from_importer(pymain); + return; + } + + FILE *fp; + if (cmdline->filename != NULL) { + fp = pymain_open_filename(pymain); + if (fp == NULL) { + return; + } } else { + fp = stdin; + } + + pymain->status = pymain_run_file(fp, cmdline->filename, &pymain->cf); +} + + +static void +pymain_repl(_PyMain *pymain) +{ + /* Check this environment variable at the end, to give programs the + opportunity to set it from Python. */ + if (!Py_InspectFlag && pymain_get_env_var("PYTHONINSPECT")) { + Py_InspectFlag = 1; + } + + if (!(Py_InspectFlag && pymain->stdin_is_interactive + && pymain->run_code)) { + return; + } + + Py_InspectFlag = 0; + pymain_run_interactive_hook(); + + int res = PyRun_AnyFileFlags(stdin, "", &pymain->cf); + pymain->status = (res != 0); +} + + +/* Parse the command line. + Handle --version and --help options directly. + + Return 1 if Python must exit. + Return 0 on success. + Set pymain->err and return -1 on failure. */ +static int +pymain_parse_cmdline(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + + int res = pymain_parse_cmdline_impl(pymain); + if (res < 0) { + return -1; + } + if (res) { + pymain_usage(1, pymain->argv[0]); + pymain->status = 2; + return 1; + } + + if (cmdline->print_help) { + pymain_usage(0, pymain->argv[0]); + pymain->status = 0; + return 1; + } - if (cmdline.filename == NULL && stdin_is_interactive) { - Py_InspectFlag = 0; /* do exit on SystemExit */ - RunStartupFile(&cf); - RunInteractiveHook(); + if (cmdline->print_version) { + printf("Python %s\n", + (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION); + return 1; + } + + pymain->run_code = (cmdline->command != NULL || cmdline->filename != NULL + || cmdline->module != NULL); + + return 0; +} + + +static wchar_t* +pymain_get_xoption(_PyMain *pymain, wchar_t *name) +{ + _Py_OptList *list = &pymain->cmdline.xoptions; + for (size_t i=0; i < list->len; i++) { + wchar_t *option = list->options[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { + return option; } - /* XXX */ + } + return NULL; +} - sts = -1; /* keep track of whether we've already run __main__ */ - if (main_importer_path != NULL) { - sts = RunMainFromImporter(main_importer_path); +static int +pymain_str_to_int(char *str, int *result) +{ + errno = 0; + char *endptr = str; + long value = strtol(str, &endptr, 10); + if (*endptr != '\0' || errno == ERANGE) { + return -1; + } + if (value < INT_MIN || value > INT_MAX) { + return -1; + } + + *result = (int)value; + return 0; +} + + +static int +pymain_wstr_to_int(wchar_t *wstr, int *result) +{ + errno = 0; + wchar_t *endptr = wstr; + long value = wcstol(wstr, &endptr, 10); + if (*endptr != '\0' || errno == ERANGE) { + return -1; + } + if (value < INT_MIN || value > INT_MAX) { + return -1; + } + + *result = (int)value; + return 0; +} + + +static int +pymain_init_tracemalloc(_PyMain *pymain) +{ + int nframe; + int valid; + + char *env = pymain_get_env_var("PYTHONTRACEMALLOC"); + if (env) { + if (!pymain_str_to_int(env, &nframe)) { + valid = (nframe >= 1); } + else { + valid = 0; + } + if (!valid) { + pymain->err = _Py_INIT_USER_ERR("PYTHONTRACEMALLOC: invalid " + "number of frames"); + return -1; + } + pymain->core_config.tracemalloc = nframe; + } - if (sts==-1 && cmdline.filename != NULL) { - fp = _Py_wfopen(cmdline.filename, L"r"); - if (fp == NULL) { - char *cfilename_buffer; - const char *cfilename; - int err = errno; - cfilename_buffer = Py_EncodeLocale(cmdline.filename, NULL); - if (cfilename_buffer != NULL) - cfilename = cfilename_buffer; - else - cfilename = ""; - fprintf(stderr, "%ls: can't open file '%s': [Errno %d] %s\n", - argv[0], cfilename, err, strerror(err)); - if (cfilename_buffer) - PyMem_Free(cfilename_buffer); - return 2; + wchar_t *xoption = pymain_get_xoption(pymain, L"tracemalloc"); + if (xoption) { + wchar_t *sep = wcschr(xoption, L'='); + if (sep) { + if (!pymain_wstr_to_int(sep + 1, &nframe)) { + valid = (nframe >= 1); } - else if (cmdline.skip_first_line) { - int ch; - /* Push back first newline so line numbers - remain the same */ - while ((ch = getc(fp)) != EOF) { - if (ch == '\n') { - (void)ungetc(ch, fp); - break; - } - } + else { + valid = 0; } - { - struct _Py_stat_struct sb; - if (_Py_fstat_noraise(fileno(fp), &sb) == 0 && - S_ISDIR(sb.st_mode)) { - fprintf(stderr, - "%ls: '%ls' is a directory, cannot continue\n", - argv[0], cmdline.filename); - fclose(fp); - return 1; - } + if (!valid) { + pymain->err = _Py_INIT_USER_ERR("-X tracemalloc=NFRAME: " + "invalid number of frames"); + return -1; } } + else { + /* -X tracemalloc behaves as -X tracemalloc=1 */ + nframe = 1; + } + pymain->core_config.tracemalloc = nframe; + } + return 0; +} + - if (sts == -1) - sts = run_file(fp, cmdline.filename, &cf); +static void +pymain_set_flag_from_env(int *flag, const char *name) +{ + char *var = pymain_get_env_var(name); + if (!var) { + return; + } + int value; + if (pymain_str_to_int(var, &value) < 0 || value < 0) { + /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ + value = 1; + } + if (*flag < value) { + *flag = value; } +} - /* Check this environment variable at the end, to give programs the - * opportunity to set it from Python. - */ - if (!Py_InspectFlag && - (p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') + +static void +pymain_set_flags_from_env(_PyMain *pymain) +{ + pymain_set_flag_from_env(&Py_DebugFlag, + "PYTHONDEBUG"); + pymain_set_flag_from_env(&Py_VerboseFlag, + "PYTHONVERBOSE"); + pymain_set_flag_from_env(&Py_OptimizeFlag, + "PYTHONOPTIMIZE"); + pymain_set_flag_from_env(&Py_InspectFlag, + "PYTHONINSPECT"); + pymain_set_flag_from_env(&Py_DontWriteBytecodeFlag, + "PYTHONDONTWRITEBYTECODE"); + pymain_set_flag_from_env(&Py_NoUserSiteDirectory, + "PYTHONNOUSERSITE"); + pymain_set_flag_from_env(&Py_UnbufferedStdioFlag, + "PYTHONUNBUFFERED"); +#ifdef MS_WINDOWS + pymain_set_flag_from_env(&Py_LegacyWindowsFSEncodingFlag, + "PYTHONLEGACYWINDOWSFSENCODING"); + pymain_set_flag_from_env(&Py_LegacyWindowsStdioFlag, + "PYTHONLEGACYWINDOWSSTDIO"); +#endif +} + + +static int +config_get_env_var_dup(wchar_t **dest, wchar_t *wname, char *name) +{ + if (Py_IgnoreEnvironmentFlag) { + *dest = NULL; + return 0; + } + +#ifdef MS_WINDOWS + wchar_t *var = _wgetenv(wname); + if (!var || var[0] == '\0') { + *dest = NULL; + return 0; + } + + wchar_t *copy = _PyMem_RawWcsdup(var); + if (copy == NULL) { + return -1; + } + + *dest = copy; +#else + char *var = getenv(name); + if (!var || var[0] == '\0') { + *dest = NULL; + return 0; + } + + size_t len; + wchar_t *wvar = Py_DecodeLocale(var, &len); + if (!wvar) { + if (len == (size_t)-2) { + return -2; + } + else { + return -1; + } + } + *dest = wvar; +#endif + return 0; +} + + +static _PyInitError +config_init_pythonpath(_PyMainInterpreterConfig *config) +{ + wchar_t *path; + int res = config_get_env_var_dup(&path, L"PYTHONPATH", "PYTHONPATH"); + if (res < 0) { + return DECODE_LOCALE_ERR("PYTHONHOME", res); + } + config->module_search_path_env = path; + return _Py_INIT_OK(); +} + + +static _PyInitError +config_init_home(_PyMainInterpreterConfig *config) +{ + wchar_t *home; + + /* If Py_SetPythonHome() was called, use its value */ + home = _Py_path_config.home; + if (home) { + config->home = _PyMem_RawWcsdup(home); + if (config->home == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); + } + + int res = config_get_env_var_dup(&home, L"PYTHONHOME", "PYTHONHOME"); + if (res < 0) { + return DECODE_LOCALE_ERR("PYTHONHOME", res); + } + config->home = home; + return _Py_INIT_OK(); +} + + +_PyInitError +_PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config) +{ + _PyInitError err = config_init_home(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_init_pythonpath(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = config_get_program_name(config); + if (_Py_INIT_FAILED(err)) { + return err; + } + + return _Py_INIT_OK(); +} + + + + +static int +pymain_parse_envvars(_PyMain *pymain) +{ + _PyCoreConfig *core_config = &pymain->core_config; + + /* Get environment variables */ + pymain_set_flags_from_env(pymain); + + /* The variable is only tested for existence here; + _Py_HashRandomization_Init will check its value further. */ + if (pymain_get_env_var("PYTHONHASHSEED")) { + Py_HashRandomizationFlag = 1; + } + + if (pymain_warnings_envvar(pymain) < 0) { + return -1; + } + + _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config); + if (_Py_INIT_FAILED(pymain->err)) { + pymain->err = err; + return -1; + } + if (pymain_get_program_name(pymain) < 0) { + return -1; + } + + core_config->allocator = Py_GETENV("PYTHONMALLOC"); + + /* -X options */ + if (pymain_get_xoption(pymain, L"showrefcount")) { + core_config->show_ref_count = 1; + } + if (pymain_get_xoption(pymain, L"showalloccount")) { + core_config->show_alloc_count = 1; + } + + /* More complex options: env var and/or -X option */ + if (pymain_get_env_var("PYTHONFAULTHANDLER") + || pymain_get_xoption(pymain, L"faulthandler")) { + core_config->faulthandler = 1; + } + if (pymain_get_env_var("PYTHONPROFILEIMPORTTIME") + || pymain_get_xoption(pymain, L"importtime")) { + core_config->import_time = 1; + } + if (pymain_init_tracemalloc(pymain) < 0) { + return -1; + } + if (pymain_get_xoption(pymain, L"dev" ) || + pymain_get_env_var("PYTHONDEVMODE")) { - Py_InspectFlag = 1; + core_config->dev_mode = 1; + core_config->faulthandler = 1; + core_config->allocator = "debug"; } + return 0; +} + + +/* Parse command line options and environment variables. + This code must not use Python runtime apart PyMem_Raw memory allocator. - if (Py_InspectFlag && stdin_is_interactive && - (cmdline.filename != NULL || cmdline.command != NULL || cmdline.module != NULL)) { - Py_InspectFlag = 0; - RunInteractiveHook(); - /* XXX */ - sts = PyRun_AnyFileFlags(stdin, "", &cf) != 0; + Return 0 on success. + Return 1 if Python is done and must exit. + Set pymain->err and return -1 on error. */ +static int +pymain_parse_cmdline_envvars_impl(_PyMain *pymain) +{ + int res = pymain_parse_cmdline(pymain); + if (res < 0) { + return -1; } + if (res > 0) { + return 1; + } + + pymain_set_global_config(pymain); + + if (pymain_parse_envvars(pymain) < 0) { + return -1; + } + + _PyInitError err = _PyMainInterpreterConfig_Read(&pymain->config); + if (_Py_INIT_FAILED(err)) { + pymain->err = err; + return -1; + } + + return 0; +} + + +static int +pymain_parse_cmdline_envvars(_PyMain *pymain) +{ + /* Force default allocator, since pymain_free() must use the same allocator + than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + int res = pymain_parse_cmdline_envvars_impl(pymain); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return res; +} + +static int +pymain_init_python(_PyMain *pymain) +{ + pymain_init_stdio(pymain); + + pymain->err = _Py_InitializeCore(&pymain->core_config); + if (_Py_INIT_FAILED(pymain->err)) { + return -1; + } + + if (pymain_add_xoptions(pymain)) { + return -1; + } + if (pymain_add_warnings_options(pymain)) { + return -1; + } + + if (pymain_init_main_interpreter(pymain)) { + return -1; + } + return 0; +} + + +static void +pymain_run_python(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; + + pymain_header(pymain); + pymain_import_readline(pymain); + + if (cmdline->filename != NULL) { + pymain->main_importer_path = pymain_get_importer(cmdline->filename); + } + + pymain_set_argv(pymain); + + if (cmdline->command) { + pymain->status = pymain_run_command(cmdline->command, &pymain->cf); + } + else if (cmdline->module) { + pymain->status = (pymain_run_module(cmdline->module, 1) != 0); + } + else { + pymain_run_filename(pymain); + } + pymain_repl(pymain); +} + + +static int +pymain_init(_PyMain *pymain) +{ + pymain->err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(pymain->err)) { + return -1; + } + + pymain->core_config._disable_importlib = 0; + pymain->config.install_signal_handlers = 1; + + orig_argc = pymain->argc; /* For Py_GetArgcArgv() */ + orig_argv = pymain->argv; + return 0; +} + + +static int +pymain_impl(_PyMain *pymain) +{ + int res = pymain_init(pymain); + if (res < 0) { + return -1; + } + + res = pymain_parse_cmdline_envvars(pymain); + if (res < 0) { + return -1; + } + if (res > 0) { + /* --help or --version command: we are done */ + return 0; + } + + res = pymain_init_python(pymain); + if (res < 0) { + return -1; + } + + pymain_run_python(pymain); if (Py_FinalizeEx() < 0) { /* Value unlikely to be confused with a non-error exit status or - other special meaning */ - sts = 120; + other special meaning */ + pymain->status = 120; } -#ifdef __INSURE__ - /* Insure++ is a memory analysis tool that aids in discovering - * memory leaks and other memory problems. On Python exit, the - * interned string dictionaries are flagged as being in use at exit - * (which it is). Under normal circumstances, this is fine because - * the memory will be automatically reclaimed by the system. Under - * memory debugging, it's a huge source of useless noise, so we - * trade off slower shutdown for less distraction in the memory - * reports. -baw - */ - _Py_ReleaseInternedUnicodeStrings(); -#endif /* __INSURE__ */ + /* _PyPathConfig_Clear() cannot be called in Py_FinalizeEx(). + Py_Initialize() and Py_Finalize() can be called multiple times, but it + must not "forget" parameters set by Py_SetProgramName(), Py_SetPath() or + Py_SetPythonHome(), whereas _PyPathConfig_Clear() clear all these + parameters. */ + _PyPathConfig_Clear(&_Py_path_config); + + return 0; +} + + +int +Py_Main(int argc, wchar_t **argv) +{ + _PyMain pymain = _PyMain_INIT; + memset(&pymain.cmdline, 0, sizeof(pymain.cmdline)); + pymain.argc = argc; + pymain.argv = argv; + + if (pymain_impl(&pymain) < 0) { + _Py_FatalInitError(pymain.err); + } + pymain_free(&pymain); - return sts; + return pymain.status; } /* this is gonna seem *real weird*, but if you put some other code between diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index ea7baf4bd087a7..6cf454573e936c 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1210,7 +1210,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - char *tagname = ""; + const char *tagname = ""; DWORD dwErr = 0; int fileno; HANDLE fh = 0; diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index 8bb4d0d3d4374e..58ee71f9010858 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -53,7 +53,7 @@ typedef unsigned long uint32_t; typedef struct { PyObject_HEAD - char *devicename; /* name of the device file */ + const char *devicename; /* name of the device file */ int fd; /* file descriptor */ int mode; /* file mode (O_RDONLY, etc.) */ Py_ssize_t icount; /* input count */ @@ -82,8 +82,8 @@ newossobject(PyObject *arg) { oss_audio_t *self; int fd, afmts, imode; - char *devicename = NULL; - char *mode = NULL; + const char *devicename = NULL; + const char *mode = NULL; /* Two ways to call open(): open(device, mode) (for consistency with builtin open()) @@ -167,7 +167,7 @@ oss_dealloc(oss_audio_t *self) static oss_mixer_t * newossmixerobject(PyObject *arg) { - char *devicename = NULL; + const char *devicename = NULL; int fd; oss_mixer_t *self; diff --git a/Modules/readline.c b/Modules/readline.c index 951bc82b16dfb7..811fca8cd92a72 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1245,7 +1245,7 @@ call_readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) char *saved_locale = strdup(setlocale(LC_CTYPE, NULL)); if (!saved_locale) Py_FatalError("not enough memory to save locale"); - setlocale(LC_CTYPE, ""); + _Py_SetLocaleFromEnv(LC_CTYPE); #endif if (sys_stdin != rl_instream || sys_stdout != rl_outstream) { @@ -1352,13 +1352,27 @@ PyInit_readline(void) if (m == NULL) return NULL; + if (PyModule_AddIntConstant(m, "_READLINE_VERSION", + RL_READLINE_VERSION) < 0) { + goto error; + } + if (PyModule_AddIntConstant(m, "_READLINE_RUNTIME_VERSION", + rl_readline_version) < 0) { + goto error; + } + if (PyModule_AddStringConstant(m, "_READLINE_LIBRARY_VERSION", + rl_library_version) < 0) + { + goto error; + } + mod_state = (readlinestate *) PyModule_GetState(m); PyOS_ReadlineFunctionPointer = call_readline; setup_readline(mod_state); - PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION); - PyModule_AddIntConstant(m, "_READLINE_RUNTIME_VERSION", rl_readline_version); - PyModule_AddStringConstant(m, "_READLINE_LIBRARY_VERSION", rl_library_version); - return m; + +error: + Py_DECREF(m); + return NULL; } diff --git a/Modules/rotatingtree.h b/Modules/rotatingtree.h index 3aa0986b49aa9a..7b3e5fde921cb3 100644 --- a/Modules/rotatingtree.h +++ b/Modules/rotatingtree.h @@ -16,12 +16,12 @@ typedef struct rotating_node_s rotating_node_t; typedef int (*rotating_tree_enum_fn) (rotating_node_t *node, void *arg); struct rotating_node_s { - void *key; - rotating_node_t *left; - rotating_node_t *right; + void *key; + rotating_node_t *left; + rotating_node_t *right; }; void RotatingTree_Add(rotating_node_t **root, rotating_node_t *node); rotating_node_t* RotatingTree_Get(rotating_node_t **root, void *key); int RotatingTree_Enum(rotating_node_t *root, rotating_tree_enum_fn enumfn, - void *arg); + void *arg); diff --git a/Modules/sre.h b/Modules/sre.h index 585d2841a6636a..a7284881457c3b 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -67,6 +67,7 @@ typedef struct { void* end; /* end of original string */ /* attributes for the match object */ PyObject* string; + Py_buffer buffer; Py_ssize_t pos, endpos; int isbytes; int charsize; /* character size */ @@ -74,11 +75,12 @@ typedef struct { Py_ssize_t lastindex; Py_ssize_t lastmark; void** mark; + int match_all; + int must_advance; /* dynamically allocated stuff */ char* data_stack; size_t data_stack_size; size_t data_stack_base; - Py_buffer buffer; /* current repeat context */ SRE_REPEAT *repeat; } SRE_STATE; diff --git a/Modules/sre_lib.h b/Modules/sre_lib.h index e13b90e8bc0eb8..44948e21ad9553 100644 --- a/Modules/sre_lib.h +++ b/Modules/sre_lib.h @@ -199,7 +199,7 @@ SRE(charset_loc_ignore)(SRE_STATE* state, SRE_CODE* set, SRE_CODE ch) return up != lo && SRE(charset)(state, set, up); } -LOCAL(Py_ssize_t) SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int match_all); +LOCAL(Py_ssize_t) SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel); LOCAL(Py_ssize_t) SRE(count)(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) @@ -510,12 +510,12 @@ do { \ #define JUMP_ASSERT 12 #define JUMP_ASSERT_NOT 13 -#define DO_JUMPX(jumpvalue, jumplabel, nextpattern, matchall) \ +#define DO_JUMPX(jumpvalue, jumplabel, nextpattern, toplevel_) \ DATA_ALLOC(SRE(match_context), nextctx); \ nextctx->last_ctx_pos = ctx_pos; \ nextctx->jump = jumpvalue; \ nextctx->pattern = nextpattern; \ - nextctx->match_all = matchall; \ + nextctx->toplevel = toplevel_; \ ctx_pos = alloc_pos; \ ctx = nextctx; \ goto entrance; \ @@ -523,7 +523,7 @@ do { \ while (0) /* gcc doesn't like labels at end of scopes */ \ #define DO_JUMP(jumpvalue, jumplabel, nextpattern) \ - DO_JUMPX(jumpvalue, jumplabel, nextpattern, ctx->match_all) + DO_JUMPX(jumpvalue, jumplabel, nextpattern, ctx->toplevel) #define DO_JUMP0(jumpvalue, jumplabel, nextpattern) \ DO_JUMPX(jumpvalue, jumplabel, nextpattern, 0) @@ -540,13 +540,13 @@ typedef struct { SRE_CODE chr; SRE_REPEAT* rep; } u; - int match_all; + int toplevel; } SRE(match_context); /* check if string matches the given pattern. returns <0 for error, 0 for failure, and 1 for success */ LOCAL(Py_ssize_t) -SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int match_all) +SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) { SRE_CHAR* end = (SRE_CHAR *)state->end; Py_ssize_t alloc_pos, ctx_pos = -1; @@ -563,7 +563,7 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int match_all) ctx->last_ctx_pos = -1; ctx->jump = JUMP_NONE; ctx->pattern = pattern; - ctx->match_all = match_all; + ctx->toplevel = toplevel; ctx_pos = alloc_pos; entrance: @@ -636,11 +636,14 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int match_all) case SRE_OP_SUCCESS: /* end of pattern */ TRACE(("|%p|%p|SUCCESS\n", ctx->pattern, ctx->ptr)); - if (!ctx->match_all || ctx->ptr == state->end) { - state->ptr = ctx->ptr; - RETURN_SUCCESS; + if (ctx->toplevel && + ((state->match_all && ctx->ptr != state->end) || + (state->must_advance && ctx->ptr == state->start))) + { + RETURN_FAILURE; } - RETURN_FAILURE; + state->ptr = ctx->ptr; + RETURN_SUCCESS; case SRE_OP_AT: /* match at given position */ @@ -856,7 +859,9 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int match_all) RETURN_FAILURE; if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS && - ctx->ptr == state->end) { + ctx->ptr == state->end && + !(ctx->toplevel && state->must_advance && ctx->ptr == state->start)) + { /* tail is empty. we're finished */ state->ptr = ctx->ptr; RETURN_SUCCESS; @@ -941,7 +946,10 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int match_all) } if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS && - (!match_all || ctx->ptr == state->end)) { + !(ctx->toplevel && + ((state->match_all && ctx->ptr != state->end) || + (state->must_advance && ctx->ptr == state->start)))) + { /* tail is empty. we're finished */ state->ptr = ctx->ptr; RETURN_SUCCESS; @@ -1417,6 +1425,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) return 0; /* literal can't match: doesn't fit in char width */ #endif end = (SRE_CHAR *)state->end; + state->must_advance = 0; while (ptr < end) { while (*ptr != c) { if (++ptr >= end) @@ -1458,6 +1467,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) return 0; i = 1; + state->must_advance = 0; do { if (*ptr == (SRE_CHAR) prefix[i]) { if (++i != prefix_len) { @@ -1487,6 +1497,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) if (charset) { /* pattern starts with a character from a known set */ end = (SRE_CHAR *)state->end; + state->must_advance = 0; for (;;) { while (ptr < end && !SRE(charset)(state, charset, *ptr)) ptr++; @@ -1503,13 +1514,15 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) } else { /* general case */ assert(ptr <= end); - while (1) { + TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); + state->start = state->ptr = ptr; + status = SRE(match)(state, pattern, 1); + state->must_advance = 0; + while (status == 0 && ptr < end) { + ptr++; TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); state->start = state->ptr = ptr; status = SRE(match)(state, pattern, 0); - if (status != 0 || ptr >= end) - break; - ptr++; } } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 37abeb95077f1e..5cae03dd416ef7 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1258,6 +1258,112 @@ Process time for profiling as nanoseconds:\n\ sum of the kernel and user-space CPU time."); +#if defined(MS_WINDOWS) +#define HAVE_THREAD_TIME +static int +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +{ + HANDLE thread; + FILETIME creation_time, exit_time, kernel_time, user_time; + ULARGE_INTEGER large; + _PyTime_t ktime, utime, t; + BOOL ok; + + thread = GetCurrentThread(); + ok = GetThreadTimes(thread, &creation_time, &exit_time, + &kernel_time, &user_time); + if (!ok) { + PyErr_SetFromWindowsErr(0); + return -1; + } + + if (info) { + info->implementation = "GetThreadTimes()"; + info->resolution = 1e-7; + info->monotonic = 1; + info->adjustable = 0; + } + + large.u.LowPart = kernel_time.dwLowDateTime; + large.u.HighPart = kernel_time.dwHighDateTime; + ktime = large.QuadPart; + + large.u.LowPart = user_time.dwLowDateTime; + large.u.HighPart = user_time.dwHighDateTime; + utime = large.QuadPart; + + /* ktime and utime have a resolution of 100 nanoseconds */ + t = _PyTime_FromNanoseconds((ktime + utime) * 100); + *tp = t; + return 0; +} + +#elif defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) +#define HAVE_THREAD_TIME +static int +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +{ + struct timespec ts; + const clockid_t clk_id = CLOCK_THREAD_CPUTIME_ID; + const char *function = "clock_gettime(CLOCK_THREAD_CPUTIME_ID)"; + + if (clock_gettime(clk_id, &ts)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + if (info) { + struct timespec res; + info->implementation = function; + info->monotonic = 1; + info->adjustable = 0; + if (clock_getres(clk_id, &res)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + } + + if (_PyTime_FromTimespec(tp, &ts) < 0) { + return -1; + } + return 0; +} +#endif + +#ifdef HAVE_THREAD_TIME +static PyObject * +time_thread_time(PyObject *self, PyObject *unused) +{ + _PyTime_t t; + if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { + return NULL; + } + return _PyFloat_FromPyTime(t); +} + +PyDoc_STRVAR(thread_time_doc, +"thread_time() -> float\n\ +\n\ +Thread time for profiling: sum of the kernel and user-space CPU time."); + +static PyObject * +time_thread_time_ns(PyObject *self, PyObject *unused) +{ + _PyTime_t t; + if (_PyTime_GetThreadTimeWithInfo(&t, NULL) < 0) { + return NULL; + } + return _PyTime_AsNanosecondsObject(t); +} + +PyDoc_STRVAR(thread_time_ns_doc, +"thread_time() -> int\n\ +\n\ +Thread time for profiling as nanoseconds:\n\ +sum of the kernel and user-space CPU time."); +#endif + + static PyObject * time_get_clock_info(PyObject *self, PyObject *args) { @@ -1311,6 +1417,13 @@ time_get_clock_info(PyObject *self, PyObject *args) return NULL; } } +#ifdef HAVE_THREAD_TIME + else if (strcmp(name, "thread_time") == 0) { + if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + return NULL; + } + } +#endif else { PyErr_SetString(PyExc_ValueError, "unknown clock"); return NULL; @@ -1519,6 +1632,10 @@ static PyMethodDef time_methods[] = { {"monotonic_ns", time_monotonic_ns, METH_NOARGS, monotonic_ns_doc}, {"process_time", time_process_time, METH_NOARGS, process_time_doc}, {"process_time_ns", time_process_time_ns, METH_NOARGS, process_time_ns_doc}, +#ifdef HAVE_THREAD_TIME + {"thread_time", time_thread_time, METH_NOARGS, thread_time_doc}, + {"thread_time_ns", time_thread_time_ns, METH_NOARGS, thread_time_ns_doc}, +#endif {"perf_counter", time_perf_counter, METH_NOARGS, perf_counter_doc}, {"perf_counter_ns", time_perf_counter_ns, METH_NOARGS, perf_counter_ns_doc}, {"get_clock_info", time_get_clock_info, METH_VARARGS, get_clock_info_doc}, diff --git a/Objects/abstract.c b/Objects/abstract.c index 3cb7a32b01ee5a..ec11a1e5f3f789 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -143,6 +143,7 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; + PyObject *meth, *result, *stack[2] = {o, key}; if (o == NULL || key == NULL) { return null_error(); @@ -168,6 +169,25 @@ PyObject_GetItem(PyObject *o, PyObject *key) "be integer, not '%.200s'", key); } + if (PyType_Check(o)) { + meth = PyObject_GetAttrString(o, "__class_getitem__"); + if (meth) { + if (!PyCallable_Check(meth)) { + PyErr_SetString(PyExc_TypeError, + "__class_getitem__ must be callable"); + Py_DECREF(meth); + return NULL; + } + result = _PyObject_FastCall(meth, stack, 2); + Py_DECREF(meth); + return result; + } + else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + } + return type_error("'%.200s' object is not subscriptable", o); } diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 83c3549d50bd07..dc1515a059cb94 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1806,7 +1806,8 @@ bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) /*[clinic end generated code: output=760412661a34ad5a input=ef7bb59b09c21d62]*/ { Py_ssize_t left, right, mysize, byteslen; - char *myptr, *bytesptr; + char *myptr; + const char *bytesptr; Py_buffer vbytes; if (bytes == Py_None) { @@ -1816,7 +1817,7 @@ bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) else { if (PyObject_GetBuffer(bytes, &vbytes, PyBUF_SIMPLE) != 0) return NULL; - bytesptr = (char *) vbytes.buf; + bytesptr = (const char *) vbytes.buf; byteslen = vbytes.len; } myptr = PyByteArray_AS_STRING(self); @@ -1847,7 +1848,8 @@ bytearray_lstrip_impl(PyByteArrayObject *self, PyObject *bytes) /*[clinic end generated code: output=d005c9d0ab909e66 input=80843f975dd7c480]*/ { Py_ssize_t left, right, mysize, byteslen; - char *myptr, *bytesptr; + char *myptr; + const char *bytesptr; Py_buffer vbytes; if (bytes == Py_None) { @@ -1857,7 +1859,7 @@ bytearray_lstrip_impl(PyByteArrayObject *self, PyObject *bytes) else { if (PyObject_GetBuffer(bytes, &vbytes, PyBUF_SIMPLE) != 0) return NULL; - bytesptr = (char *) vbytes.buf; + bytesptr = (const char *) vbytes.buf; byteslen = vbytes.len; } myptr = PyByteArray_AS_STRING(self); @@ -1885,7 +1887,8 @@ bytearray_rstrip_impl(PyByteArrayObject *self, PyObject *bytes) /*[clinic end generated code: output=030e2fbd2f7276bd input=e728b994954cfd91]*/ { Py_ssize_t right, mysize, byteslen; - char *myptr, *bytesptr; + char *myptr; + const char *bytesptr; Py_buffer vbytes; if (bytes == Py_None) { @@ -1895,7 +1898,7 @@ bytearray_rstrip_impl(PyByteArrayObject *self, PyObject *bytes) else { if (PyObject_GetBuffer(bytes, &vbytes, PyBUF_SIMPLE) != 0) return NULL; - bytesptr = (char *) vbytes.buf; + bytesptr = (const char *) vbytes.buf; byteslen = vbytes.len; } myptr = PyByteArray_AS_STRING(self); diff --git a/Objects/classobject.c b/Objects/classobject.c index 063c24a7171e8f..e88c95cbfbd0b2 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -543,7 +543,7 @@ instancemethod_repr(PyObject *self) { PyObject *func = PyInstanceMethod_Function(self); PyObject *funcname = NULL , *result = NULL; - char *defname = "?"; + const char *defname = "?"; if (func == NULL) { PyErr_BadInternalCall(); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index f312f338a9b4eb..0509b8e6400fb8 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -124,12 +124,20 @@ PyCode_New(int argcount, int kwonlyargcount, if (PyUnicode_READY(filename) < 0) return NULL; - n_cellvars = PyTuple_GET_SIZE(cellvars); intern_strings(names); intern_strings(varnames); intern_strings(freevars); intern_strings(cellvars); intern_string_constants(consts); + + /* Check for any inner or outer closure references */ + n_cellvars = PyTuple_GET_SIZE(cellvars); + if (!n_cellvars && !PyTuple_GET_SIZE(freevars)) { + flags |= CO_NOFREE; + } else { + flags &= ~CO_NOFREE; + } + /* Create mapping between cells and arguments if needed. */ if (n_cellvars) { Py_ssize_t total_args = argcount + kwonlyargcount + diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 3bf37ee437f557..2c886c7a7685db 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -363,9 +363,9 @@ complex_repr(PyComplexObject *v) /* These do not need to be freed. re is either an alias for pre or a pointer to a constant. lead and tail are pointers to constants. */ - char *re = NULL; - char *lead = ""; - char *tail = ""; + const char *re = NULL; + const char *lead = ""; + const char *tail = ""; if (v->cval.real == 0. && copysign(1.0, v->cval.real)==1.0) { /* Real part is +0: just output the imaginary part and do not @@ -914,10 +914,10 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) if (s_buffer == NULL) { return NULL; } + assert(PyUnicode_IS_ASCII(s_buffer)); + /* Simply get a pointer to existing ASCII characters. */ s = PyUnicode_AsUTF8AndSize(s_buffer, &len); - if (s == NULL) { - goto exit; - } + assert(s != NULL); } else { PyErr_Format(PyExc_TypeError, @@ -928,7 +928,6 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v) result = _Py_string_to_number_with_underscores(s, len, "complex", v, type, complex_from_string_inner); - exit: Py_DECREF(s_buffer); return result; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 779746a31aa19f..b20b85c909e333 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2183,16 +2183,25 @@ dict_update_common(PyObject *self, PyObject *args, PyObject *kwds, PyObject *arg = NULL; int result = 0; - if (!PyArg_UnpackTuple(args, methname, 0, 1, &arg)) + if (!PyArg_UnpackTuple(args, methname, 0, 1, &arg)) { result = -1; - + } else if (arg != NULL) { _Py_IDENTIFIER(keys); - if (_PyObject_HasAttrId(arg, &PyId_keys)) + PyObject *func = _PyObject_GetAttrId(arg, &PyId_keys); + if (func != NULL) { + Py_DECREF(func); result = PyDict_Merge(self, arg, 1); - else + } + else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); result = PyDict_MergeFromSeq2(self, arg, 1); + } + else { + result = -1; + } } + if (result == 0 && kwds != NULL) { if (PyArg_ValidateKeywordArguments(kwds)) result = PyDict_Merge(self, kwds, 1); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 637d7660d357e3..4901eb1cc3c560 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -117,7 +117,11 @@ static PyObject * BaseException_repr(PyBaseExceptionObject *self) { const char *name = _PyType_Name(Py_TYPE(self)); - return PyUnicode_FromFormat("%s%R", name, self->args); + if (PyTuple_GET_SIZE(self->args) == 1) + return PyUnicode_FromFormat("%s(%R)", name, + PyTuple_GET_ITEM(self->args, 0)); + else + return PyUnicode_FromFormat("%s%R", name, self->args); } /* Pickling support */ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 8d7a55ac6f6401..47a174c241e371 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -176,11 +176,10 @@ PyFloat_FromString(PyObject *v) s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v); if (s_buffer == NULL) return NULL; + assert(PyUnicode_IS_ASCII(s_buffer)); + /* Simply get a pointer to existing ASCII characters. */ s = PyUnicode_AsUTF8AndSize(s_buffer, &len); - if (s == NULL) { - Py_DECREF(s_buffer); - return NULL; - } + assert(s != NULL); } else if (PyBytes_Check(v)) { s = PyBytes_AS_STRING(v); diff --git a/Objects/genobject.c b/Objects/genobject.c index 7793a54fb3d682..00a882379fcabe 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -152,7 +152,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) PyObject *result; if (gen->gi_running) { - char *msg = "generator already executing"; + const char *msg = "generator already executing"; if (PyCoro_CheckExact(gen)) { msg = "coroutine already executing"; } @@ -186,8 +186,8 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) if (f->f_lasti == -1) { if (arg && arg != Py_None) { - char *msg = "can't send non-None value to a " - "just-started generator"; + const char *msg = "can't send non-None value to a " + "just-started generator"; if (PyCoro_CheckExact(gen)) { msg = NON_INIT_CORO_MSG; } @@ -410,7 +410,7 @@ gen_close(PyGenObject *gen, PyObject *args) PyErr_SetNone(PyExc_GeneratorExit); retval = gen_send_ex(gen, Py_None, 1, 1); if (retval) { - char *msg = "generator ignored GeneratorExit"; + const char *msg = "generator ignored GeneratorExit"; if (PyCoro_CheckExact(gen)) { msg = "coroutine ignored GeneratorExit"; } else if (PyAsyncGen_CheckExact(gen)) { diff --git a/Objects/listobject.c b/Objects/listobject.c index 7eba61e7fed1fd..8794e37364a81f 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -85,15 +85,10 @@ static size_t count_reuse = 0; static void show_alloc(void) { - PyObject *xoptions, *value; - _Py_IDENTIFIER(showalloccount); - - xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return; - value = _PyDict_GetItemId(xoptions, &PyId_showalloccount); - if (value != Py_True) + PyInterpreterState *interp = PyThreadState_GET()->interp; + if (!inter->core_config.show_alloc_count) { return; + } fprintf(stderr, "List allocations: %" PY_FORMAT_SIZE_T "d\n", count_alloc); @@ -369,10 +364,7 @@ list_repr(PyListObject *v) goto error; } - if (Py_EnterRecursiveCall(" while getting the repr of a list")) - goto error; s = PyObject_Repr(v->ob_item[i]); - Py_LeaveRecursiveCall(); if (s == NULL) goto error; diff --git a/Objects/longobject.c b/Objects/longobject.c index 6fd4feba12afed..a7f496825ebd1f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2024,7 +2024,7 @@ long_from_binary_base(const char **str, int base, PyLongObject **res) const char *p = *str; const char *start = p; char prev = 0; - int digits = 0; + Py_ssize_t digits = 0; int bits_per_char; Py_ssize_t n; PyLongObject *z; @@ -2267,8 +2267,9 @@ just 1 digit at the start, so that the copying code was exercised for every digit beyond the first. ***/ twodigits c; /* current input character */ + double fsize_z; Py_ssize_t size_z; - int digits = 0; + Py_ssize_t digits = 0; int i; int convwidth; twodigits convmultmax, convmult; @@ -2330,7 +2331,14 @@ digit beyond the first. * need to initialize z->ob_digit -- no slot is read up before * being stored into. */ - size_z = (Py_ssize_t)(digits * log_base_BASE[base]) + 1; + fsize_z = digits * log_base_BASE[base] + 1; + if (fsize_z > MAX_LONG_DIGITS) { + /* The same exception as in _PyLong_New(). */ + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + return NULL; + } + size_z = (Py_ssize_t)fsize_z; /* Uncomment next line to test exceedingly rare copy code */ /* size_z = 1; */ assert(size_z > 0); @@ -2509,21 +2517,18 @@ PyLong_FromUnicodeObject(PyObject *u, int base) asciidig = _PyUnicode_TransformDecimalAndSpaceToASCII(u); if (asciidig == NULL) return NULL; + assert(PyUnicode_IS_ASCII(asciidig)); + /* Simply get a pointer to existing ASCII characters. */ buffer = PyUnicode_AsUTF8AndSize(asciidig, &buflen); - if (buffer == NULL) { - Py_DECREF(asciidig); - if (!PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - return NULL; - } - else { - result = PyLong_FromString(buffer, &end, base); - if (end == NULL || (result != NULL && end == buffer + buflen)) { - Py_DECREF(asciidig); - return result; - } + assert(buffer != NULL); + + result = PyLong_FromString(buffer, &end, base); + if (end == NULL || (result != NULL && end == buffer + buflen)) { Py_DECREF(asciidig); - Py_XDECREF(result); + return result; } + Py_DECREF(asciidig); + Py_XDECREF(result); PyErr_Format(PyExc_ValueError, "invalid literal for int() with base %d: %.200R", base, u); @@ -4811,7 +4816,7 @@ long_new_impl(PyTypeObject *type, PyObject *x, PyObject *obase) return NULL; if ((base != 0 && base < 2) || base > 36) { PyErr_SetString(PyExc_ValueError, - "int() base must be >= 2 and <= 36"); + "int() base must be >= 2 and <= 36, or 0"); return NULL; } diff --git a/Objects/object.c b/Objects/object.c index ed8a62a163aa65..a0d651d0805a0b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -29,17 +29,6 @@ _Py_GetRefTotal(void) return total; } -PyObject * -_PyDebug_XOptionShowRefCount(void) -{ - PyObject *xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return NULL; - - _Py_IDENTIFIER(showrefcount); - return _PyDict_GetItemId(xoptions, &PyId_showrefcount); -} - void _PyDebug_PrintTotalRefs(void) { fprintf(stderr, @@ -106,16 +95,10 @@ extern Py_ssize_t null_strings, one_strings; void dump_counts(FILE* f) { - PyTypeObject *tp; - PyObject *xoptions, *value; - _Py_IDENTIFIER(showalloccount); - - xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return; - value = _PyDict_GetItemId(xoptions, &PyId_showalloccount); - if (value != Py_True) + PyInterpreterState *interp = PyThreadState_GET()->interp; + if (!inter->core_config.show_alloc_count) { return; + } for (tp = type_list; tp; tp = tp->tp_next) fprintf(f, "%s alloc'd: %" PY_FORMAT_SIZE_T "d, " @@ -480,7 +463,12 @@ PyObject_Repr(PyObject *v) assert(!PyErr_Occurred()); #endif + /* It is possible for a type to have a tp_repr representation that loops + infinitely. */ + if (Py_EnterRecursiveCall(" while getting the repr of an object")) + return NULL; res = (*v->ob_type->tp_repr)(v); + Py_LeaveRecursiveCall(); if (res == NULL) return NULL; if (!PyUnicode_Check(res)) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7f5306f9dcbc38..7911028ad83f6a 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,6 +1,4 @@ #include "Python.h" -#include "internal/mem.h" -#include "internal/pystate.h" #include @@ -28,6 +26,8 @@ static void _PyMem_DebugFree(void *ctx, void *p); static void _PyObject_DebugDumpAddress(const void *p); static void _PyMem_DebugCheckAddress(char api_id, const void *p); +static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain); + #if defined(__has_feature) /* Clang */ #if __has_feature(address_sanitizer) /* is ASAN enabled? */ #define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS \ @@ -151,14 +151,18 @@ _PyObject_ArenaFree(void *ctx, void *ptr, size_t size) } #endif +#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} +#ifdef WITH_PYMALLOC +# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} +#endif -#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree +#define PYRAW_ALLOC MALLOC_ALLOC #ifdef WITH_PYMALLOC -# define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free +# define PYOBJ_ALLOC PYMALLOC_ALLOC #else -# define PYOBJ_FUNCS PYRAW_FUNCS +# define PYOBJ_ALLOC MALLOC_ALLOC #endif -#define PYMEM_FUNCS PYOBJ_FUNCS +#define PYMEM_ALLOC PYOBJ_ALLOC typedef struct { /* We tag each block with an API ID in order to tag API violations */ @@ -170,96 +174,118 @@ static struct { debug_alloc_api_t mem; debug_alloc_api_t obj; } _PyMem_Debug = { - {'r', {NULL, PYRAW_FUNCS}}, - {'m', {NULL, PYMEM_FUNCS}}, - {'o', {NULL, PYOBJ_FUNCS}} + {'r', PYRAW_ALLOC}, + {'m', PYMEM_ALLOC}, + {'o', PYOBJ_ALLOC} }; -#define PYRAWDBG_FUNCS \ - _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree -#define PYDBG_FUNCS \ - _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree - +#define PYDBGRAW_ALLOC \ + {&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree} +#define PYDBGMEM_ALLOC \ + {&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} +#define PYDBGOBJ_ALLOC \ + {&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} -#define _PyMem_Raw _PyRuntime.mem.allocators.raw -static const PyMemAllocatorEx _pymem_raw = { #ifdef Py_DEBUG - &_PyMem_Debug.raw, PYRAWDBG_FUNCS +static PyMemAllocatorEx _PyMem_Raw = PYDBGRAW_ALLOC; +static PyMemAllocatorEx _PyMem = PYDBGMEM_ALLOC; +static PyMemAllocatorEx _PyObject = PYDBGOBJ_ALLOC; #else - NULL, PYRAW_FUNCS +static PyMemAllocatorEx _PyMem_Raw = PYRAW_ALLOC; +static PyMemAllocatorEx _PyMem = PYMEM_ALLOC; +static PyMemAllocatorEx _PyObject = PYOBJ_ALLOC; #endif - }; -#define _PyMem _PyRuntime.mem.allocators.mem -static const PyMemAllocatorEx _pymem = { -#ifdef Py_DEBUG - &_PyMem_Debug.mem, PYDBG_FUNCS -#else - NULL, PYMEM_FUNCS -#endif - }; -#define _PyObject _PyRuntime.mem.allocators.obj -static const PyMemAllocatorEx _pyobject = { +static int +pymem_set_default_allocator(PyMemAllocatorDomain domain, int debug, + PyMemAllocatorEx *old_alloc) +{ + if (old_alloc != NULL) { + PyMem_GetAllocator(domain, old_alloc); + } + + + PyMemAllocatorEx new_alloc; + switch(domain) + { + case PYMEM_DOMAIN_RAW: + new_alloc = (PyMemAllocatorEx)PYRAW_ALLOC; + break; + case PYMEM_DOMAIN_MEM: + new_alloc = (PyMemAllocatorEx)PYMEM_ALLOC; + break; + case PYMEM_DOMAIN_OBJ: + new_alloc = (PyMemAllocatorEx)PYOBJ_ALLOC; + break; + default: + /* unknown domain */ + return -1; + } + PyMem_SetAllocator(domain, &new_alloc); + if (debug) { + _PyMem_SetupDebugHooksDomain(domain); + } + return 0; +} + + +int +_PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain, + PyMemAllocatorEx *old_alloc) +{ #ifdef Py_DEBUG - &_PyMem_Debug.obj, PYDBG_FUNCS + const int debug = 1; #else - NULL, PYOBJ_FUNCS + const int debug = 0; #endif - }; + return pymem_set_default_allocator(domain, debug, old_alloc); +} + int _PyMem_SetupAllocators(const char *opt) { if (opt == NULL || *opt == '\0') { /* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line - options): use default allocators */ -#ifdef Py_DEBUG -# ifdef WITH_PYMALLOC - opt = "pymalloc_debug"; -# else - opt = "malloc_debug"; -# endif -#else - /* !Py_DEBUG */ -# ifdef WITH_PYMALLOC - opt = "pymalloc"; -# else - opt = "malloc"; -# endif -#endif + options): use default memory allocators */ + opt = "default"; } - if (strcmp(opt, "debug") == 0) { - PyMem_SetupDebugHooks(); + if (strcmp(opt, "default") == 0) { + (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL); + (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_MEM, NULL); + (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_OBJ, NULL); } - else if (strcmp(opt, "malloc") == 0 || strcmp(opt, "malloc_debug") == 0) - { - PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); - - if (strcmp(opt, "malloc_debug") == 0) - PyMem_SetupDebugHooks(); + else if (strcmp(opt, "debug") == 0) { + (void)pymem_set_default_allocator(PYMEM_DOMAIN_RAW, 1, NULL); + (void)pymem_set_default_allocator(PYMEM_DOMAIN_MEM, 1, NULL); + (void)pymem_set_default_allocator(PYMEM_DOMAIN_OBJ, 1, NULL); } #ifdef WITH_PYMALLOC - else if (strcmp(opt, "pymalloc") == 0 - || strcmp(opt, "pymalloc_debug") == 0) - { - PyMemAllocatorEx raw_alloc = {NULL, PYRAW_FUNCS}; - PyMemAllocatorEx mem_alloc = {NULL, PYMEM_FUNCS}; - PyMemAllocatorEx obj_alloc = {NULL, PYOBJ_FUNCS}; + else if (strcmp(opt, "pymalloc") == 0 || strcmp(opt, "pymalloc_debug") == 0) { + PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &raw_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &mem_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &obj_alloc); + PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC; + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &pymalloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &pymalloc); - if (strcmp(opt, "pymalloc_debug") == 0) + if (strcmp(opt, "pymalloc_debug") == 0) { PyMem_SetupDebugHooks(); + } } #endif + else if (strcmp(opt, "malloc") == 0 || strcmp(opt, "malloc_debug") == 0) { + PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &malloc_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &malloc_alloc); + + if (strcmp(opt, "malloc_debug") == 0) { + PyMem_SetupDebugHooks(); + } + } else { /* unknown allocator */ return -1; @@ -267,13 +293,76 @@ _PyMem_SetupAllocators(const char *opt) return 0; } -#undef PYRAW_FUNCS -#undef PYMEM_FUNCS -#undef PYOBJ_FUNCS -#undef PYRAWDBG_FUNCS -#undef PYDBG_FUNCS -static const PyObjectArenaAllocator _PyObject_Arena = {NULL, +static int +pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b) +{ + return (memcmp(a, b, sizeof(PyMemAllocatorEx)) == 0); +} + + +const char* +_PyMem_GetAllocatorsName(void) +{ + PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; +#ifdef WITH_PYMALLOC + PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC; +#endif + + if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) && + pymemallocator_eq(&_PyMem, &malloc_alloc) && + pymemallocator_eq(&_PyObject, &malloc_alloc)) + { + return "malloc"; + } +#ifdef WITH_PYMALLOC + if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) && + pymemallocator_eq(&_PyMem, &pymalloc) && + pymemallocator_eq(&_PyObject, &pymalloc)) + { + return "pymalloc"; + } +#endif + + PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC; + PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC; + PyMemAllocatorEx dbg_obj = PYDBGOBJ_ALLOC; + + if (pymemallocator_eq(&_PyMem_Raw, &dbg_raw) && + pymemallocator_eq(&_PyMem, &dbg_mem) && + pymemallocator_eq(&_PyObject, &dbg_obj)) + { + /* Debug hooks installed */ + if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) && + pymemallocator_eq(&_PyMem_Debug.mem.alloc, &malloc_alloc) && + pymemallocator_eq(&_PyMem_Debug.obj.alloc, &malloc_alloc)) + { + return "malloc_debug"; + } +#ifdef WITH_PYMALLOC + if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) && + pymemallocator_eq(&_PyMem_Debug.mem.alloc, &pymalloc) && + pymemallocator_eq(&_PyMem_Debug.obj.alloc, &pymalloc)) + { + return "pymalloc_debug"; + } +#endif + } + return NULL; +} + + +#undef MALLOC_ALLOC +#undef PYMALLOC_ALLOC +#undef PYRAW_ALLOC +#undef PYMEM_ALLOC +#undef PYOBJ_ALLOC +#undef PYDBGRAW_ALLOC +#undef PYDBGMEM_ALLOC +#undef PYDBGOBJ_ALLOC + + +static PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree #elif defined(ARENAS_USE_MMAP) @@ -283,34 +372,6 @@ static const PyObjectArenaAllocator _PyObject_Arena = {NULL, #endif }; -void -_PyObject_Initialize(struct _pyobj_runtime_state *state) -{ - state->allocator_arenas = _PyObject_Arena; -} - -void -_PyMem_Initialize(struct _pymem_runtime_state *state) -{ - state->allocators.raw = _pymem_raw; - state->allocators.mem = _pymem; - state->allocators.obj = _pyobject; - -#ifdef WITH_PYMALLOC - Py_BUILD_ASSERT(NB_SMALL_SIZE_CLASSES == 64); - - for (int i = 0; i < 8; i++) { - for (int j = 0; j < 8; j++) { - int x = i * 8 + j; - poolp *addr = &(state->usedpools[2*(x)]); - poolp val = (poolp)((uint8_t *)addr - 2*sizeof(pyblock *)); - state->usedpools[x * 2] = val; - state->usedpools[x * 2 + 1] = val; - }; - }; -#endif /* WITH_PYMALLOC */ -} - #ifdef WITH_PYMALLOC static int _PyMem_DebugEnabled(void) @@ -330,40 +391,62 @@ _PyMem_PymallocEnabled(void) } #endif -void -PyMem_SetupDebugHooks(void) + +static void +_PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) { PyMemAllocatorEx alloc; - alloc.malloc = _PyMem_DebugRawMalloc; - alloc.calloc = _PyMem_DebugRawCalloc; - alloc.realloc = _PyMem_DebugRawRealloc; - alloc.free = _PyMem_DebugRawFree; + if (domain == PYMEM_DOMAIN_RAW) { + if (_PyMem_Raw.malloc == _PyMem_DebugRawMalloc) { + return; + } - if (_PyMem_Raw.malloc != _PyMem_DebugRawMalloc) { - alloc.ctx = &_PyMem_Debug.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); + alloc.ctx = &_PyMem_Debug.raw; + alloc.malloc = _PyMem_DebugRawMalloc; + alloc.calloc = _PyMem_DebugRawCalloc; + alloc.realloc = _PyMem_DebugRawRealloc; + alloc.free = _PyMem_DebugRawFree; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } + else if (domain == PYMEM_DOMAIN_MEM) { + if (_PyMem.malloc == _PyMem_DebugMalloc) { + return; + } - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; - - if (_PyMem.malloc != _PyMem_DebugMalloc) { - alloc.ctx = &_PyMem_Debug.mem; PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); + alloc.ctx = &_PyMem_Debug.mem; + alloc.malloc = _PyMem_DebugMalloc; + alloc.calloc = _PyMem_DebugCalloc; + alloc.realloc = _PyMem_DebugRealloc; + alloc.free = _PyMem_DebugFree; PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); } + else if (domain == PYMEM_DOMAIN_OBJ) { + if (_PyObject.malloc == _PyMem_DebugMalloc) { + return; + } - if (_PyObject.malloc != _PyMem_DebugMalloc) { - alloc.ctx = &_PyMem_Debug.obj; PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); + alloc.ctx = &_PyMem_Debug.obj; + alloc.malloc = _PyMem_DebugMalloc; + alloc.calloc = _PyMem_DebugCalloc; + alloc.realloc = _PyMem_DebugRealloc; + alloc.free = _PyMem_DebugFree; PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); } } + +void +PyMem_SetupDebugHooks(void) +{ + _PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_RAW); + _PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_MEM); + _PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_OBJ); +} + void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) { @@ -397,13 +480,13 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) { - *allocator = _PyRuntime.obj.allocator_arenas; + *allocator = _PyObject_Arena; } void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { - _PyRuntime.obj.allocator_arenas = *allocator; + _PyObject_Arena = *allocator; } void * @@ -438,8 +521,7 @@ PyMem_RawRealloc(void *ptr, size_t new_size) return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); } -void -PyMem_RawFree(void *ptr) +void PyMem_RawFree(void *ptr) { _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); } @@ -479,16 +561,35 @@ PyMem_Free(void *ptr) } +wchar_t* +_PyMem_RawWcsdup(const wchar_t *str) +{ + assert(str != NULL); + + size_t len = wcslen(str); + if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { + return NULL; + } + + size_t size = (len + 1) * sizeof(wchar_t); + wchar_t *str2 = PyMem_RawMalloc(size); + if (str2 == NULL) { + return NULL; + } + + memcpy(str2, str, size); + return str2; +} + char * _PyMem_RawStrdup(const char *str) { - size_t size; - char *copy; - - size = strlen(str) + 1; - copy = PyMem_RawMalloc(size); - if (copy == NULL) + assert(str != NULL); + size_t size = strlen(str) + 1; + char *copy = PyMem_RawMalloc(size); + if (copy == NULL) { return NULL; + } memcpy(copy, str, size); return copy; } @@ -496,13 +597,12 @@ _PyMem_RawStrdup(const char *str) char * _PyMem_Strdup(const char *str) { - size_t size; - char *copy; - - size = strlen(str) + 1; - copy = PyMem_Malloc(size); - if (copy == NULL) + assert(str != NULL); + size_t size = strlen(str) + 1; + char *copy = PyMem_Malloc(size); + if (copy == NULL) { return NULL; + } memcpy(copy, str, size); return copy; } @@ -559,10 +659,497 @@ static int running_on_valgrind = -1; #endif +/* An object allocator for Python. + + Here is an introduction to the layers of the Python memory architecture, + showing where the object allocator is actually used (layer +2), It is + called for every object allocation and deallocation (PyObject_New/Del), + unless the object-specific allocators implement a proprietary allocation + scheme (ex.: ints use a simple free list). This is also the place where + the cyclic garbage collector operates selectively on container objects. + + + Object-specific allocators + _____ ______ ______ ________ + [ int ] [ dict ] [ list ] ... [ string ] Python core | ++3 | <----- Object-specific memory -----> | <-- Non-object memory --> | + _______________________________ | | + [ Python's object allocator ] | | ++2 | ####### Object memory ####### | <------ Internal buffers ------> | + ______________________________________________________________ | + [ Python's raw memory allocator (PyMem_ API) ] | ++1 | <----- Python memory (under PyMem manager's control) ------> | | + __________________________________________________________________ + [ Underlying general-purpose allocator (ex: C library malloc) ] + 0 | <------ Virtual memory allocated for the python process -------> | + + ========================================================================= + _______________________________________________________________________ + [ OS-specific Virtual Memory Manager (VMM) ] +-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | + __________________________________ __________________________________ + [ ] [ ] +-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | + +*/ +/*==========================================================================*/ + +/* A fast, special-purpose memory allocator for small blocks, to be used + on top of a general-purpose malloc -- heavily based on previous art. */ + +/* Vladimir Marangozov -- August 2000 */ + +/* + * "Memory management is where the rubber meets the road -- if we do the wrong + * thing at any level, the results will not be good. And if we don't make the + * levels work well together, we are in serious trouble." (1) + * + * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, + * "Dynamic Storage Allocation: A Survey and Critical Review", + * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. + */ + +/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ + +/*==========================================================================*/ + +/* + * Allocation strategy abstract: + * + * For small requests, the allocator sub-allocates blocks of memory. + * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the + * system's allocator. + * + * Small requests are grouped in size classes spaced 8 bytes apart, due + * to the required valid alignment of the returned address. Requests of + * a particular size are serviced from memory pools of 4K (one VMM page). + * Pools are fragmented on demand and contain free lists of blocks of one + * particular size class. In other words, there is a fixed-size allocator + * for each size class. Free pools are shared by the different allocators + * thus minimizing the space reserved for a particular size class. + * + * This allocation strategy is a variant of what is known as "simple + * segregated storage based on array of free lists". The main drawback of + * simple segregated storage is that we might end up with lot of reserved + * memory for the different free lists, which degenerate in time. To avoid + * this, we partition each free list in pools and we share dynamically the + * reserved space between all free lists. This technique is quite efficient + * for memory intensive programs which allocate mainly small-sized blocks. + * + * For small requests we have the following table: + * + * Request in bytes Size of allocated block Size class idx + * ---------------------------------------------------------------- + * 1-8 8 0 + * 9-16 16 1 + * 17-24 24 2 + * 25-32 32 3 + * 33-40 40 4 + * 41-48 48 5 + * 49-56 56 6 + * 57-64 64 7 + * 65-72 72 8 + * ... ... ... + * 497-504 504 62 + * 505-512 512 63 + * + * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying + * allocator. + */ + +/*==========================================================================*/ + +/* + * -- Main tunable settings section -- + */ + +/* + * Alignment of addresses returned to the user. 8-bytes alignment works + * on most current architectures (with 32-bit or 64-bit address busses). + * The alignment value is also used for grouping small requests in size + * classes spaced ALIGNMENT bytes apart. + * + * You shouldn't change this unless you know what you are doing. + */ +#define ALIGNMENT 8 /* must be 2^N */ +#define ALIGNMENT_SHIFT 3 + +/* Return the number of bytes in size class I, as a uint. */ +#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) + +/* + * Max size threshold below which malloc requests are considered to be + * small enough in order to use preallocated memory pools. You can tune + * this value according to your application behaviour and memory needs. + * + * Note: a size threshold of 512 guarantees that newly created dictionaries + * will be allocated from preallocated memory pools on 64-bit. + * + * The following invariants must hold: + * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 + * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT + * + * Although not required, for better performance and space efficiency, + * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. + */ +#define SMALL_REQUEST_THRESHOLD 512 +#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) + +/* + * The system's VMM page size can be obtained on most unices with a + * getpagesize() call or deduced from various header files. To make + * things simpler, we assume that it is 4K, which is OK for most systems. + * It is probably better if this is the native page size, but it doesn't + * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page + * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation + * violation fault. 4K is apparently OK for all the platforms that python + * currently targets. + */ +#define SYSTEM_PAGE_SIZE (4 * 1024) +#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) + +/* + * Maximum amount of memory managed by the allocator for small requests. + */ +#ifdef WITH_MEMORY_LIMITS +#ifndef SMALL_MEMORY_LIMIT +#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */ +#endif +#endif + +/* + * The allocator sub-allocates blocks of memory (called arenas) aligned + * on a page boundary. This is a reserved virtual address space for the + * current process (obtained through a malloc()/mmap() call). In no way this + * means that the memory arenas will be used entirely. A malloc() is + * usually an address range reservation for bytes, unless all pages within + * this space are referenced subsequently. So malloc'ing big blocks and not + * using them does not mean "wasting memory". It's an addressable range + * wastage... + * + * Arenas are allocated with mmap() on systems supporting anonymous memory + * mappings to reduce heap fragmentation. + */ +#define ARENA_SIZE (256 << 10) /* 256KB */ + +#ifdef WITH_MEMORY_LIMITS +#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) +#endif + +/* + * Size of the pools used for small blocks. Should be a power of 2, + * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. + */ +#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ +#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK + +/* + * -- End of tunable settings section -- + */ + +/*==========================================================================*/ + +/* + * Locking + * + * To reduce lock contention, it would probably be better to refine the + * crude function locking with per size class locking. I'm not positive + * however, whether it's worth switching to such locking policy because + * of the performance penalty it might introduce. + * + * The following macros describe the simplest (should also be the fastest) + * lock object on a particular platform and the init/fini/lock/unlock + * operations on it. The locks defined here are not expected to be recursive + * because it is assumed that they will always be called in the order: + * INIT, [LOCK, UNLOCK]*, FINI. + */ + +/* + * Python's threads are serialized, so object malloc locking is disabled. + */ +#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ +#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ +#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ +#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ +#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ + +/* When you say memory, my mind reasons in terms of (pointers to) blocks */ +typedef uint8_t block; + +/* Pool for small blocks. */ +struct pool_header { + union { block *_padding; + uint count; } ref; /* number of allocated blocks */ + block *freeblock; /* pool's free list head */ + struct pool_header *nextpool; /* next pool of this size class */ + struct pool_header *prevpool; /* previous pool "" */ + uint arenaindex; /* index into arenas of base adr */ + uint szidx; /* block size class index */ + uint nextoffset; /* bytes to virgin block */ + uint maxnextoffset; /* largest valid nextoffset */ +}; + +typedef struct pool_header *poolp; + +/* Record keeping for arenas. */ +struct arena_object { + /* The address of the arena, as returned by malloc. Note that 0 + * will never be returned by a successful malloc, and is used + * here to mark an arena_object that doesn't correspond to an + * allocated arena. + */ + uintptr_t address; + + /* Pool-aligned pointer to the next pool to be carved off. */ + block* pool_address; + + /* The number of available pools in the arena: free pools + never- + * allocated pools. + */ + uint nfreepools; + + /* The total number of pools in the arena, whether or not available. */ + uint ntotalpools; + + /* Singly-linked list of available pools. */ + struct pool_header* freepools; + + /* Whenever this arena_object is not associated with an allocated + * arena, the nextarena member is used to link all unassociated + * arena_objects in the singly-linked `unused_arena_objects` list. + * The prevarena member is unused in this case. + * + * When this arena_object is associated with an allocated arena + * with at least one available pool, both members are used in the + * doubly-linked `usable_arenas` list, which is maintained in + * increasing order of `nfreepools` values. + * + * Else this arena_object is associated with an allocated arena + * all of whose pools are in use. `nextarena` and `prevarena` + * are both meaningless in this case. + */ + struct arena_object* nextarena; + struct arena_object* prevarena; +}; + +#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) + +#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ + +/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ +#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) + +/* Return total number of blocks in pool of size index I, as a uint. */ +#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) + +/*==========================================================================*/ + +/* + * This malloc lock + */ +SIMPLELOCK_DECL(_malloc_lock) +#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) +#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) +#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) +#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) + +/* + * Pool table -- headed, circular, doubly-linked lists of partially used pools. + +This is involved. For an index i, usedpools[i+i] is the header for a list of +all partially used pools holding small blocks with "size class idx" i. So +usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size +16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to +the start of a singly-linked list of free blocks within the pool. When a +block is freed, it's inserted at the front of its pool's freeblock list. Note +that the available blocks in a pool are *not* linked all together when a pool +is initialized. Instead only "the first two" (lowest addresses) blocks are +set up, returning the first such block, and setting pool->freeblock to a +one-block list holding the second such block. This is consistent with that +pymalloc strives at all levels (arena, pool, and block) never to touch a piece +of memory until it's actually needed. + +So long as a pool is in the used state, we're certain there *is* a block +available for allocating, and pool->freeblock is not NULL. If pool->freeblock +points to the end of the free list before we've carved the entire pool into +blocks, that means we simply haven't yet gotten to one of the higher-address +blocks. The offset from the pool_header to the start of "the next" virgin +block is stored in the pool_header nextoffset member, and the largest value +of nextoffset that makes sense is stored in the maxnextoffset member when a +pool is initialized. All the blocks in a pool have been passed out at least +once when and only when nextoffset > maxnextoffset. + + +Major obscurity: While the usedpools vector is declared to have poolp +entries, it doesn't really. It really contains two pointers per (conceptual) +poolp entry, the nextpool and prevpool members of a pool_header. The +excruciating initialization code below fools C so that + + usedpool[i+i] + +"acts like" a genuine poolp, but only so long as you only reference its +nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is +compensating for that a pool_header's nextpool and prevpool members +immediately follow a pool_header's first two members: + + union { block *_padding; + uint count; } ref; + block *freeblock; + +each of which consume sizeof(block *) bytes. So what usedpools[i+i] really +contains is a fudged-up pointer p such that *if* C believes it's a poolp +pointer, then p->nextpool and p->prevpool are both p (meaning that the headed +circular list is empty). + +It's unclear why the usedpools setup is so convoluted. It could be to +minimize the amount of cache required to hold this heavily-referenced table +(which only *needs* the two interpool pointer members of a pool_header). OTOH, +referencing code has to remember to "double the index" and doing so isn't +free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying +on that C doesn't insert any padding anywhere in a pool_header at or before +the prevpool member. +**************************************************************************** */ + +#define PTA(x) ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *))) +#define PT(x) PTA(x), PTA(x) + +static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = { + PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7) +#if NB_SMALL_SIZE_CLASSES > 8 + , PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15) +#if NB_SMALL_SIZE_CLASSES > 16 + , PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23) +#if NB_SMALL_SIZE_CLASSES > 24 + , PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31) +#if NB_SMALL_SIZE_CLASSES > 32 + , PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39) +#if NB_SMALL_SIZE_CLASSES > 40 + , PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47) +#if NB_SMALL_SIZE_CLASSES > 48 + , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55) +#if NB_SMALL_SIZE_CLASSES > 56 + , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63) +#if NB_SMALL_SIZE_CLASSES > 64 +#error "NB_SMALL_SIZE_CLASSES should be less than 64" +#endif /* NB_SMALL_SIZE_CLASSES > 64 */ +#endif /* NB_SMALL_SIZE_CLASSES > 56 */ +#endif /* NB_SMALL_SIZE_CLASSES > 48 */ +#endif /* NB_SMALL_SIZE_CLASSES > 40 */ +#endif /* NB_SMALL_SIZE_CLASSES > 32 */ +#endif /* NB_SMALL_SIZE_CLASSES > 24 */ +#endif /* NB_SMALL_SIZE_CLASSES > 16 */ +#endif /* NB_SMALL_SIZE_CLASSES > 8 */ +}; + +/*========================================================================== +Arena management. + +`arenas` is a vector of arena_objects. It contains maxarenas entries, some of +which may not be currently used (== they're arena_objects that aren't +currently associated with an allocated arena). Note that arenas proper are +separately malloc'ed. + +Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, +we do try to free() arenas, and use some mild heuristic strategies to increase +the likelihood that arenas eventually can be freed. + +unused_arena_objects + + This is a singly-linked list of the arena_objects that are currently not + being used (no arena is associated with them). Objects are taken off the + head of the list in new_arena(), and are pushed on the head of the list in + PyObject_Free() when the arena is empty. Key invariant: an arena_object + is on this list if and only if its .address member is 0. + +usable_arenas + + This is a doubly-linked list of the arena_objects associated with arenas + that have pools available. These pools are either waiting to be reused, + or have not been used before. The list is sorted to have the most- + allocated arenas first (ascending order based on the nfreepools member). + This means that the next allocation will come from a heavily used arena, + which gives the nearly empty arenas a chance to be returned to the system. + In my unscientific tests this dramatically improved the number of arenas + that could be freed. + +Note that an arena_object associated with an arena all of whose pools are +currently in use isn't on either list. +*/ + +/* Array of objects used to track chunks of memory (arenas). */ +static struct arena_object* arenas = NULL; +/* Number of slots currently allocated in the `arenas` vector. */ +static uint maxarenas = 0; + +/* The head of the singly-linked, NULL-terminated list of available + * arena_objects. + */ +static struct arena_object* unused_arena_objects = NULL; + +/* The head of the doubly-linked, NULL-terminated at each end, list of + * arena_objects associated with arenas that have pools available. + */ +static struct arena_object* usable_arenas = NULL; + +/* How many arena_objects do we initially allocate? + * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the + * `arenas` vector. + */ +#define INITIAL_ARENA_OBJECTS 16 + +/* Number of arenas allocated that haven't been free()'d. */ +static size_t narenas_currently_allocated = 0; + +/* Total number of times malloc() called to allocate an arena. */ +static size_t ntimes_arena_allocated = 0; +/* High water mark (max value ever seen) for narenas_currently_allocated. */ +static size_t narenas_highwater = 0; + +static Py_ssize_t _Py_AllocatedBlocks = 0; + Py_ssize_t _Py_GetAllocatedBlocks(void) { - return _PyRuntime.mem.num_allocated_blocks; + return _Py_AllocatedBlocks; } @@ -586,7 +1173,7 @@ new_arena(void) if (debug_stats) _PyObject_DebugMallocStats(stderr); - if (_PyRuntime.mem.unused_arena_objects == NULL) { + if (unused_arena_objects == NULL) { uint i; uint numarenas; size_t nbytes; @@ -594,18 +1181,18 @@ new_arena(void) /* Double the number of arena objects on each allocation. * Note that it's possible for `numarenas` to overflow. */ - numarenas = _PyRuntime.mem.maxarenas ? _PyRuntime.mem.maxarenas << 1 : INITIAL_ARENA_OBJECTS; - if (numarenas <= _PyRuntime.mem.maxarenas) + numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS; + if (numarenas <= maxarenas) return NULL; /* overflow */ #if SIZEOF_SIZE_T <= SIZEOF_INT - if (numarenas > SIZE_MAX / sizeof(*_PyRuntime.mem.arenas)) + if (numarenas > SIZE_MAX / sizeof(*arenas)) return NULL; /* overflow */ #endif - nbytes = numarenas * sizeof(*_PyRuntime.mem.arenas); - arenaobj = (struct arena_object *)PyMem_RawRealloc(_PyRuntime.mem.arenas, nbytes); + nbytes = numarenas * sizeof(*arenas); + arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes); if (arenaobj == NULL) return NULL; - _PyRuntime.mem.arenas = arenaobj; + arenas = arenaobj; /* We might need to fix pointers that were copied. However, * new_arena only gets called when all the pages in the @@ -613,45 +1200,45 @@ new_arena(void) * into the old array. Thus, we don't have to worry about * invalid pointers. Just to be sure, some asserts: */ - assert(_PyRuntime.mem.usable_arenas == NULL); - assert(_PyRuntime.mem.unused_arena_objects == NULL); + assert(usable_arenas == NULL); + assert(unused_arena_objects == NULL); /* Put the new arenas on the unused_arena_objects list. */ - for (i = _PyRuntime.mem.maxarenas; i < numarenas; ++i) { - _PyRuntime.mem.arenas[i].address = 0; /* mark as unassociated */ - _PyRuntime.mem.arenas[i].nextarena = i < numarenas - 1 ? - &_PyRuntime.mem.arenas[i+1] : NULL; + for (i = maxarenas; i < numarenas; ++i) { + arenas[i].address = 0; /* mark as unassociated */ + arenas[i].nextarena = i < numarenas - 1 ? + &arenas[i+1] : NULL; } /* Update globals. */ - _PyRuntime.mem.unused_arena_objects = &_PyRuntime.mem.arenas[_PyRuntime.mem.maxarenas]; - _PyRuntime.mem.maxarenas = numarenas; + unused_arena_objects = &arenas[maxarenas]; + maxarenas = numarenas; } /* Take the next available arena object off the head of the list. */ - assert(_PyRuntime.mem.unused_arena_objects != NULL); - arenaobj = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = arenaobj->nextarena; + assert(unused_arena_objects != NULL); + arenaobj = unused_arena_objects; + unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); - address = _PyRuntime.obj.allocator_arenas.alloc(_PyRuntime.obj.allocator_arenas.ctx, ARENA_SIZE); + address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); if (address == NULL) { /* The allocation failed: return NULL after putting the * arenaobj back. */ - arenaobj->nextarena = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = arenaobj; + arenaobj->nextarena = unused_arena_objects; + unused_arena_objects = arenaobj; return NULL; } arenaobj->address = (uintptr_t)address; - ++_PyRuntime.mem.narenas_currently_allocated; - ++_PyRuntime.mem.ntimes_arena_allocated; - if (_PyRuntime.mem.narenas_currently_allocated > _PyRuntime.mem.narenas_highwater) - _PyRuntime.mem.narenas_highwater = _PyRuntime.mem.narenas_currently_allocated; + ++narenas_currently_allocated; + ++ntimes_arena_allocated; + if (narenas_currently_allocated > narenas_highwater) + narenas_highwater = narenas_currently_allocated; arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ - arenaobj->pool_address = (pyblock*)arenaobj->address; + arenaobj->pool_address = (block*)arenaobj->address; arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); excess = (uint)(arenaobj->address & POOL_SIZE_MASK); @@ -749,9 +1336,9 @@ address_in_range(void *p, poolp pool) // the GIL. The following dance forces the compiler to read pool->arenaindex // only once. uint arenaindex = *((volatile uint *)&pool->arenaindex); - return arenaindex < _PyRuntime.mem.maxarenas && - (uintptr_t)p - _PyRuntime.mem.arenas[arenaindex].address < ARENA_SIZE && - _PyRuntime.mem.arenas[arenaindex].address != 0; + return arenaindex < maxarenas && + (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE && + arenas[arenaindex].address != 0; } @@ -773,7 +1360,7 @@ address_in_range(void *p, poolp pool) static int pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) { - pyblock *bp; + block *bp; poolp pool; poolp next; uint size; @@ -799,7 +1386,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * Most frequent paths first */ size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; - pool = _PyRuntime.mem.usedpools[size + size]; + pool = usedpools[size + size]; if (pool != pool->nextpool) { /* * There is a used pool for this size class. @@ -808,7 +1395,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) ++pool->ref.count; bp = pool->freeblock; assert(bp != NULL); - if ((pool->freeblock = *(pyblock **)bp) != NULL) { + if ((pool->freeblock = *(block **)bp) != NULL) { goto success; } @@ -817,10 +1404,10 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ if (pool->nextoffset <= pool->maxnextoffset) { /* There is room for another block. */ - pool->freeblock = (pyblock*)pool + + pool->freeblock = (block*)pool + pool->nextoffset; pool->nextoffset += INDEX2SIZE(size); - *(pyblock **)(pool->freeblock) = NULL; + *(block **)(pool->freeblock) = NULL; goto success; } @@ -835,27 +1422,27 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) /* There isn't a pool of the right size class immediately * available: use a free pool. */ - if (_PyRuntime.mem.usable_arenas == NULL) { + if (usable_arenas == NULL) { /* No arena has a free pool: allocate a new arena. */ #ifdef WITH_MEMORY_LIMITS - if (_PyRuntime.mem.narenas_currently_allocated >= MAX_ARENAS) { + if (narenas_currently_allocated >= MAX_ARENAS) { goto failed; } #endif - _PyRuntime.mem.usable_arenas = new_arena(); - if (_PyRuntime.mem.usable_arenas == NULL) { + usable_arenas = new_arena(); + if (usable_arenas == NULL) { goto failed; } - _PyRuntime.mem.usable_arenas->nextarena = - _PyRuntime.mem.usable_arenas->prevarena = NULL; + usable_arenas->nextarena = + usable_arenas->prevarena = NULL; } - assert(_PyRuntime.mem.usable_arenas->address != 0); + assert(usable_arenas->address != 0); /* Try to get a cached free pool. */ - pool = _PyRuntime.mem.usable_arenas->freepools; + pool = usable_arenas->freepools; if (pool != NULL) { /* Unlink from cached pools. */ - _PyRuntime.mem.usable_arenas->freepools = pool->nextpool; + usable_arenas->freepools = pool->nextpool; /* This arena already had the smallest nfreepools * value, so decreasing nfreepools doesn't change @@ -864,18 +1451,18 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * become wholly allocated, we need to remove its * arena_object from usable_arenas. */ - --_PyRuntime.mem.usable_arenas->nfreepools; - if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { + --usable_arenas->nfreepools; + if (usable_arenas->nfreepools == 0) { /* Wholly allocated: remove. */ - assert(_PyRuntime.mem.usable_arenas->freepools == NULL); - assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || - _PyRuntime.mem.usable_arenas->nextarena->prevarena == - _PyRuntime.mem.usable_arenas); - - _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; - if (_PyRuntime.mem.usable_arenas != NULL) { - _PyRuntime.mem.usable_arenas->prevarena = NULL; - assert(_PyRuntime.mem.usable_arenas->address != 0); + assert(usable_arenas->freepools == NULL); + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); + + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); } } else { @@ -884,15 +1471,15 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) * off all the arena's pools for the first * time. */ - assert(_PyRuntime.mem.usable_arenas->freepools != NULL || - _PyRuntime.mem.usable_arenas->pool_address <= - (pyblock*)_PyRuntime.mem.usable_arenas->address + + assert(usable_arenas->freepools != NULL || + usable_arenas->pool_address <= + (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); } init_pool: /* Frontlink to used pools. */ - next = _PyRuntime.mem.usedpools[size + size]; /* == prev */ + next = usedpools[size + size]; /* == prev */ pool->nextpool = next; pool->prevpool = next; next->nextpool = pool; @@ -905,7 +1492,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ bp = pool->freeblock; assert(bp != NULL); - pool->freeblock = *(pyblock **)bp; + pool->freeblock = *(block **)bp; goto success; } /* @@ -915,35 +1502,35 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes) */ pool->szidx = size; size = INDEX2SIZE(size); - bp = (pyblock *)pool + POOL_OVERHEAD; + bp = (block *)pool + POOL_OVERHEAD; pool->nextoffset = POOL_OVERHEAD + (size << 1); pool->maxnextoffset = POOL_SIZE - size; pool->freeblock = bp + size; - *(pyblock **)(pool->freeblock) = NULL; + *(block **)(pool->freeblock) = NULL; goto success; } /* Carve off a new pool. */ - assert(_PyRuntime.mem.usable_arenas->nfreepools > 0); - assert(_PyRuntime.mem.usable_arenas->freepools == NULL); - pool = (poolp)_PyRuntime.mem.usable_arenas->pool_address; - assert((pyblock*)pool <= (pyblock*)_PyRuntime.mem.usable_arenas->address + + assert(usable_arenas->nfreepools > 0); + assert(usable_arenas->freepools == NULL); + pool = (poolp)usable_arenas->pool_address; + assert((block*)pool <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE); - pool->arenaindex = (uint)(_PyRuntime.mem.usable_arenas - _PyRuntime.mem.arenas); - assert(&_PyRuntime.mem.arenas[pool->arenaindex] == _PyRuntime.mem.usable_arenas); + pool->arenaindex = (uint)(usable_arenas - arenas); + assert(&arenas[pool->arenaindex] == usable_arenas); pool->szidx = DUMMY_SIZE_IDX; - _PyRuntime.mem.usable_arenas->pool_address += POOL_SIZE; - --_PyRuntime.mem.usable_arenas->nfreepools; + usable_arenas->pool_address += POOL_SIZE; + --usable_arenas->nfreepools; - if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { - assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || - _PyRuntime.mem.usable_arenas->nextarena->prevarena == - _PyRuntime.mem.usable_arenas); + if (usable_arenas->nfreepools == 0) { + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); /* Unlink the arena: it is completely allocated. */ - _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; - if (_PyRuntime.mem.usable_arenas != NULL) { - _PyRuntime.mem.usable_arenas->prevarena = NULL; - assert(_PyRuntime.mem.usable_arenas->address != 0); + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); } } @@ -966,13 +1553,13 @@ _PyObject_Malloc(void *ctx, size_t nbytes) { void* ptr; if (pymalloc_alloc(ctx, &ptr, nbytes)) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; return ptr; } ptr = PyMem_RawMalloc(nbytes); if (ptr != NULL) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; } return ptr; } @@ -988,13 +1575,13 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize) if (pymalloc_alloc(ctx, &ptr, nbytes)) { memset(ptr, 0, nbytes); - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; return ptr; } ptr = PyMem_RawCalloc(nelem, elsize); if (ptr != NULL) { - _PyRuntime.mem.num_allocated_blocks++; + _Py_AllocatedBlocks++; } return ptr; } @@ -1007,7 +1594,7 @@ static int pymalloc_free(void *ctx, void *p) { poolp pool; - pyblock *lastfree; + block *lastfree; poolp next, prev; uint size; @@ -1034,8 +1621,8 @@ pymalloc_free(void *ctx, void *p) * list in any case). */ assert(pool->ref.count > 0); /* else it was empty */ - *(pyblock **)p = lastfree = pool->freeblock; - pool->freeblock = (pyblock *)p; + *(block **)p = lastfree = pool->freeblock; + pool->freeblock = (block *)p; if (!lastfree) { /* Pool was full, so doesn't currently live in any list: * link it to the front of the appropriate usedpools[] list. @@ -1046,7 +1633,7 @@ pymalloc_free(void *ctx, void *p) --pool->ref.count; assert(pool->ref.count > 0); /* else the pool is empty */ size = pool->szidx; - next = _PyRuntime.mem.usedpools[size + size]; + next = usedpools[size + size]; prev = next->prevpool; /* insert pool before next: prev <-> pool <-> next */ @@ -1080,7 +1667,7 @@ pymalloc_free(void *ctx, void *p) /* Link the pool to freepools. This is a singly-linked * list, and pool->prevpool isn't used there. */ - ao = &_PyRuntime.mem.arenas[pool->arenaindex]; + ao = &arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; nf = ++ao->nfreepools; @@ -1109,9 +1696,9 @@ pymalloc_free(void *ctx, void *p) * usable_arenas pointer. */ if (ao->prevarena == NULL) { - _PyRuntime.mem.usable_arenas = ao->nextarena; - assert(_PyRuntime.mem.usable_arenas == NULL || - _PyRuntime.mem.usable_arenas->address != 0); + usable_arenas = ao->nextarena; + assert(usable_arenas == NULL || + usable_arenas->address != 0); } else { assert(ao->prevarena->nextarena == ao); @@ -1127,14 +1714,14 @@ pymalloc_free(void *ctx, void *p) /* Record that this arena_object slot is * available to be reused. */ - ao->nextarena = _PyRuntime.mem.unused_arena_objects; - _PyRuntime.mem.unused_arena_objects = ao; + ao->nextarena = unused_arena_objects; + unused_arena_objects = ao; /* Free the entire arena. */ - _PyRuntime.obj.allocator_arenas.free(_PyRuntime.obj.allocator_arenas.ctx, + _PyObject_Arena.free(_PyObject_Arena.ctx, (void *)ao->address, ARENA_SIZE); ao->address = 0; /* mark unassociated */ - --_PyRuntime.mem.narenas_currently_allocated; + --narenas_currently_allocated; goto success; } @@ -1145,12 +1732,12 @@ pymalloc_free(void *ctx, void *p) * ao->nfreepools was 0 before, ao isn't * currently on the usable_arenas list. */ - ao->nextarena = _PyRuntime.mem.usable_arenas; + ao->nextarena = usable_arenas; ao->prevarena = NULL; - if (_PyRuntime.mem.usable_arenas) - _PyRuntime.mem.usable_arenas->prevarena = ao; - _PyRuntime.mem.usable_arenas = ao; - assert(_PyRuntime.mem.usable_arenas->address != 0); + if (usable_arenas) + usable_arenas->prevarena = ao; + usable_arenas = ao; + assert(usable_arenas->address != 0); goto success; } @@ -1179,8 +1766,8 @@ pymalloc_free(void *ctx, void *p) } else { /* ao is at the head of the list */ - assert(_PyRuntime.mem.usable_arenas == ao); - _PyRuntime.mem.usable_arenas = ao->nextarena; + assert(usable_arenas == ao); + usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; @@ -1205,7 +1792,7 @@ pymalloc_free(void *ctx, void *p) assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools); assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao); - assert((_PyRuntime.mem.usable_arenas == ao && ao->prevarena == NULL) + assert((usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao); goto success; @@ -1224,7 +1811,7 @@ _PyObject_Free(void *ctx, void *p) return; } - _PyRuntime.mem.num_allocated_blocks--; + _Py_AllocatedBlocks--; if (!pymalloc_free(ctx, p)) { /* pymalloc didn't allocate this address */ PyMem_RawFree(p); @@ -1349,13 +1936,15 @@ _Py_GetAllocatedBlocks(void) #define DEADBYTE 0xDB /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ +static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ + /* serialno is always incremented via calling this routine. The point is * to supply a single place to set a breakpoint. */ static void bumpserialno(void) { - ++_PyRuntime.mem.serialno; + ++serialno; } #define SST SIZEOF_SIZE_T @@ -1462,7 +2051,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, _PyRuntime.mem.serialno); + write_size_t(tail + SST, serialno); return data; } @@ -1522,7 +2111,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; - size_t serialno; + size_t block_serialno; #define ERASED_SIZE 64 uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ @@ -1538,7 +2127,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) total = nbytes + 4*SST; tail = data + original_nbytes; - serialno = read_size_t(tail + SST); + block_serialno = read_size_t(tail + SST); /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ @@ -1561,7 +2150,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) else { head = r; bumpserialno(); - serialno = _PyRuntime.mem.serialno; + block_serialno = serialno; } write_size_t(head, nbytes); @@ -1571,7 +2160,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, serialno); + write_size_t(tail + SST, block_serialno); /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { @@ -1646,7 +2235,7 @@ _PyMem_DebugCheckAddress(char api, const void *p) { const uint8_t *q = (const uint8_t *)p; char msgbuf[64]; - char *msg; + const char *msg; size_t nbytes; const uint8_t *tail; int i; @@ -1661,7 +2250,7 @@ _PyMem_DebugCheckAddress(char api, const void *p) id = (char)q[-SST]; if (id != api) { msg = msgbuf; - snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api); + snprintf(msgbuf, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api); msgbuf[sizeof(msgbuf)-1] = 0; goto error; } @@ -1919,16 +2508,16 @@ _PyObject_DebugMallocStats(FILE *out) * to march over all the arenas. If we're lucky, most of the memory * will be living in full pools -- would be a shame to miss them. */ - for (i = 0; i < _PyRuntime.mem.maxarenas; ++i) { + for (i = 0; i < maxarenas; ++i) { uint j; - uintptr_t base = _PyRuntime.mem.arenas[i].address; + uintptr_t base = arenas[i].address; /* Skip arenas which are not allocated. */ - if (_PyRuntime.mem.arenas[i].address == (uintptr_t)NULL) + if (arenas[i].address == (uintptr_t)NULL) continue; narenas += 1; - numfreepools += _PyRuntime.mem.arenas[i].nfreepools; + numfreepools += arenas[i].nfreepools; /* round up to pool alignment */ if (base & (uintptr_t)POOL_SIZE_MASK) { @@ -1938,8 +2527,8 @@ _PyObject_DebugMallocStats(FILE *out) } /* visit every pool in the arena */ - assert(base <= (uintptr_t) _PyRuntime.mem.arenas[i].pool_address); - for (j = 0; base < (uintptr_t) _PyRuntime.mem.arenas[i].pool_address; + assert(base <= (uintptr_t) arenas[i].pool_address); + for (j = 0; base < (uintptr_t) arenas[i].pool_address; ++j, base += POOL_SIZE) { poolp p = (poolp)base; const uint sz = p->szidx; @@ -1948,7 +2537,7 @@ _PyObject_DebugMallocStats(FILE *out) if (p->ref.count == 0) { /* currently unused */ #ifdef Py_DEBUG - assert(pool_is_in_list(p, _PyRuntime.mem.arenas[i].freepools)); + assert(pool_is_in_list(p, arenas[i].freepools)); #endif continue; } @@ -1958,11 +2547,11 @@ _PyObject_DebugMallocStats(FILE *out) numfreeblocks[sz] += freeblocks; #ifdef Py_DEBUG if (freeblocks > 0) - assert(pool_is_in_list(p, _PyRuntime.mem.usedpools[sz + sz])); + assert(pool_is_in_list(p, usedpools[sz + sz])); #endif } } - assert(narenas == _PyRuntime.mem.narenas_currently_allocated); + assert(narenas == narenas_currently_allocated); fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" @@ -1990,10 +2579,10 @@ _PyObject_DebugMallocStats(FILE *out) } fputc('\n', out); if (_PyMem_DebugEnabled()) - (void)printone(out, "# times object malloc called", _PyRuntime.mem.serialno); - (void)printone(out, "# arenas allocated total", _PyRuntime.mem.ntimes_arena_allocated); - (void)printone(out, "# arenas reclaimed", _PyRuntime.mem.ntimes_arena_allocated - narenas); - (void)printone(out, "# arenas highwater mark", _PyRuntime.mem.narenas_highwater); + (void)printone(out, "# times object malloc called", serialno); + (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", narenas_highwater); (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), diff --git a/Objects/odictobject.c b/Objects/odictobject.c index afacb36f6b3b55..218aa2c5d9ce03 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1625,7 +1625,7 @@ odict_init(PyObject *self, PyObject *args, PyObject *kwds) if (len == -1) return -1; if (len > 1) { - char *msg = "expected at most 1 arguments, got %d"; + const char *msg = "expected at most 1 arguments, got %d"; PyErr_Format(PyExc_TypeError, msg, len); return -1; } @@ -2337,21 +2337,18 @@ mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) assert(args == NULL || PyTuple_Check(args)); len = (args != NULL) ? PyTuple_GET_SIZE(args) : 0; if (len > 1) { - char *msg = "update() takes at most 1 positional argument (%d given)"; + const char *msg = "update() takes at most 1 positional argument (%d given)"; PyErr_Format(PyExc_TypeError, msg, len); return NULL; } if (len) { + PyObject *func; PyObject *other = PyTuple_GET_ITEM(args, 0); /* borrowed reference */ assert(other != NULL); Py_INCREF(other); - if PyDict_CheckExact(other) { - PyObject *items; - if (PyDict_CheckExact(other)) - items = PyDict_Items(other); - else - items = _PyObject_CallMethodId(other, &PyId_items, NULL); + if (PyDict_CheckExact(other)) { + PyObject *items = PyDict_Items(other); Py_DECREF(other); if (items == NULL) return NULL; @@ -2359,10 +2356,14 @@ mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) Py_DECREF(items); if (res == -1) return NULL; + goto handle_kwargs; } - else if (_PyObject_HasAttrId(other, &PyId_keys)) { /* never fails */ + + func = _PyObject_GetAttrId(other, &PyId_keys); + if (func != NULL) { PyObject *keys, *iterator, *key; - keys = _PyObject_CallMethodIdObjArgs(other, &PyId_keys, NULL); + keys = _PyObject_CallNoArg(func); + Py_DECREF(func); if (keys == NULL) { Py_DECREF(other); return NULL; @@ -2388,29 +2389,45 @@ mutablemapping_update(PyObject *self, PyObject *args, PyObject *kwargs) Py_DECREF(iterator); if (res != 0 || PyErr_Occurred()) return NULL; + goto handle_kwargs; + } + else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(other); + return NULL; + } + else { + PyErr_Clear(); } - else if (_PyObject_HasAttrId(other, &PyId_items)) { /* never fails */ + + func = _PyObject_GetAttrId(other, &PyId_items); + if (func != NULL) { PyObject *items; - if (PyDict_CheckExact(other)) - items = PyDict_Items(other); - else - items = _PyObject_CallMethodId(other, &PyId_items, NULL); Py_DECREF(other); + items = _PyObject_CallNoArg(func); + Py_DECREF(func); if (items == NULL) return NULL; res = mutablemapping_add_pairs(self, items); Py_DECREF(items); if (res == -1) return NULL; + goto handle_kwargs; } - else { - res = mutablemapping_add_pairs(self, other); + else if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { Py_DECREF(other); - if (res != 0) - return NULL; + return NULL; } + else { + PyErr_Clear(); + } + + res = mutablemapping_add_pairs(self, other); + Py_DECREF(other); + if (res != 0) + return NULL; } + handle_kwargs: /* now handle kwargs */ assert(kwargs == NULL || PyDict_Check(kwargs)); if (kwargs != NULL && PyDict_GET_SIZE(kwargs)) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 35decd8bb802f6..3a60946125133f 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -44,15 +44,10 @@ static Py_ssize_t count_tracked = 0; static void show_track(void) { - PyObject *xoptions, *value; - _Py_IDENTIFIER(showalloccount); - - xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return; - value = _PyDict_GetItemId(xoptions, &PyId_showalloccount); - if (value != Py_True) + PyInterpreterState *interp = PyThreadState_GET()->interp; + if (!inter->core_config.show_alloc_count) { return; + } fprintf(stderr, "Tuples created: %" PY_FORMAT_SIZE_T "d\n", count_tracked + count_untracked); @@ -308,10 +303,7 @@ tuplerepr(PyTupleObject *v) goto error; } - if (Py_EnterRecursiveCall(" while getting the repr of a tuple")) - goto error; s = PyObject_Repr(v->ob_item[i]); - Py_LeaveRecursiveCall(); if (s == NULL) goto error; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2a8118b43c5a0a..4713082e19dac4 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2377,6 +2377,15 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds) nbases = 1; } else { + for (i = 0; i < nbases; i++) { + tmp = PyTuple_GET_ITEM(bases, i); + if (!PyType_Check(tmp) && PyObject_GetAttrString(tmp, "__mro_entries__")) { + PyErr_SetString(PyExc_TypeError, + "type() doesn't support MRO entry resolution; " + "use types.new_class()"); + return NULL; + } + } /* Search the bases for the proper metatype to deal with this: */ winner = _PyType_CalculateMetaclass(metatype, bases); if (winner == NULL) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 194c5bcdcf11e2..8d4fea8ede15da 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -840,9 +840,6 @@ ensure_unicode(PyObject *obj) /* --- Unicode Object ----------------------------------------------------- */ -static PyObject * -fixup(PyObject *self, Py_UCS4 (*fixfct)(PyObject *s)); - static inline Py_ssize_t findchar(const void *s, int kind, Py_ssize_t size, Py_UCS4 ch, @@ -2174,19 +2171,6 @@ kind_maxchar_limit(unsigned int kind) } } -static inline Py_UCS4 -align_maxchar(Py_UCS4 maxchar) -{ - if (maxchar <= 127) - return 127; - else if (maxchar <= 255) - return 255; - else if (maxchar <= 65535) - return 65535; - else - return MAX_UNICODE; -} - static PyObject* _PyUnicode_FromUCS1(const Py_UCS1* u, Py_ssize_t size) { @@ -8396,8 +8380,8 @@ charmap_encoding_error( Py_ssize_t collstartpos = *inpos; Py_ssize_t collendpos = *inpos+1; Py_ssize_t collpos; - char *encoding = "charmap"; - char *reason = "character maps to "; + const char *encoding = "charmap"; + const char *reason = "character maps to "; charmapencode_result x; Py_UCS4 ch; int val; @@ -8928,7 +8912,7 @@ _PyUnicode_TranslateCharmap(PyObject *input, /* output buffer */ _PyUnicodeWriter writer; /* error handler */ - char *reason = "character maps to "; + const char *reason = "character maps to "; PyObject *errorHandler = NULL; PyObject *exc = NULL; int ignore; @@ -9062,42 +9046,6 @@ PyUnicode_Translate(PyObject *str, return _PyUnicode_TranslateCharmap(str, mapping, errors); } -static Py_UCS4 -fix_decimal_and_space_to_ascii(PyObject *self) -{ - /* No need to call PyUnicode_READY(self) because this function is only - called as a callback from fixup() which does it already. */ - const Py_ssize_t len = PyUnicode_GET_LENGTH(self); - const int kind = PyUnicode_KIND(self); - void *data = PyUnicode_DATA(self); - Py_UCS4 maxchar = 127, ch, fixed; - int modified = 0; - Py_ssize_t i; - - for (i = 0; i < len; ++i) { - ch = PyUnicode_READ(kind, data, i); - fixed = 0; - if (ch > 127) { - if (Py_UNICODE_ISSPACE(ch)) - fixed = ' '; - else { - const int decimal = Py_UNICODE_TODECIMAL(ch); - if (decimal >= 0) - fixed = '0' + decimal; - } - if (fixed != 0) { - modified = 1; - maxchar = Py_MAX(maxchar, fixed); - PyUnicode_WRITE(kind, data, i, fixed); - } - else - maxchar = Py_MAX(maxchar, ch); - } - } - - return (modified) ? maxchar : 0; -} - PyObject * _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) { @@ -9107,12 +9055,42 @@ _PyUnicode_TransformDecimalAndSpaceToASCII(PyObject *unicode) } if (PyUnicode_READY(unicode) == -1) return NULL; - if (PyUnicode_MAX_CHAR_VALUE(unicode) <= 127) { + if (PyUnicode_IS_ASCII(unicode)) { /* If the string is already ASCII, just return the same string */ Py_INCREF(unicode); return unicode; } - return fixup(unicode, fix_decimal_and_space_to_ascii); + + Py_ssize_t len = PyUnicode_GET_LENGTH(unicode); + PyObject *result = PyUnicode_New(len, 127); + if (result == NULL) { + return NULL; + } + + Py_UCS1 *out = PyUnicode_1BYTE_DATA(result); + int kind = PyUnicode_KIND(unicode); + const void *data = PyUnicode_DATA(unicode); + Py_ssize_t i; + for (i = 0; i < len; ++i) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch < 127) { + out[i] = ch; + } + else if (Py_UNICODE_ISSPACE(ch)) { + out[i] = ' '; + } + else { + int decimal = Py_UNICODE_TODECIMAL(ch); + if (decimal < 0) { + out[i] = '?'; + _PyUnicode_LENGTH(result) = i + 1; + break; + } + out[i] = '0' + decimal; + } + } + + return result; } PyObject * @@ -9588,69 +9566,6 @@ PyUnicode_Tailmatch(PyObject *str, return tailmatch(str, substr, start, end, direction); } -/* Apply fixfct filter to the Unicode object self and return a - reference to the modified object */ - -static PyObject * -fixup(PyObject *self, - Py_UCS4 (*fixfct)(PyObject *s)) -{ - PyObject *u; - Py_UCS4 maxchar_old, maxchar_new = 0; - PyObject *v; - - u = _PyUnicode_Copy(self); - if (u == NULL) - return NULL; - maxchar_old = PyUnicode_MAX_CHAR_VALUE(u); - - /* fix functions return the new maximum character in a string, - if the kind of the resulting unicode object does not change, - everything is fine. Otherwise we need to change the string kind - and re-run the fix function. */ - maxchar_new = fixfct(u); - - if (maxchar_new == 0) { - /* no changes */; - if (PyUnicode_CheckExact(self)) { - Py_DECREF(u); - Py_INCREF(self); - return self; - } - else - return u; - } - - maxchar_new = align_maxchar(maxchar_new); - - if (maxchar_new == maxchar_old) - return u; - - /* In case the maximum character changed, we need to - convert the string to the new category. */ - v = PyUnicode_New(PyUnicode_GET_LENGTH(self), maxchar_new); - if (v == NULL) { - Py_DECREF(u); - return NULL; - } - if (maxchar_new > maxchar_old) { - /* If the maxchar increased so that the kind changed, not all - characters are representable anymore and we need to fix the - string again. This only happens in very few cases. */ - _PyUnicode_FastCopyCharacters(v, 0, - self, 0, PyUnicode_GET_LENGTH(self)); - maxchar_old = fixfct(v); - assert(maxchar_old > 0 && maxchar_old <= maxchar_new); - } - else { - _PyUnicode_FastCopyCharacters(v, 0, - u, 0, PyUnicode_GET_LENGTH(self)); - } - Py_DECREF(u); - assert(_PyUnicode_CheckConsistency(v, 1)); - return v; -} - static PyObject * ascii_upper_or_lower(PyObject *self, int lower) { diff --git a/PC/_msi.c b/PC/_msi.c index df6c881b4ec44f..000d81f139f354 100644 --- a/PC/_msi.c +++ b/PC/_msi.c @@ -315,6 +315,12 @@ msierror(int status) case ERROR_INVALID_PARAMETER: PyErr_SetString(MSIError, "invalid parameter"); return NULL; + case ERROR_OPEN_FAILED: + PyErr_SetString(MSIError, "open failed"); + return NULL; + case ERROR_CREATE_FAILED: + PyErr_SetString(MSIError, "create failed"); + return NULL; default: PyErr_Format(MSIError, "unknown error %x", status); return NULL; @@ -569,6 +575,8 @@ summary_getproperty(msiobj* si, PyObject *args) if (sval != sbuf) free(sval); return result; + case VT_EMPTY: + Py_RETURN_NONE; } PyErr_Format(PyExc_NotImplementedError, "result of type %d", type); return NULL; @@ -723,8 +731,12 @@ view_fetch(msiobj *view, PyObject*args) int status; MSIHANDLE result; - if ((status = MsiViewFetch(view->h, &result)) != ERROR_SUCCESS) + status = MsiViewFetch(view->h, &result); + if (status == ERROR_NO_MORE_ITEMS) { + Py_RETURN_NONE; + } else if (status != ERROR_SUCCESS) { return msierror(status); + } return record_new(result); } diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c index 4f9ef6c2cc4c77..04323ebd170681 100644 --- a/PC/bdist_wininst/install.c +++ b/PC/bdist_wininst/install.c @@ -1614,7 +1614,7 @@ SelectPythonDlgProc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) if (count == 0) { char Buffer[4096]; - char *msg; + const char *msg; if (target_version && target_version[0]) { wsprintf(Buffer, "Python version %s required, which was not found" diff --git a/PC/getpathp.c b/PC/getpathp.c index 9bbc7bf0b27ba5..08ed8ccc83f913 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -80,6 +80,7 @@ #include "Python.h" +#include "internal/pystate.h" #include "osdefs.h" #include @@ -116,14 +117,22 @@ #define LANDMARK L"lib\\os.py" #endif -static wchar_t prefix[MAXPATHLEN+1]; -static wchar_t progpath[MAXPATHLEN+1]; -static wchar_t dllpath[MAXPATHLEN+1]; -static wchar_t *module_search_path = NULL; +typedef struct { + wchar_t *path_env; /* PATH environment variable */ + wchar_t *home; /* PYTHONHOME environment variable */ + /* Registry key "Software\Python\PythonCore\PythonPath" */ + wchar_t *machine_path; /* from HKEY_LOCAL_MACHINE */ + wchar_t *user_path; /* from HKEY_CURRENT_USER */ + wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t zip_path[MAXPATHLEN+1]; +} PyCalculatePath; + + +/* determine if "ch" is a separator character */ static int -is_sep(wchar_t ch) /* determine if "ch" is a separator character */ +is_sep(wchar_t ch) { #ifdef ALTSEP return ch == SEP || ch == ALTSEP; @@ -132,28 +141,31 @@ is_sep(wchar_t ch) /* determine if "ch" is a separator character */ #endif } + /* assumes 'dir' null terminated in bounds. Never writes - beyond existing terminator. -*/ + beyond existing terminator. */ static void reduce(wchar_t *dir) { size_t i = wcsnlen_s(dir, MAXPATHLEN+1); - if (i >= MAXPATHLEN+1) + if (i >= MAXPATHLEN+1) { Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } while (i > 0 && !is_sep(dir[i])) --i; dir[i] = '\0'; } + static int change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) { size_t src_len = wcsnlen_s(src, MAXPATHLEN+1); size_t i = src_len; - if (i >= MAXPATHLEN+1) + if (i >= MAXPATHLEN+1) { Py_FatalError("buffer overflow in getpathp.c's reduce()"); + } while (i > 0 && src[i] != '.' && !is_sep(src[i])) --i; @@ -163,11 +175,13 @@ change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) return -1; } - if (is_sep(src[i])) + if (is_sep(src[i])) { i = src_len; + } if (wcsncpy_s(dest, MAXPATHLEN+1, src, i) || - wcscat_s(dest, MAXPATHLEN+1, ext)) { + wcscat_s(dest, MAXPATHLEN+1, ext)) + { dest[0] = '\0'; return -1; } @@ -175,22 +189,25 @@ change_ext(wchar_t *dest, const wchar_t *src, const wchar_t *ext) return 0; } + static int exists(wchar_t *filename) { return GetFileAttributesW(filename) != 0xFFFFFFFF; } -/* Assumes 'filename' MAXPATHLEN+1 bytes long - - may extend 'filename' by one character. -*/ + +/* Is module -- check for .pyc too. + Assumes 'filename' MAXPATHLEN+1 bytes long - + may extend 'filename' by one character. */ static int -ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc too */ +ismodule(wchar_t *filename, int update_filename) { size_t n; - if (exists(filename)) + if (exists(filename)) { return 1; + } /* Check for the compiled version of prefix. */ n = wcsnlen_s(filename, MAXPATHLEN+1); @@ -199,13 +216,15 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc filename[n] = L'c'; filename[n + 1] = L'\0'; exist = exists(filename); - if (!update_filename) + if (!update_filename) { filename[n] = L'\0'; + } return exist; } return 0; } + /* Add a path component, by appending stuff to buffer. buffer must have at least MAXPATHLEN + 1 bytes allocated, and contain a NUL-terminated string with no more than MAXPATHLEN characters (not counting @@ -217,7 +236,9 @@ ismodule(wchar_t *filename, int update_filename) /* Is module -- check for .pyc */ static int _PathCchCombineEx_Initialized = 0; -typedef HRESULT(__stdcall *PPathCchCombineEx)(PWSTR pszPathOut, size_t cchPathOut, PCWSTR pszPathIn, PCWSTR pszMore, unsigned long dwFlags); +typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, size_t cchPathOut, + PCWSTR pszPathIn, PCWSTR pszMore, + unsigned long dwFlags); static PPathCchCombineEx _PathCchCombineEx; static void @@ -225,28 +246,32 @@ join(wchar_t *buffer, const wchar_t *stuff) { if (_PathCchCombineEx_Initialized == 0) { HMODULE pathapi = LoadLibraryW(L"api-ms-win-core-path-l1-1-0.dll"); - if (pathapi) + if (pathapi) { _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress(pathapi, "PathCchCombineEx"); - else + } + else { _PathCchCombineEx = NULL; + } _PathCchCombineEx_Initialized = 1; } if (_PathCchCombineEx) { - if (FAILED(_PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) + if (FAILED(_PathCchCombineEx(buffer, MAXPATHLEN+1, buffer, stuff, 0))) { Py_FatalError("buffer overflow in getpathp.c's join()"); + } } else { - if (!PathCombineW(buffer, buffer, stuff)) + if (!PathCombineW(buffer, buffer, stuff)) { Py_FatalError("buffer overflow in getpathp.c's join()"); + } } } + /* gotlandmark only called by search_for_prefix, which ensures 'prefix' is null terminated in bounds. join() ensures - 'landmark' can not overflow prefix if too long. -*/ + 'landmark' can not overflow prefix if too long. */ static int -gotlandmark(const wchar_t *landmark) +gotlandmark(wchar_t *prefix, const wchar_t *landmark) { int ok; Py_ssize_t n = wcsnlen_s(prefix, MAXPATHLEN); @@ -257,27 +282,29 @@ gotlandmark(const wchar_t *landmark) return ok; } + /* assumes argv0_path is MAXPATHLEN+1 bytes long, already \0 term'd. - assumption provided by only caller, calculate_path() */ + assumption provided by only caller, calculate_path_impl() */ static int -search_for_prefix(wchar_t *argv0_path, const wchar_t *landmark) +search_for_prefix(wchar_t *prefix, wchar_t *argv0_path, const wchar_t *landmark) { /* Search from argv0_path, until landmark is found */ wcscpy_s(prefix, MAXPATHLEN + 1, argv0_path); do { - if (gotlandmark(landmark)) + if (gotlandmark(prefix, landmark)) { return 1; + } reduce(prefix); } while (prefix[0]); return 0; } + #ifdef Py_ENABLE_SHARED /* a string loaded from the DLL at startup.*/ extern const char *PyWin_DLLVersionString; - /* Load a PYTHONPATH value from the registry. Load from either HKEY_LOCAL_MACHINE or HKEY_CURRENT_USER. @@ -290,7 +317,6 @@ extern const char *PyWin_DLLVersionString; work on Win16, where the buffer sizes werent available in advance. It could be simplied now Win16/Win32s is dead! */ - static wchar_t * getpythonregpath(HKEY keyBase, int skipcore) { @@ -315,7 +341,9 @@ getpythonregpath(HKEY keyBase, int skipcore) sizeof(WCHAR)*(versionLen-1) + sizeof(keySuffix); keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); - if (keyBuf==NULL) goto done; + if (keyBuf==NULL) { + goto done; + } memcpy_s(keyBufPtr, keyBufLen, keyPrefix, sizeof(keyPrefix)-sizeof(WCHAR)); keyBufPtr += Py_ARRAY_LENGTH(keyPrefix) - 1; @@ -329,17 +357,25 @@ getpythonregpath(HKEY keyBase, int skipcore) 0, /* reserved */ KEY_READ, &newKey); - if (rc!=ERROR_SUCCESS) goto done; + if (rc!=ERROR_SUCCESS) { + goto done; + } /* Find out how big our core buffer is, and how many subkeys we have */ rc = RegQueryInfoKey(newKey, NULL, NULL, NULL, &numKeys, NULL, NULL, NULL, NULL, &dataSize, NULL, NULL); - if (rc!=ERROR_SUCCESS) goto done; - if (skipcore) dataSize = 0; /* Only count core ones if we want them! */ + if (rc!=ERROR_SUCCESS) { + goto done; + } + if (skipcore) { + dataSize = 0; /* Only count core ones if we want them! */ + } /* Allocate a temp array of char buffers, so we only need to loop reading the registry once */ ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys ); - if (ppPaths==NULL) goto done; + if (ppPaths==NULL) { + goto done; + } memset(ppPaths, 0, sizeof(WCHAR *) * numKeys); /* Loop over all subkeys, allocating a temp sub-buffer. */ for(index=0;indexdll_path = _PyMem_RawWcsdup(dll_path); + if (config->dll_path == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); +} + + +static _PyInitError +get_program_full_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) +{ + wchar_t program_full_path[MAXPATHLEN+1]; + memset(program_full_path, 0, sizeof(program_full_path)); + + if (GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { + goto done; + } /* If there is no slash in the argv0 path, then we have to * assume python is on the user's $PATH, since there's no @@ -454,12 +514,16 @@ get_progpath(void) * $PATH isn't exported, you lose. */ #ifdef ALTSEP - if (wcschr(prog, SEP) || wcschr(prog, ALTSEP)) + if (wcschr(main_config->program_name, SEP) || + wcschr(main_config->program_name, ALTSEP)) #else - if (wcschr(prog, SEP)) + if (wcschr(main_config->program_name, SEP)) #endif - wcsncpy(progpath, prog, MAXPATHLEN); - else if (path) { + { + wcsncpy(program_full_path, main_config->program_name, MAXPATHLEN); + } + else if (calculate->path_env) { + wchar_t *path = calculate->path_env; while (1) { wchar_t *delim = wcschr(path, DELIM); @@ -467,28 +531,39 @@ get_progpath(void) size_t len = delim - path; /* ensure we can't overwrite buffer */ len = min(MAXPATHLEN,len); - wcsncpy(progpath, path, len); - *(progpath + len) = '\0'; + wcsncpy(program_full_path, path, len); + program_full_path[len] = '\0'; + } + else { + wcsncpy(program_full_path, path, MAXPATHLEN); } - else - wcsncpy(progpath, path, MAXPATHLEN); /* join() is safe for MAXPATHLEN+1 size buffer */ - join(progpath, prog); - if (exists(progpath)) + join(program_full_path, main_config->program_name); + if (exists(program_full_path)) { break; + } if (!delim) { - progpath[0] = '\0'; + program_full_path[0] = '\0'; break; } path = delim + 1; } } - else - progpath[0] = '\0'; + else { + program_full_path[0] = '\0'; + } + +done: + config->program_full_path = _PyMem_RawWcsdup(program_full_path); + if (config->program_full_path == NULL) { + return _Py_INIT_NO_MEMORY(); + } + return _Py_INIT_OK(); } + static int find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) { @@ -502,15 +577,18 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) PyObject * decoded; size_t n; - if (p == NULL) + if (p == NULL) { break; + } n = strlen(p); if (p[n - 1] != '\n') { /* line has overflowed - bail */ break; } - if (p[0] == '#') /* Comment - skip */ + if (p[0] == '#') { + /* Comment - skip */ continue; + } decoded = PyUnicode_DecodeUTF8(buffer, n, "surrogateescape"); if (decoded != NULL) { Py_ssize_t k; @@ -537,12 +615,15 @@ find_env_config_value(FILE * env_file, const wchar_t * key, wchar_t * value) return result; } + static int -read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) +read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path, + int *isolated, int *nosite) { FILE *sp_file = _Py_wfopen(path, L"r"); - if (sp_file == NULL) - return -1; + if (sp_file == NULL) { + return 0; + } wcscpy_s(prefix, MAXPATHLEN+1, path); reduce(prefix); @@ -558,10 +639,12 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) while (!feof(sp_file)) { char line[MAXPATHLEN + 1]; char *p = fgets(line, MAXPATHLEN + 1, sp_file); - if (!p) + if (!p) { break; - if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') + } + if (*p == '\0' || *p == '\r' || *p == '\n' || *p == '#') { continue; + } while (*++p) { if (*p == '\r' || *p == '\n') { *p = '\0'; @@ -611,124 +694,147 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite) PyMem_RawFree(wline); } - module_search_path = buf; - fclose(sp_file); - return 0; + config->module_search_path = buf; + return 1; error: PyMem_RawFree(buf); fclose(sp_file); - return -1; + return 0; } static void -calculate_path(void) +calculate_init(PyCalculatePath *calculate, + const _PyMainInterpreterConfig *main_config) { - wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t *buf; - size_t bufsz; - wchar_t *pythonhome = Py_GetPythonHome(); - wchar_t *envpath = NULL; - - int skiphome, skipdefault; - wchar_t *machinepath = NULL; - wchar_t *userpath = NULL; - wchar_t zip_path[MAXPATHLEN+1]; + calculate->home = main_config->home; + calculate->path_env = _wgetenv(L"PATH"); +} - if (!Py_IgnoreEnvironmentFlag) { - envpath = _wgetenv(L"PYTHONPATH"); - } - get_progpath(); - /* progpath guaranteed \0 terminated in MAXPATH+1 bytes. */ - wcscpy_s(argv0_path, MAXPATHLEN+1, progpath); - reduce(argv0_path); +static int +get_pth_filename(wchar_t *spbuffer, _PyPathConfig *config) +{ + if (config->dll_path[0]) { + if (!change_ext(spbuffer, config->dll_path, L"._pth") && + exists(spbuffer)) + { + return 1; + } + } + if (config->program_full_path[0]) { + if (!change_ext(spbuffer, config->program_full_path, L"._pth") && + exists(spbuffer)) + { + return 1; + } + } + return 0; +} - /* Search for a sys.path file */ - { - wchar_t spbuffer[MAXPATHLEN+1]; - if ((dllpath[0] && !change_ext(spbuffer, dllpath, L"._pth") && exists(spbuffer)) || - (progpath[0] && !change_ext(spbuffer, progpath, L"._pth") && exists(spbuffer))) { +static int +calculate_pth_file(_PyPathConfig *config, wchar_t *prefix) +{ + wchar_t spbuffer[MAXPATHLEN+1]; - if (!read_pth_file(spbuffer, prefix, &Py_IsolatedFlag, &Py_NoSiteFlag)) { - return; - } - } + if (!get_pth_filename(spbuffer, config)) { + return 0; } - /* Search for an environment configuration file, first in the - executable's directory and then in the parent directory. - If found, open it for use when searching for prefixes. - */ + return read_pth_file(config, prefix, spbuffer, + &Py_IsolatedFlag, &Py_NoSiteFlag); +} - { - wchar_t envbuffer[MAXPATHLEN+1]; - wchar_t tmpbuffer[MAXPATHLEN+1]; - const wchar_t *env_cfg = L"pyvenv.cfg"; - FILE * env_file = NULL; - wcscpy_s(envbuffer, MAXPATHLEN+1, argv0_path); +/* Search for an environment configuration file, first in the + executable's directory and then in the parent directory. + If found, open it for use when searching for prefixes. +*/ +static void +calculate_pyvenv_file(PyCalculatePath *calculate) +{ + wchar_t envbuffer[MAXPATHLEN+1]; + const wchar_t *env_cfg = L"pyvenv.cfg"; + + wcscpy_s(envbuffer, MAXPATHLEN+1, calculate->argv0_path); + join(envbuffer, env_cfg); + + FILE *env_file = _Py_wfopen(envbuffer, L"r"); + if (env_file == NULL) { + errno = 0; + reduce(envbuffer); + reduce(envbuffer); join(envbuffer, env_cfg); env_file = _Py_wfopen(envbuffer, L"r"); if (env_file == NULL) { errno = 0; - reduce(envbuffer); - reduce(envbuffer); - join(envbuffer, env_cfg); - env_file = _Py_wfopen(envbuffer, L"r"); - if (env_file == NULL) { - errno = 0; - } - } - if (env_file != NULL) { - /* Look for a 'home' variable and set argv0_path to it, if found */ - if (find_env_config_value(env_file, L"home", tmpbuffer)) { - wcscpy_s(argv0_path, MAXPATHLEN+1, tmpbuffer); - } - fclose(env_file); - env_file = NULL; } } - /* Calculate zip archive path from DLL or exe path */ - change_ext(zip_path, dllpath[0] ? dllpath : progpath, L".zip"); + if (env_file == NULL) { + return; + } - if (pythonhome == NULL || *pythonhome == '\0') { - if (zip_path[0] && exists(zip_path)) { - wcscpy_s(prefix, MAXPATHLEN+1, zip_path); - reduce(prefix); - pythonhome = prefix; - } else if (search_for_prefix(argv0_path, LANDMARK)) - pythonhome = prefix; - else - pythonhome = NULL; + /* Look for a 'home' variable and set argv0_path to it, if found */ + wchar_t tmpbuffer[MAXPATHLEN+1]; + if (find_env_config_value(env_file, L"home", tmpbuffer)) { + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer); } - else - wcscpy_s(prefix, MAXPATHLEN+1, pythonhome); + fclose(env_file); +} - if (envpath && *envpath == '\0') - envpath = NULL; +#define INIT_ERR_BUFFER_OVERFLOW() _Py_INIT_ERR("buffer overflow") - skiphome = pythonhome==NULL ? 0 : 1; + +static void +calculate_home_prefix(PyCalculatePath *calculate, wchar_t *prefix) +{ + if (calculate->home == NULL || *calculate->home == '\0') { + if (calculate->zip_path[0] && exists(calculate->zip_path)) { + wcscpy_s(prefix, MAXPATHLEN+1, calculate->zip_path); + reduce(prefix); + calculate->home = prefix; + } + else if (search_for_prefix(prefix, calculate->argv0_path, LANDMARK)) { + calculate->home = prefix; + } + else { + calculate->home = NULL; + } + } + else { + wcscpy_s(prefix, MAXPATHLEN+1, calculate->home); + } +} + + +static _PyInitError +calculate_module_search_path(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config, + wchar_t *prefix) +{ + int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED - machinepath = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); - userpath = getpythonregpath(HKEY_CURRENT_USER, skiphome); + calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); + calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ - skipdefault = envpath!=NULL || pythonhome!=NULL || \ - machinepath!=NULL || userpath!=NULL; + int skipdefault = (main_config->module_search_path_env != NULL || + calculate->home != NULL || + calculate->machine_path != NULL || + calculate->user_path != NULL); /* We need to construct a path from the following parts. (1) the PYTHONPATH environment variable, if set; (2) for Win32, the zip archive file path; - (3) for Win32, the machinepath and userpath, if set; + (3) for Win32, the machine_path and user_path, if set; (4) the PYTHONPATH config macro, with the leading "." - of each component replaced with pythonhome, if set; + of each component replaced with home, if set; (5) the directory containing the executable (argv0_path). The length calculation calculates #4 first. Extra rules: @@ -737,74 +843,81 @@ calculate_path(void) */ /* Calculate size of return buffer */ - if (pythonhome != NULL) { + size_t bufsz = 0; + if (calculate->home != NULL) { wchar_t *p; bufsz = 1; for (p = PYTHONPATH; *p; p++) { - if (*p == DELIM) + if (*p == DELIM) { bufsz++; /* number of DELIM plus one */ + } } - bufsz *= wcslen(pythonhome); + bufsz *= wcslen(calculate->home); } - else - bufsz = 0; bufsz += wcslen(PYTHONPATH) + 1; - bufsz += wcslen(argv0_path) + 1; - if (userpath) - bufsz += wcslen(userpath) + 1; - if (machinepath) - bufsz += wcslen(machinepath) + 1; - bufsz += wcslen(zip_path) + 1; - if (envpath != NULL) - bufsz += wcslen(envpath) + 1; - - module_search_path = buf = PyMem_RawMalloc(bufsz*sizeof(wchar_t)); + bufsz += wcslen(calculate->argv0_path) + 1; + if (calculate->user_path) { + bufsz += wcslen(calculate->user_path) + 1; + } + if (calculate->machine_path) { + bufsz += wcslen(calculate->machine_path) + 1; + } + bufsz += wcslen(calculate->zip_path) + 1; + if (main_config->module_search_path_env != NULL) { + bufsz += wcslen(main_config->module_search_path_env) + 1; + } + + wchar_t *buf, *start_buf; + buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); if (buf == NULL) { /* We can't exit, so print a warning and limp along */ fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n"); - if (envpath) { + if (main_config->module_search_path_env) { fprintf(stderr, "Using environment $PYTHONPATH.\n"); - module_search_path = envpath; + config->module_search_path = main_config->module_search_path_env; } else { fprintf(stderr, "Using default static path.\n"); - module_search_path = PYTHONPATH; + config->module_search_path = PYTHONPATH; } - PyMem_RawFree(machinepath); - PyMem_RawFree(userpath); - return; + return _Py_INIT_OK(); } + start_buf = buf; - if (envpath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), envpath)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (main_config->module_search_path_env) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), + main_config->module_search_path_env)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } - if (zip_path[0]) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), zip_path)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->zip_path[0]) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->zip_path)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } - if (userpath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), userpath)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->user_path) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->user_path)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; - PyMem_RawFree(userpath); } - if (machinepath) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), machinepath)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (calculate->machine_path) { + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->machine_path)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; - PyMem_RawFree(machinepath); } - if (pythonhome == NULL) { + if (calculate->home == NULL) { if (!skipdefault) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), PYTHONPATH)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (wcscpy_s(buf, bufsz - (buf - start_buf), PYTHONPATH)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); *buf++ = DELIM; } @@ -814,13 +927,16 @@ calculate_path(void) size_t n; for (;;) { q = wcschr(p, DELIM); - if (q == NULL) + if (q == NULL) { n = wcslen(p); - else + } + else { n = q-p; + } if (p[0] == '.' && is_sep(p[1])) { - if (wcscpy_s(buf, bufsz - (buf - module_search_path), pythonhome)) - Py_FatalError("buffer overflow in getpathp.c's calculate_path()"); + if (wcscpy_s(buf, bufsz - (buf - start_buf), calculate->home)) { + return INIT_ERR_BUFFER_OVERFLOW(); + } buf = wcschr(buf, L'\0'); p++; n--; @@ -828,17 +944,19 @@ calculate_path(void) wcsncpy(buf, p, n); buf += n; *buf++ = DELIM; - if (q == NULL) + if (q == NULL) { break; + } p = q+1; } } - if (argv0_path) { - wcscpy(buf, argv0_path); + if (calculate->argv0_path) { + wcscpy(buf, calculate->argv0_path); buf = wcschr(buf, L'\0'); *buf++ = DELIM; } *(buf - 1) = L'\0'; + /* Now to pull one last hack/trick. If sys.prefix is empty, then try and find it somewhere on the paths we calculated. We scan backwards, as our general policy @@ -847,7 +965,7 @@ calculate_path(void) on the path, and that our 'prefix' directory is the parent of that. */ - if (*prefix==L'\0') { + if (prefix[0] == L'\0') { wchar_t lookBuf[MAXPATHLEN+1]; wchar_t *look = buf - 1; /* 'buf' is at the end of the buffer */ while (1) { @@ -857,75 +975,111 @@ calculate_path(void) start of the path in question - even if this is one character before the start of the buffer */ - while (look >= module_search_path && *look != DELIM) + while (look >= start_buf && *look != DELIM) look--; nchars = lookEnd-look; wcsncpy(lookBuf, look+1, nchars); lookBuf[nchars] = L'\0'; /* Up one level to the parent */ reduce(lookBuf); - if (search_for_prefix(lookBuf, LANDMARK)) { + if (search_for_prefix(prefix, lookBuf, LANDMARK)) { break; } /* If we are out of paths to search - give up */ - if (look < module_search_path) + if (look < start_buf) { break; + } look--; } } -} + config->module_search_path = start_buf; + return _Py_INIT_OK(); +} -/* External interface */ -void -Py_SetPath(const wchar_t *path) +static _PyInitError +calculate_path_impl(const _PyMainInterpreterConfig *main_config, + PyCalculatePath *calculate, _PyPathConfig *config) { - if (module_search_path != NULL) { - PyMem_RawFree(module_search_path); - module_search_path = NULL; - } - if (path != NULL) { - extern wchar_t *Py_GetProgramName(void); - wchar_t *prog = Py_GetProgramName(); - wcsncpy(progpath, prog, MAXPATHLEN); - prefix[0] = L'\0'; - module_search_path = PyMem_RawMalloc((wcslen(path) + 1) * sizeof(wchar_t)); - if (module_search_path != NULL) - wcscpy(module_search_path, path); + _PyInitError err; + + err = get_dll_path(calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; } -} -wchar_t * -Py_GetPath(void) -{ - if (!module_search_path) - calculate_path(); - return module_search_path; -} + err = get_program_full_path(main_config, calculate, config); + if (_Py_INIT_FAILED(err)) { + return err; + } -wchar_t * -Py_GetPrefix(void) -{ - if (!module_search_path) - calculate_path(); - return prefix; + /* program_full_path guaranteed \0 terminated in MAXPATH+1 bytes. */ + wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, config->program_full_path); + reduce(calculate->argv0_path); + + wchar_t prefix[MAXPATHLEN+1]; + memset(prefix, 0, sizeof(prefix)); + + /* Search for a sys.path file */ + if (calculate_pth_file(config, prefix)) { + goto done; + } + + calculate_pyvenv_file(calculate); + + /* Calculate zip archive path from DLL or exe path */ + change_ext(calculate->zip_path, + config->dll_path[0] ? config->dll_path : config->program_full_path, + L".zip"); + + calculate_home_prefix(calculate, prefix); + + err = calculate_module_search_path(main_config, calculate, config, prefix); + if (_Py_INIT_FAILED(err)) { + return err; + } + +done: + config->prefix = _PyMem_RawWcsdup(prefix); + if (config->prefix == NULL) { + return _Py_INIT_NO_MEMORY(); + } + + return _Py_INIT_OK(); } -wchar_t * -Py_GetExecPrefix(void) + +static void +calculate_free(PyCalculatePath *calculate) { - return Py_GetPrefix(); + PyMem_RawFree(calculate->machine_path); + PyMem_RawFree(calculate->user_path); } -wchar_t * -Py_GetProgramFullPath(void) + +_PyInitError +_PyPathConfig_Calculate(_PyPathConfig *config, + const _PyMainInterpreterConfig *main_config) { - if (!module_search_path) - calculate_path(); - return progpath; + PyCalculatePath calculate; + memset(&calculate, 0, sizeof(calculate)); + + calculate_init(&calculate, main_config); + + _PyInitError err = calculate_path_impl(main_config, &calculate, config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _Py_INIT_OK(); + +done: + calculate_free(&calculate); + return err; } + /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -935,24 +1089,27 @@ Py_GetProgramFullPath(void) static int python3_checked = 0; static HANDLE hPython3; int -_Py_CheckPython3() +_Py_CheckPython3(void) { wchar_t py3path[MAXPATHLEN+1]; wchar_t *s; - if (python3_checked) + if (python3_checked) { return hPython3 != NULL; + } python3_checked = 1; /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, dllpath); + wcscpy(py3path, _Py_path_config.dll_path); s = wcsrchr(py3path, L'\\'); - if (!s) + if (!s) { s = py3path; + } wcscpy(s, L"\\python3.dll"); hPython3 = LoadLibraryExW(py3path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); - if (hPython3 != NULL) + if (hPython3 != NULL) { return 1; + } /* Check sys.prefix\DLLs\python3.dll */ wcscpy(py3path, Py_GetPrefix()); diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index ab9fb05adead99..423e68c57b27fc 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -57,7 +57,7 @@ - _CONSOLE;%(PreprocessorDefinitions) + Py_BUILD_CORE;_CONSOLE;%(PreprocessorDefinitions) Console @@ -113,4 +113,4 @@ $(_PGOPath) - \ No newline at end of file + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 246de78463d95b..b430e05b629b1c 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -116,7 +116,6 @@ - @@ -382,6 +381,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index b37e9930e3146c..c9aa3da355e99f 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -141,9 +141,6 @@ Include - - Include - Include @@ -899,6 +896,9 @@ Python + + Python + Python @@ -1031,4 +1031,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index a4b7d4930391b4..d28435b4c4d7d9 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -25,6 +25,8 @@ external dependencies. To build, simply run the "build.bat" script without any arguments. After this succeeds, you can open the "pcbuild.sln" solution in Visual Studio to continue development. +To build an installer package, refer to the README in the Tools/msi folder. + The solution currently supports two platforms. The Win32 platform is used to build standard x86-compatible 32-bit binaries, output into the win32 sub-directory. The x64 platform is used for building 64-bit AMD64 diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 0286d6652f5ef3..69583d914f63fe 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -497,18 +497,15 @@ def isSimpleType(self, field): def visitField(self, field, name, sum=None, prod=None, depth=0): ctype = get_c_type(field.type) - if field.opt: - check = "exists_not_none(obj, &PyId_%s)" % (field.name,) + if not field.opt: + self.emit("tmp = _PyObject_GetAttrId(obj, &PyId_%s);" % field.name, depth) else: - check = "_PyObject_HasAttrId(obj, &PyId_%s)" % (field.name,) - self.emit("if (%s) {" % (check,), depth, reflow=False) + self.emit("tmp = get_not_none(obj, &PyId_%s);" % field.name, depth) + self.emit("if (tmp != NULL) {", depth) self.emit("int res;", depth+1) if field.seq: self.emit("Py_ssize_t len;", depth+1) self.emit("Py_ssize_t i;", depth+1) - self.emit("tmp = _PyObject_GetAttrId(obj, &PyId_%s);" % field.name, depth+1) - self.emit("if (tmp == NULL) goto failed;", depth+1) - if field.seq: self.emit("if (!PyList_Check(tmp)) {", depth+1) self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must " "be a list, not a %%.200s\", tmp->ob_type->tp_name);" % @@ -542,13 +539,19 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("if (res != 0) goto failed;", depth+1) self.emit("Py_CLEAR(tmp);", depth+1) - self.emit("} else {", depth) if not field.opt: + self.emit("} else {", depth) + self.emit("if (PyErr_ExceptionMatches(PyExc_AttributeError)) {", depth+1) message = "required field \\\"%s\\\" missing from %s" % (field.name, name) format = "PyErr_SetString(PyExc_TypeError, \"%s\");" - self.emit(format % message, depth+1, reflow=False) + self.emit(format % message, depth+2, reflow=False) + self.emit("}", depth+1) self.emit("return 1;", depth+1) else: + self.emit("} else if (PyErr_Occurred()) {", depth) + self.emit("return 1;", depth+1) + self.emit("} else {", depth) + if self.isNumeric(field): self.emit("%s = 0;" % field.name, depth+1) elif not self.isSimpleType(field): @@ -660,13 +663,17 @@ def visitModule(self, mod): int res = -1; PyObject *key, *value, *fields; fields = _PyObject_GetAttrId((PyObject*)Py_TYPE(self), &PyId__fields); - if (!fields) - PyErr_Clear(); if (fields) { numfields = PySequence_Size(fields); if (numfields == -1) goto cleanup; } + else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + goto cleanup; + } res = 0; /* if no error occurs, this stays 0 to the end */ if (numfields < PyTuple_GET_SIZE(args)) { @@ -958,17 +965,20 @@ def visitModule(self, mod): return 0; } -static int exists_not_none(PyObject *obj, _Py_Identifier *id) +static PyObject *get_not_none(PyObject *obj, _Py_Identifier *id) { - int isnone; PyObject *attr = _PyObject_GetAttrId(obj, id); if (!attr) { - PyErr_Clear(); - return 0; + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + return NULL; + } + else if (attr == Py_None) { + Py_DECREF(attr); + return NULL; } - isnone = attr == Py_None; - Py_DECREF(attr); - return !isnone; + return attr; } """, 0, reflow=False) diff --git a/Parser/parser.h b/Parser/parser.h index 403236d1ea2b87..39df9487285c84 100644 --- a/Parser/parser.h +++ b/Parser/parser.h @@ -10,23 +10,23 @@ extern "C" { #define MAXSTACK 1500 typedef struct { - int s_state; /* State in current DFA */ - dfa *s_dfa; /* Current DFA */ - struct _node *s_parent; /* Where to add next node */ + int s_state; /* State in current DFA */ + dfa *s_dfa; /* Current DFA */ + struct _node *s_parent; /* Where to add next node */ } stackentry; typedef struct { - stackentry *s_top; /* Top entry */ - stackentry s_base[MAXSTACK];/* Array of stack entries */ - /* NB The stack grows down */ + stackentry *s_top; /* Top entry */ + stackentry s_base[MAXSTACK];/* Array of stack entries */ + /* NB The stack grows down */ } stack; typedef struct { - stack p_stack; /* Stack of parser states */ - grammar *p_grammar; /* Grammar to use */ - node *p_tree; /* Top of parse tree */ + stack p_stack; /* Stack of parser states */ + grammar *p_grammar; /* Grammar to use */ + node *p_tree; /* Top of parse tree */ #ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD - unsigned long p_flags; /* see co_flags in Include/code.h */ + unsigned long p_flags; /* see co_flags in Include/code.h */ #endif } parser_state; diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 8f3e2f7c4ee5d7..20ef8a71f710bc 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -27,13 +27,11 @@ int Py_DebugFlag; int Py_VerboseFlag; int Py_IgnoreEnvironmentFlag; -_PyRuntimeState _PyRuntime = {0, 0}; +_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT; /* Forward */ grammar *getgrammar(const char *filename); -void Py_Exit(int) _Py_NO_RETURN; - void Py_Exit(int sts) { @@ -62,8 +60,6 @@ main(int argc, char **argv) filename = argv[1]; graminit_h = argv[2]; graminit_c = argv[3]; - _PyObject_Initialize(&_PyRuntime.obj); - _PyMem_Initialize(&_PyRuntime.mem); g = getgrammar(filename); fp = fopen(graminit_c, "w"); if (fp == NULL) { diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 28254e103318c9..fbc98880c9a5b1 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -18,6 +18,9 @@ #include "abstract.h" #endif /* PGEN */ +/* Alternate tab spacing */ +#define ALTTABSIZE 1 + #define is_potential_identifier_start(c) (\ (c >= 'a' && c <= 'z')\ || (c >= 'A' && c <= 'Z')\ @@ -133,9 +136,6 @@ tok_new(void) tok->prompt = tok->nextprompt = NULL; tok->lineno = 0; tok->level = 0; - tok->altwarning = 1; - tok->alterror = 1; - tok->alttabsize = 1; tok->altindstack[0] = 0; tok->decoding_state = STATE_INIT; tok->decoding_erred = 0; @@ -1283,22 +1283,9 @@ PyToken_ThreeChars(int c1, int c2, int c3) static int indenterror(struct tok_state *tok) { - if (tok->alterror) { - tok->done = E_TABSPACE; - tok->cur = tok->inp; - return 1; - } - if (tok->altwarning) { -#ifdef PGEN - PySys_WriteStderr("inconsistent use of tabs and spaces " - "in indentation\n"); -#else - PySys_FormatStderr("%U: inconsistent use of tabs and spaces " - "in indentation\n", tok->filename); -#endif - tok->altwarning = 0; - } - return 0; + tok->done = E_TABSPACE; + tok->cur = tok->inp; + return ERRORTOKEN; } #ifdef PGEN @@ -1378,9 +1365,8 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) col++, altcol++; } else if (c == '\t') { - col = (col/tok->tabsize + 1) * tok->tabsize; - altcol = (altcol/tok->alttabsize + 1) - * tok->alttabsize; + col = (col / tok->tabsize + 1) * tok->tabsize; + altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE; } else if (c == '\014') {/* Control-L (formfeed) */ col = altcol = 0; /* For Emacs users */ @@ -1409,9 +1395,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) if (col == tok->indstack[tok->indent]) { /* No change */ if (altcol != tok->altindstack[tok->indent]) { - if (indenterror(tok)) { - return ERRORTOKEN; - } + return indenterror(tok); } } else if (col > tok->indstack[tok->indent]) { @@ -1422,9 +1406,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) return ERRORTOKEN; } if (altcol <= tok->altindstack[tok->indent]) { - if (indenterror(tok)) { - return ERRORTOKEN; - } + return indenterror(tok); } tok->pendin++; tok->indstack[++tok->indent] = col; @@ -1443,9 +1425,7 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end) return ERRORTOKEN; } if (altcol != tok->altindstack[tok->indent]) { - if (indenterror(tok)) { - return ERRORTOKEN; - } + return indenterror(tok); } } } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index ad8b1c80171263..2e31d8624da7ca 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -47,9 +47,6 @@ struct tok_state { (Grammar/Grammar). */ PyObject *filename; #endif - int altwarning; /* Issue warning if alternate tabs don't match */ - int alterror; /* Issue error if alternate tabs don't match */ - int alttabsize; /* Alternate tab spacing */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ enum decoding_state decoding_state; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 1069966a1876ed..b8b630cfed3626 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -81,7 +81,10 @@ main(int argc, char *argv[]) Py_SetProgramName(L"./_freeze_importlib"); /* Don't install importlib, since it could execute outdated bytecode. */ - _Py_InitializeEx_Private(1, 0); + _PyInitError err = _Py_InitializeEx_Private(1, 0); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } if (strstr(inpath, "_external") != NULL) { is_bootstrap = 0; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e68e68de327b12..a528f3e3aa0af9 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1,4 +1,5 @@ #include +#include "pythread.h" #include #include @@ -125,6 +126,76 @@ static int test_forced_io_encoding(void) return 0; } + +/********************************************************* + * Test parts of the C-API that work before initialization + *********************************************************/ + +static int test_pre_initialization_api(void) +{ + /* Leading "./" ensures getpath.c can still find the standard library */ + wchar_t *program = Py_DecodeLocale("./spam", NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode program name\n"); + return 1; + } + Py_SetProgramName(program); + + Py_Initialize(); + Py_Finalize(); + + PyMem_RawFree(program); + return 0; +} + +static void bpo20891_thread(void *lockp) +{ + PyThread_type_lock lock = *((PyThread_type_lock*)lockp); + + PyGILState_STATE state = PyGILState_Ensure(); + if (!PyGILState_Check()) { + fprintf(stderr, "PyGILState_Check failed!"); + abort(); + } + + PyGILState_Release(state); + + PyThread_release_lock(lock); + + PyThread_exit_thread(); +} + +static int test_bpo20891(void) +{ + /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread before + calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must + call PyEval_InitThreads() for us in this case. */ + PyThread_type_lock lock = PyThread_allocate_lock(); + if (!lock) { + fprintf(stderr, "PyThread_allocate_lock failed!"); + return 1; + } + + _testembed_Py_Initialize(); + + unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); + if (thrd == PYTHREAD_INVALID_THREAD_ID) { + fprintf(stderr, "PyThread_start_new_thread failed!"); + return 1; + } + PyThread_acquire_lock(lock, WAIT_LOCK); + + Py_BEGIN_ALLOW_THREADS + /* wait until the thread exit */ + PyThread_acquire_lock(lock, WAIT_LOCK); + Py_END_ALLOW_THREADS + + PyThread_free_lock(lock); + + return 0; +} + + /* ********************************************************* * List of test cases and the function that implements it. * @@ -146,6 +217,8 @@ struct TestCase static struct TestCase TestCases[] = { { "forced_io_encoding", test_forced_io_encoding }, { "repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters }, + { "pre_initialization_api", test_pre_initialization_api }, + { "bpo20891", test_bpo20891 }, { NULL, NULL } }; diff --git a/Programs/python.c b/Programs/python.c index 4f6b9198c8504e..22d55bbc4ceb0e 100644 --- a/Programs/python.c +++ b/Programs/python.c @@ -1,6 +1,7 @@ /* Minimal main program -- everything is loaded from the library */ #include "Python.h" +#include "internal/pystate.h" #include #ifdef __FreeBSD__ @@ -22,15 +23,19 @@ main(int argc, char **argv) wchar_t **argv_copy; /* We need a second copy, as Python might modify the first one. */ wchar_t **argv_copy2; - int i, res; + int i, status; char *oldloc; - /* Force malloc() allocator to bootstrap Python */ -#ifdef Py_DEBUG - (void)_PyMem_SetupAllocators("malloc_debug"); -# else - (void)_PyMem_SetupAllocators("malloc"); -# endif + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + fprintf(stderr, "Fatal Python error: %s\n", err.msg); + fflush(stderr); + exit(1); + } + + /* Force default allocator, to be able to release memory above + with a known allocator. */ + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL); argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); @@ -54,15 +59,8 @@ main(int argc, char **argv) return 1; } -#ifdef __ANDROID__ - /* Passing "" to setlocale() on Android requests the C locale rather - * than checking environment variables, so request C.UTF-8 explicitly - */ - setlocale(LC_ALL, "C.UTF-8"); -#else /* Reconfigure the locale to the default for this process */ - setlocale(LC_ALL, ""); -#endif + _Py_SetLocaleFromEnv(LC_ALL); /* The legacy C locale assumes ASCII as the default text encoding, which * causes problems not only for the CPython runtime, but also other @@ -95,21 +93,17 @@ main(int argc, char **argv) setlocale(LC_ALL, oldloc); PyMem_RawFree(oldloc); - res = Py_Main(argc, argv_copy); + status = Py_Main(argc, argv_copy); - /* Force again malloc() allocator to release memory blocks allocated - before Py_Main() */ -#ifdef Py_DEBUG - (void)_PyMem_SetupAllocators("malloc_debug"); -# else - (void)_PyMem_SetupAllocators("malloc"); -# endif + /* Py_Main() can change PyMem_RawMalloc() allocator, so restore the default + to release memory blocks allocated before Py_Main() */ + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL); for (i = 0; i < argc; i++) { PyMem_RawFree(argv_copy2[i]); } PyMem_RawFree(argv_copy); PyMem_RawFree(argv_copy2); - return res; + return status; } #endif diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 2f1e737ea656f6..81226c8a060d47 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -546,13 +546,17 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) int res = -1; PyObject *key, *value, *fields; fields = _PyObject_GetAttrId((PyObject*)Py_TYPE(self), &PyId__fields); - if (!fields) - PyErr_Clear(); if (fields) { numfields = PySequence_Size(fields); if (numfields == -1) goto cleanup; } + else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + else { + goto cleanup; + } res = 0; /* if no error occurs, this stays 0 to the end */ if (numfields < PyTuple_GET_SIZE(args)) { @@ -844,17 +848,20 @@ static int add_ast_fields(void) return 0; } -static int exists_not_none(PyObject *obj, _Py_Identifier *id) +static PyObject *get_not_none(PyObject *obj, _Py_Identifier *id) { - int isnone; PyObject *attr = _PyObject_GetAttrId(obj, id); if (!attr) { - PyErr_Clear(); - return 0; + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + return NULL; } - isnone = attr == Py_None; - Py_DECREF(attr); - return !isnone; + else if (attr == Py_None) { + Py_DECREF(attr); + return NULL; + } + return attr; } @@ -4005,12 +4012,11 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) asdl_seq* body; string docstring; - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Module field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4030,16 +4036,19 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Module"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Module"); + } return 1; } - if (exists_not_none(obj, &PyId_docstring)) { + tmp = get_not_none(obj, &PyId_docstring); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_docstring); - if (tmp == NULL) goto failed; res = obj2ast_string(tmp, &docstring, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { docstring = NULL; } @@ -4054,12 +4063,11 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { asdl_seq* body; - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Interactive field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4079,7 +4087,9 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Interactive"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Interactive"); + } return 1; } *out = Interactive(body, arena); @@ -4093,15 +4103,16 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { expr_ty body; - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &body, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Expression"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Expression"); + } return 1; } *out = Expression(body, arena); @@ -4115,12 +4126,11 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) if (isinstance) { asdl_seq* body; - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Suite field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4140,7 +4150,9 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Suite"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Suite"); + } return 1; } *out = Suite(body, arena); @@ -4167,26 +4179,28 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) *out = NULL; return 0; } - if (_PyObject_HasAttrId(obj, &PyId_lineno)) { + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_lineno); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from stmt"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from stmt"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from stmt"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from stmt"); + } return 1; } isinstance = PyObject_IsInstance(obj, (PyObject*)FunctionDef_type); @@ -4201,34 +4215,35 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty returns; string docstring; - if (_PyObject_HasAttrId(obj, &PyId_name)) { + tmp = _PyObject_GetAttrId(obj, &PyId_name); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_name); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from FunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from FunctionDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_args)) { + tmp = _PyObject_GetAttrId(obj, &PyId_args); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_args); - if (tmp == NULL) goto failed; res = obj2ast_arguments(tmp, &args, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from FunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from FunctionDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "FunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4248,15 +4263,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from FunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from FunctionDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) { + tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "FunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4276,26 +4292,30 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from FunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from FunctionDef"); + } return 1; } - if (exists_not_none(obj, &PyId_returns)) { + tmp = get_not_none(obj, &PyId_returns); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_returns); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &returns, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { returns = NULL; } - if (exists_not_none(obj, &PyId_docstring)) { + tmp = get_not_none(obj, &PyId_docstring); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_docstring); - if (tmp == NULL) goto failed; res = obj2ast_string(tmp, &docstring, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { docstring = NULL; } @@ -4316,34 +4336,35 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty returns; string docstring; - if (_PyObject_HasAttrId(obj, &PyId_name)) { + tmp = _PyObject_GetAttrId(obj, &PyId_name); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_name); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from AsyncFunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from AsyncFunctionDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_args)) { + tmp = _PyObject_GetAttrId(obj, &PyId_args); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_args); - if (tmp == NULL) goto failed; res = obj2ast_arguments(tmp, &args, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from AsyncFunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from AsyncFunctionDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4363,15 +4384,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFunctionDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) { + tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "AsyncFunctionDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4391,26 +4413,30 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from AsyncFunctionDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from AsyncFunctionDef"); + } return 1; } - if (exists_not_none(obj, &PyId_returns)) { + tmp = get_not_none(obj, &PyId_returns); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_returns); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &returns, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { returns = NULL; } - if (exists_not_none(obj, &PyId_docstring)) { + tmp = get_not_none(obj, &PyId_docstring); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_docstring); - if (tmp == NULL) goto failed; res = obj2ast_string(tmp, &docstring, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { docstring = NULL; } @@ -4431,23 +4457,23 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* decorator_list; string docstring; - if (_PyObject_HasAttrId(obj, &PyId_name)) { + tmp = _PyObject_GetAttrId(obj, &PyId_name); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_name); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ClassDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from ClassDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_bases)) { + tmp = _PyObject_GetAttrId(obj, &PyId_bases); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_bases); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"bases\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4467,15 +4493,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"bases\" missing from ClassDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"bases\" missing from ClassDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_keywords)) { + tmp = _PyObject_GetAttrId(obj, &PyId_keywords); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_keywords); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4495,15 +4522,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from ClassDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from ClassDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4523,15 +4551,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from ClassDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from ClassDef"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_decorator_list)) { + tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_decorator_list); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ClassDef field \"decorator_list\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4551,16 +4580,19 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef"); + } return 1; } - if (exists_not_none(obj, &PyId_docstring)) { + tmp = get_not_none(obj, &PyId_docstring); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_docstring); - if (tmp == NULL) goto failed; res = obj2ast_string(tmp, &docstring, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { docstring = NULL; } @@ -4576,13 +4608,14 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (exists_not_none(obj, &PyId_value)) { + tmp = get_not_none(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { value = NULL; } @@ -4597,12 +4630,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* targets; - if (_PyObject_HasAttrId(obj, &PyId_targets)) { + tmp = _PyObject_GetAttrId(obj, &PyId_targets); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_targets); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Delete field \"targets\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4622,7 +4654,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"targets\" missing from Delete"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"targets\" missing from Delete"); + } return 1; } *out = Delete(targets, lineno, col_offset, arena); @@ -4637,12 +4671,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* targets; expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_targets)) { + tmp = _PyObject_GetAttrId(obj, &PyId_targets); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_targets); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Assign field \"targets\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4662,18 +4695,21 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"targets\" missing from Assign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"targets\" missing from Assign"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Assign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Assign"); + } return 1; } *out = Assign(targets, value, lineno, col_offset, arena); @@ -4689,37 +4725,40 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) operator_ty op; expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_target)) { + tmp = _PyObject_GetAttrId(obj, &PyId_target); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_target); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AugAssign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AugAssign"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_op)) { + tmp = _PyObject_GetAttrId(obj, &PyId_op); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_op); - if (tmp == NULL) goto failed; res = obj2ast_operator(tmp, &op, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from AugAssign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from AugAssign"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from AugAssign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from AugAssign"); + } return 1; } *out = AugAssign(target, op, value, lineno, col_offset, arena); @@ -4736,47 +4775,51 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty value; int simple; - if (_PyObject_HasAttrId(obj, &PyId_target)) { + tmp = _PyObject_GetAttrId(obj, &PyId_target); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_target); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AnnAssign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AnnAssign"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_annotation)) { + tmp = _PyObject_GetAttrId(obj, &PyId_annotation); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_annotation); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &annotation, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"annotation\" missing from AnnAssign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"annotation\" missing from AnnAssign"); + } return 1; } - if (exists_not_none(obj, &PyId_value)) { + tmp = get_not_none(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { value = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_simple)) { + tmp = _PyObject_GetAttrId(obj, &PyId_simple); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_simple); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &simple, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"simple\" missing from AnnAssign"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"simple\" missing from AnnAssign"); + } return 1; } *out = AnnAssign(target, annotation, value, simple, lineno, col_offset, @@ -4794,34 +4837,35 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (_PyObject_HasAttrId(obj, &PyId_target)) { + tmp = _PyObject_GetAttrId(obj, &PyId_target); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_target); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from For"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from For"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_iter)) { + tmp = _PyObject_GetAttrId(obj, &PyId_iter); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_iter); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &iter, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from For"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from For"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "For field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4841,15 +4885,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from For"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from For"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_orelse)) { + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_orelse); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "For field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4869,7 +4914,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from For"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from For"); + } return 1; } *out = For(target, iter, body, orelse, lineno, col_offset, arena); @@ -4886,34 +4933,35 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (_PyObject_HasAttrId(obj, &PyId_target)) { + tmp = _PyObject_GetAttrId(obj, &PyId_target); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_target); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AsyncFor"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from AsyncFor"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_iter)) { + tmp = _PyObject_GetAttrId(obj, &PyId_iter); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_iter); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &iter, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from AsyncFor"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from AsyncFor"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "AsyncFor field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4933,15 +4981,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFor"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncFor"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_orelse)) { + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_orelse); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "AsyncFor field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -4961,7 +5010,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from AsyncFor"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from AsyncFor"); + } return 1; } *out = AsyncFor(target, iter, body, orelse, lineno, col_offset, arena); @@ -4977,23 +5028,23 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (_PyObject_HasAttrId(obj, &PyId_test)) { + tmp = _PyObject_GetAttrId(obj, &PyId_test); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_test); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from While"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from While"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "While field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5013,15 +5064,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from While"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from While"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_orelse)) { + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_orelse); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "While field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5041,7 +5093,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from While"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from While"); + } return 1; } *out = While(test, body, orelse, lineno, col_offset, arena); @@ -5057,23 +5111,23 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* body; asdl_seq* orelse; - if (_PyObject_HasAttrId(obj, &PyId_test)) { + tmp = _PyObject_GetAttrId(obj, &PyId_test); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_test); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from If"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from If"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "If field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5093,15 +5147,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from If"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from If"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_orelse)) { + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_orelse); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "If field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5121,7 +5176,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from If"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from If"); + } return 1; } *out = If(test, body, orelse, lineno, col_offset, arena); @@ -5136,12 +5193,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* items; asdl_seq* body; - if (_PyObject_HasAttrId(obj, &PyId_items)) { + tmp = _PyObject_GetAttrId(obj, &PyId_items); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_items); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "With field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5161,15 +5217,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from With"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from With"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "With field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5189,7 +5246,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from With"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from With"); + } return 1; } *out = With(items, body, lineno, col_offset, arena); @@ -5204,12 +5263,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* items; asdl_seq* body; - if (_PyObject_HasAttrId(obj, &PyId_items)) { + tmp = _PyObject_GetAttrId(obj, &PyId_items); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_items); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "AsyncWith field \"items\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5229,15 +5287,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from AsyncWith"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"items\" missing from AsyncWith"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "AsyncWith field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5257,7 +5316,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncWith"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from AsyncWith"); + } return 1; } *out = AsyncWith(items, body, lineno, col_offset, arena); @@ -5272,23 +5333,25 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty exc; expr_ty cause; - if (exists_not_none(obj, &PyId_exc)) { + tmp = get_not_none(obj, &PyId_exc); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_exc); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &exc, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { exc = NULL; } - if (exists_not_none(obj, &PyId_cause)) { + tmp = get_not_none(obj, &PyId_cause); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_cause); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &cause, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { cause = NULL; } @@ -5306,12 +5369,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* orelse; asdl_seq* finalbody; - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Try field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5331,15 +5393,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Try"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Try"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_handlers)) { + tmp = _PyObject_GetAttrId(obj, &PyId_handlers); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_handlers); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Try field \"handlers\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5359,15 +5422,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"handlers\" missing from Try"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"handlers\" missing from Try"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_orelse)) { + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_orelse); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Try field \"orelse\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5387,15 +5451,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from Try"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from Try"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_finalbody)) { + tmp = _PyObject_GetAttrId(obj, &PyId_finalbody); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_finalbody); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Try field \"finalbody\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5415,7 +5480,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"finalbody\" missing from Try"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"finalbody\" missing from Try"); + } return 1; } *out = Try(body, handlers, orelse, finalbody, lineno, col_offset, @@ -5431,24 +5498,26 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) expr_ty test; expr_ty msg; - if (_PyObject_HasAttrId(obj, &PyId_test)) { + tmp = _PyObject_GetAttrId(obj, &PyId_test); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_test); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from Assert"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from Assert"); + } return 1; } - if (exists_not_none(obj, &PyId_msg)) { + tmp = get_not_none(obj, &PyId_msg); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_msg); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &msg, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { msg = NULL; } @@ -5463,12 +5532,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* names; - if (_PyObject_HasAttrId(obj, &PyId_names)) { + tmp = _PyObject_GetAttrId(obj, &PyId_names); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_names); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Import field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5488,7 +5556,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Import"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Import"); + } return 1; } *out = Import(names, lineno, col_offset, arena); @@ -5504,22 +5574,22 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) asdl_seq* names; int level; - if (exists_not_none(obj, &PyId_module)) { + tmp = get_not_none(obj, &PyId_module); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_module); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &module, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { module = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_names)) { + tmp = _PyObject_GetAttrId(obj, &PyId_names); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_names); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ImportFrom field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5539,16 +5609,19 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from ImportFrom"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from ImportFrom"); + } return 1; } - if (exists_not_none(obj, &PyId_level)) { + tmp = get_not_none(obj, &PyId_level); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_level); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &level, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { level = 0; } @@ -5563,12 +5636,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* names; - if (_PyObject_HasAttrId(obj, &PyId_names)) { + tmp = _PyObject_GetAttrId(obj, &PyId_names); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_names); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Global field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5588,7 +5660,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Global"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Global"); + } return 1; } *out = Global(names, lineno, col_offset, arena); @@ -5602,12 +5676,11 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { asdl_seq* names; - if (_PyObject_HasAttrId(obj, &PyId_names)) { + tmp = _PyObject_GetAttrId(obj, &PyId_names); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_names); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Nonlocal field \"names\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5627,7 +5700,9 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Nonlocal"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"names\" missing from Nonlocal"); + } return 1; } *out = Nonlocal(names, lineno, col_offset, arena); @@ -5641,15 +5716,16 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Expr"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Expr"); + } return 1; } *out = Expr(value, lineno, col_offset, arena); @@ -5706,26 +5782,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) *out = NULL; return 0; } - if (_PyObject_HasAttrId(obj, &PyId_lineno)) { + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_lineno); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from expr"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from expr"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from expr"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from expr"); + } return 1; } isinstance = PyObject_IsInstance(obj, (PyObject*)BoolOp_type); @@ -5736,23 +5814,23 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) boolop_ty op; asdl_seq* values; - if (_PyObject_HasAttrId(obj, &PyId_op)) { + tmp = _PyObject_GetAttrId(obj, &PyId_op); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_op); - if (tmp == NULL) goto failed; res = obj2ast_boolop(tmp, &op, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BoolOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BoolOp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_values)) { + tmp = _PyObject_GetAttrId(obj, &PyId_values); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_values); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "BoolOp field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5772,7 +5850,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from BoolOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from BoolOp"); + } return 1; } *out = BoolOp(op, values, lineno, col_offset, arena); @@ -5788,37 +5868,40 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) operator_ty op; expr_ty right; - if (_PyObject_HasAttrId(obj, &PyId_left)) { + tmp = _PyObject_GetAttrId(obj, &PyId_left); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_left); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &left, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from BinOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from BinOp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_op)) { + tmp = _PyObject_GetAttrId(obj, &PyId_op); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_op); - if (tmp == NULL) goto failed; res = obj2ast_operator(tmp, &op, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BinOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from BinOp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_right)) { + tmp = _PyObject_GetAttrId(obj, &PyId_right); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_right); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &right, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"right\" missing from BinOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"right\" missing from BinOp"); + } return 1; } *out = BinOp(left, op, right, lineno, col_offset, arena); @@ -5833,26 +5916,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) unaryop_ty op; expr_ty operand; - if (_PyObject_HasAttrId(obj, &PyId_op)) { + tmp = _PyObject_GetAttrId(obj, &PyId_op); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_op); - if (tmp == NULL) goto failed; res = obj2ast_unaryop(tmp, &op, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from UnaryOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"op\" missing from UnaryOp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_operand)) { + tmp = _PyObject_GetAttrId(obj, &PyId_operand); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_operand); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &operand, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"operand\" missing from UnaryOp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"operand\" missing from UnaryOp"); + } return 1; } *out = UnaryOp(op, operand, lineno, col_offset, arena); @@ -5867,26 +5952,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) arguments_ty args; expr_ty body; - if (_PyObject_HasAttrId(obj, &PyId_args)) { + tmp = _PyObject_GetAttrId(obj, &PyId_args); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_args); - if (tmp == NULL) goto failed; res = obj2ast_arguments(tmp, &args, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Lambda"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Lambda"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &body, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Lambda"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Lambda"); + } return 1; } *out = Lambda(args, body, lineno, col_offset, arena); @@ -5902,37 +5989,40 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty body; expr_ty orelse; - if (_PyObject_HasAttrId(obj, &PyId_test)) { + tmp = _PyObject_GetAttrId(obj, &PyId_test); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_test); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &test, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from IfExp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"test\" missing from IfExp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &body, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from IfExp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from IfExp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_orelse)) { + tmp = _PyObject_GetAttrId(obj, &PyId_orelse); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_orelse); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &orelse, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from IfExp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"orelse\" missing from IfExp"); + } return 1; } *out = IfExp(test, body, orelse, lineno, col_offset, arena); @@ -5947,12 +6037,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* keys; asdl_seq* values; - if (_PyObject_HasAttrId(obj, &PyId_keys)) { + tmp = _PyObject_GetAttrId(obj, &PyId_keys); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_keys); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Dict field \"keys\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -5972,15 +6061,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"keys\" missing from Dict"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"keys\" missing from Dict"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_values)) { + tmp = _PyObject_GetAttrId(obj, &PyId_values); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_values); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Dict field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6000,7 +6090,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from Dict"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from Dict"); + } return 1; } *out = Dict(keys, values, lineno, col_offset, arena); @@ -6014,12 +6106,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { asdl_seq* elts; - if (_PyObject_HasAttrId(obj, &PyId_elts)) { + tmp = _PyObject_GetAttrId(obj, &PyId_elts); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_elts); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Set field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6039,7 +6130,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from Set"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from Set"); + } return 1; } *out = Set(elts, lineno, col_offset, arena); @@ -6054,23 +6147,23 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty elt; asdl_seq* generators; - if (_PyObject_HasAttrId(obj, &PyId_elt)) { + tmp = _PyObject_GetAttrId(obj, &PyId_elt); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_elt); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &elt, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from ListComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from ListComp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_generators)) { + tmp = _PyObject_GetAttrId(obj, &PyId_generators); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_generators); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ListComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6090,7 +6183,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from ListComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from ListComp"); + } return 1; } *out = ListComp(elt, generators, lineno, col_offset, arena); @@ -6105,23 +6200,23 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty elt; asdl_seq* generators; - if (_PyObject_HasAttrId(obj, &PyId_elt)) { + tmp = _PyObject_GetAttrId(obj, &PyId_elt); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_elt); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &elt, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from SetComp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_generators)) { + tmp = _PyObject_GetAttrId(obj, &PyId_generators); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_generators); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "SetComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6141,7 +6236,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from SetComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from SetComp"); + } return 1; } *out = SetComp(elt, generators, lineno, col_offset, arena); @@ -6157,34 +6254,35 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty value; asdl_seq* generators; - if (_PyObject_HasAttrId(obj, &PyId_key)) { + tmp = _PyObject_GetAttrId(obj, &PyId_key); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_key); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &key, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"key\" missing from DictComp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from DictComp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_generators)) { + tmp = _PyObject_GetAttrId(obj, &PyId_generators); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_generators); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "DictComp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6204,7 +6302,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from DictComp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from DictComp"); + } return 1; } *out = DictComp(key, value, generators, lineno, col_offset, arena); @@ -6219,23 +6319,23 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty elt; asdl_seq* generators; - if (_PyObject_HasAttrId(obj, &PyId_elt)) { + tmp = _PyObject_GetAttrId(obj, &PyId_elt); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_elt); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &elt, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from GeneratorExp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"elt\" missing from GeneratorExp"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_generators)) { + tmp = _PyObject_GetAttrId(obj, &PyId_generators); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_generators); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "GeneratorExp field \"generators\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6255,7 +6355,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from GeneratorExp"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"generators\" missing from GeneratorExp"); + } return 1; } *out = GeneratorExp(elt, generators, lineno, col_offset, arena); @@ -6269,15 +6371,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Await"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Await"); + } return 1; } *out = Await(value, lineno, col_offset, arena); @@ -6291,13 +6394,14 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (exists_not_none(obj, &PyId_value)) { + tmp = get_not_none(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { value = NULL; } @@ -6312,15 +6416,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from YieldFrom"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from YieldFrom"); + } return 1; } *out = YieldFrom(value, lineno, col_offset, arena); @@ -6336,23 +6441,23 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_int_seq* ops; asdl_seq* comparators; - if (_PyObject_HasAttrId(obj, &PyId_left)) { + tmp = _PyObject_GetAttrId(obj, &PyId_left); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_left); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &left, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from Compare"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"left\" missing from Compare"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ops)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ops); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_ops); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Compare field \"ops\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6372,15 +6477,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ops\" missing from Compare"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ops\" missing from Compare"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_comparators)) { + tmp = _PyObject_GetAttrId(obj, &PyId_comparators); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_comparators); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Compare field \"comparators\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6400,7 +6506,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"comparators\" missing from Compare"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"comparators\" missing from Compare"); + } return 1; } *out = Compare(left, ops, comparators, lineno, col_offset, arena); @@ -6416,23 +6524,23 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* args; asdl_seq* keywords; - if (_PyObject_HasAttrId(obj, &PyId_func)) { + tmp = _PyObject_GetAttrId(obj, &PyId_func); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_func); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &func, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"func\" missing from Call"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"func\" missing from Call"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_args)) { + tmp = _PyObject_GetAttrId(obj, &PyId_args); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_args); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Call field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6452,15 +6560,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Call"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from Call"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_keywords)) { + tmp = _PyObject_GetAttrId(obj, &PyId_keywords); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_keywords); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Call field \"keywords\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6480,7 +6589,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from Call"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from Call"); + } return 1; } *out = Call(func, args, keywords, lineno, col_offset, arena); @@ -6494,15 +6605,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { object n; - if (_PyObject_HasAttrId(obj, &PyId_n)) { + tmp = _PyObject_GetAttrId(obj, &PyId_n); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_n); - if (tmp == NULL) goto failed; res = obj2ast_object(tmp, &n, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"n\" missing from Num"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"n\" missing from Num"); + } return 1; } *out = Num(n, lineno, col_offset, arena); @@ -6516,15 +6628,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { string s; - if (_PyObject_HasAttrId(obj, &PyId_s)) { + tmp = _PyObject_GetAttrId(obj, &PyId_s); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_s); - if (tmp == NULL) goto failed; res = obj2ast_string(tmp, &s, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); + } return 1; } *out = Str(s, lineno, col_offset, arena); @@ -6540,34 +6653,37 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) int conversion; expr_ty format_spec; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from FormattedValue"); + } return 1; } - if (exists_not_none(obj, &PyId_conversion)) { + tmp = get_not_none(obj, &PyId_conversion); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_conversion); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &conversion, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { conversion = 0; } - if (exists_not_none(obj, &PyId_format_spec)) { + tmp = get_not_none(obj, &PyId_format_spec); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_format_spec); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &format_spec, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { format_spec = NULL; } @@ -6583,12 +6699,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { asdl_seq* values; - if (_PyObject_HasAttrId(obj, &PyId_values)) { + tmp = _PyObject_GetAttrId(obj, &PyId_values); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_values); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "JoinedStr field \"values\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6608,7 +6723,9 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"values\" missing from JoinedStr"); + } return 1; } *out = JoinedStr(values, lineno, col_offset, arena); @@ -6622,15 +6739,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { bytes s; - if (_PyObject_HasAttrId(obj, &PyId_s)) { + tmp = _PyObject_GetAttrId(obj, &PyId_s); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_s); - if (tmp == NULL) goto failed; res = obj2ast_bytes(tmp, &s, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Bytes"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Bytes"); + } return 1; } *out = Bytes(s, lineno, col_offset, arena); @@ -6644,15 +6762,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { singleton value; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_singleton(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NameConstant"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NameConstant"); + } return 1; } *out = NameConstant(value, lineno, col_offset, arena); @@ -6676,15 +6795,16 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) if (isinstance) { constant value; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_constant(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant"); + } return 1; } *out = Constant(value, lineno, col_offset, arena); @@ -6700,37 +6820,40 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) identifier attr; expr_context_ty ctx; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Attribute"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Attribute"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_attr)) { + tmp = _PyObject_GetAttrId(obj, &PyId_attr); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_attr); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &attr, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"attr\" missing from Attribute"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"attr\" missing from Attribute"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ctx)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_ctx); - if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Attribute"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Attribute"); + } return 1; } *out = Attribute(value, attr, ctx, lineno, col_offset, arena); @@ -6746,37 +6869,40 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) slice_ty slice; expr_context_ty ctx; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Subscript"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Subscript"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_slice)) { + tmp = _PyObject_GetAttrId(obj, &PyId_slice); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_slice); - if (tmp == NULL) goto failed; res = obj2ast_slice(tmp, &slice, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"slice\" missing from Subscript"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"slice\" missing from Subscript"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ctx)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_ctx); - if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Subscript"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Subscript"); + } return 1; } *out = Subscript(value, slice, ctx, lineno, col_offset, arena); @@ -6791,26 +6917,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) expr_ty value; expr_context_ty ctx; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Starred"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Starred"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ctx)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_ctx); - if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Starred"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Starred"); + } return 1; } *out = Starred(value, ctx, lineno, col_offset, arena); @@ -6825,26 +6953,28 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) identifier id; expr_context_ty ctx; - if (_PyObject_HasAttrId(obj, &PyId_id)) { + tmp = _PyObject_GetAttrId(obj, &PyId_id); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_id); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &id, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"id\" missing from Name"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"id\" missing from Name"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ctx)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_ctx); - if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Name"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Name"); + } return 1; } *out = Name(id, ctx, lineno, col_offset, arena); @@ -6859,12 +6989,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* elts; expr_context_ty ctx; - if (_PyObject_HasAttrId(obj, &PyId_elts)) { + tmp = _PyObject_GetAttrId(obj, &PyId_elts); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_elts); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "List field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6884,18 +7013,21 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from List"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from List"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ctx)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_ctx); - if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from List"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from List"); + } return 1; } *out = List(elts, ctx, lineno, col_offset, arena); @@ -6910,12 +7042,11 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) asdl_seq* elts; expr_context_ty ctx; - if (_PyObject_HasAttrId(obj, &PyId_elts)) { + tmp = _PyObject_GetAttrId(obj, &PyId_elts); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_elts); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "Tuple field \"elts\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -6935,18 +7066,21 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from Tuple"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"elts\" missing from Tuple"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ctx)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ctx); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_ctx); - if (tmp == NULL) goto failed; res = obj2ast_expr_context(tmp, &ctx, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Tuple"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ctx\" missing from Tuple"); + } return 1; } *out = Tuple(elts, ctx, lineno, col_offset, arena); @@ -7038,33 +7172,36 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) expr_ty upper; expr_ty step; - if (exists_not_none(obj, &PyId_lower)) { + tmp = get_not_none(obj, &PyId_lower); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_lower); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &lower, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { lower = NULL; } - if (exists_not_none(obj, &PyId_upper)) { + tmp = get_not_none(obj, &PyId_upper); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_upper); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &upper, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { upper = NULL; } - if (exists_not_none(obj, &PyId_step)) { + tmp = get_not_none(obj, &PyId_step); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_step); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &step, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { step = NULL; } @@ -7079,12 +7216,11 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) if (isinstance) { asdl_seq* dims; - if (_PyObject_HasAttrId(obj, &PyId_dims)) { + tmp = _PyObject_GetAttrId(obj, &PyId_dims); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_dims); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ExtSlice field \"dims\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7104,7 +7240,9 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"dims\" missing from ExtSlice"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"dims\" missing from ExtSlice"); + } return 1; } *out = ExtSlice(dims, arena); @@ -7118,15 +7256,16 @@ obj2ast_slice(PyObject* obj, slice_ty* out, PyArena* arena) if (isinstance) { expr_ty value; - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Index"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Index"); + } return 1; } *out = Index(value, arena); @@ -7421,34 +7560,35 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena) asdl_seq* ifs; int is_async; - if (_PyObject_HasAttrId(obj, &PyId_target)) { + tmp = _PyObject_GetAttrId(obj, &PyId_target); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_target); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &target, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from comprehension"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"target\" missing from comprehension"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_iter)) { + tmp = _PyObject_GetAttrId(obj, &PyId_iter); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_iter); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &iter, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from comprehension"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"iter\" missing from comprehension"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_ifs)) { + tmp = _PyObject_GetAttrId(obj, &PyId_ifs); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_ifs); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "comprehension field \"ifs\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7468,18 +7608,21 @@ obj2ast_comprehension(PyObject* obj, comprehension_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"ifs\" missing from comprehension"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"ifs\" missing from comprehension"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_is_async)) { + tmp = _PyObject_GetAttrId(obj, &PyId_is_async); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_is_async); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &is_async, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"is_async\" missing from comprehension"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"is_async\" missing from comprehension"); + } return 1; } *out = comprehension(target, iter, ifs, is_async, arena); @@ -7502,26 +7645,28 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) *out = NULL; return 0; } - if (_PyObject_HasAttrId(obj, &PyId_lineno)) { + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_lineno); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from excepthandler"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from excepthandler"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from excepthandler"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from excepthandler"); + } return 1; } isinstance = PyObject_IsInstance(obj, (PyObject*)ExceptHandler_type); @@ -7533,32 +7678,33 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) identifier name; asdl_seq* body; - if (exists_not_none(obj, &PyId_type)) { + tmp = get_not_none(obj, &PyId_type); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_type); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &type, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { type = NULL; } - if (exists_not_none(obj, &PyId_name)) { + tmp = get_not_none(obj, &PyId_name); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_name); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { name = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_body)) { + tmp = _PyObject_GetAttrId(obj, &PyId_body); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_body); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "ExceptHandler field \"body\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7578,7 +7724,9 @@ obj2ast_excepthandler(PyObject* obj, excepthandler_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from ExceptHandler"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from ExceptHandler"); + } return 1; } *out = ExceptHandler(type, name, body, lineno, col_offset, arena); @@ -7603,12 +7751,11 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) arg_ty kwarg; asdl_seq* defaults; - if (_PyObject_HasAttrId(obj, &PyId_args)) { + tmp = _PyObject_GetAttrId(obj, &PyId_args); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_args); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"args\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7628,25 +7775,27 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from arguments"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"args\" missing from arguments"); + } return 1; } - if (exists_not_none(obj, &PyId_vararg)) { + tmp = get_not_none(obj, &PyId_vararg); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_vararg); - if (tmp == NULL) goto failed; res = obj2ast_arg(tmp, &vararg, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { vararg = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_kwonlyargs)) { + tmp = _PyObject_GetAttrId(obj, &PyId_kwonlyargs); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_kwonlyargs); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"kwonlyargs\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7666,15 +7815,16 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"kwonlyargs\" missing from arguments"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"kwonlyargs\" missing from arguments"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_kw_defaults)) { + tmp = _PyObject_GetAttrId(obj, &PyId_kw_defaults); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_kw_defaults); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"kw_defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7694,25 +7844,27 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"kw_defaults\" missing from arguments"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"kw_defaults\" missing from arguments"); + } return 1; } - if (exists_not_none(obj, &PyId_kwarg)) { + tmp = get_not_none(obj, &PyId_kwarg); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_kwarg); - if (tmp == NULL) goto failed; res = obj2ast_arg(tmp, &kwarg, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { kwarg = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_defaults)) { + tmp = _PyObject_GetAttrId(obj, &PyId_defaults); + if (tmp != NULL) { int res; Py_ssize_t len; Py_ssize_t i; - tmp = _PyObject_GetAttrId(obj, &PyId_defaults); - if (tmp == NULL) goto failed; if (!PyList_Check(tmp)) { PyErr_Format(PyExc_TypeError, "arguments field \"defaults\" must be a list, not a %.200s", tmp->ob_type->tp_name); goto failed; @@ -7732,7 +7884,9 @@ obj2ast_arguments(PyObject* obj, arguments_ty* out, PyArena* arena) } Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"defaults\" missing from arguments"); + } return 1; } *out = arguments(args, vararg, kwonlyargs, kw_defaults, kwarg, defaults, @@ -7752,47 +7906,51 @@ obj2ast_arg(PyObject* obj, arg_ty* out, PyArena* arena) int lineno; int col_offset; - if (_PyObject_HasAttrId(obj, &PyId_arg)) { + tmp = _PyObject_GetAttrId(obj, &PyId_arg); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_arg); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &arg, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from arg"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from arg"); + } return 1; } - if (exists_not_none(obj, &PyId_annotation)) { + tmp = get_not_none(obj, &PyId_annotation); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_annotation); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &annotation, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { annotation = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_lineno)) { + tmp = _PyObject_GetAttrId(obj, &PyId_lineno); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_lineno); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &lineno, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"lineno\" missing from arg"); + } return 1; } - if (_PyObject_HasAttrId(obj, &PyId_col_offset)) { + tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_col_offset); - if (tmp == NULL) goto failed; res = obj2ast_int(tmp, &col_offset, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"col_offset\" missing from arg"); + } return 1; } *out = arg(arg, annotation, lineno, col_offset, arena); @@ -7809,25 +7967,27 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena) identifier arg; expr_ty value; - if (exists_not_none(obj, &PyId_arg)) { + tmp = get_not_none(obj, &PyId_arg); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_arg); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &arg, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { arg = NULL; } - if (_PyObject_HasAttrId(obj, &PyId_value)) { + tmp = _PyObject_GetAttrId(obj, &PyId_value); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_value); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &value, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from keyword"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from keyword"); + } return 1; } *out = keyword(arg, value, arena); @@ -7844,24 +8004,26 @@ obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena) identifier name; identifier asname; - if (_PyObject_HasAttrId(obj, &PyId_name)) { + tmp = _PyObject_GetAttrId(obj, &PyId_name); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_name); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &name, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from alias"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"name\" missing from alias"); + } return 1; } - if (exists_not_none(obj, &PyId_asname)) { + tmp = get_not_none(obj, &PyId_asname); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_asname); - if (tmp == NULL) goto failed; res = obj2ast_identifier(tmp, &asname, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { asname = NULL; } @@ -7879,24 +8041,26 @@ obj2ast_withitem(PyObject* obj, withitem_ty* out, PyArena* arena) expr_ty context_expr; expr_ty optional_vars; - if (_PyObject_HasAttrId(obj, &PyId_context_expr)) { + tmp = _PyObject_GetAttrId(obj, &PyId_context_expr); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_context_expr); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &context_expr, arena); if (res != 0) goto failed; Py_CLEAR(tmp); } else { - PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem"); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_SetString(PyExc_TypeError, "required field \"context_expr\" missing from withitem"); + } return 1; } - if (exists_not_none(obj, &PyId_optional_vars)) { + tmp = get_not_none(obj, &PyId_optional_vars); + if (tmp != NULL) { int res; - tmp = _PyObject_GetAttrId(obj, &PyId_optional_vars); - if (tmp == NULL) goto failed; res = obj2ast_expr(tmp, &optional_vars, arena); if (res != 0) goto failed; Py_CLEAR(tmp); + } else if (PyErr_Occurred()) { + return 1; } else { optional_vars = NULL; } diff --git a/Python/_warnings.c b/Python/_warnings.c index aa80395caa8ae5..29370369e25d87 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -11,6 +11,9 @@ MODULE_NAME " provides basic warning filtering support.\n" _Py_IDENTIFIER(argv); _Py_IDENTIFIER(stderr); +_Py_IDENTIFIER(ignore); +_Py_IDENTIFIER(error); +_Py_static_string(PyId_default, "default"); static int check_matched(PyObject *obj, PyObject *arg) @@ -35,15 +38,15 @@ check_matched(PyObject *obj, PyObject *arg) A NULL return value can mean false or an error. */ static PyObject * -get_warnings_attr(const char *attr, int try_import) +get_warnings_attr(_Py_Identifier *attr_id, int try_import) { - static PyObject *warnings_str = NULL; + PyObject *warnings_str; PyObject *warnings_module, *obj; + _Py_IDENTIFIER(warnings); + warnings_str = _PyUnicode_FromId(&PyId_warnings); if (warnings_str == NULL) { - warnings_str = PyUnicode_InternFromString("warnings"); - if (warnings_str == NULL) - return NULL; + return NULL; } /* don't try to import after the start of the Python finallization */ @@ -52,7 +55,9 @@ get_warnings_attr(const char *attr, int try_import) if (warnings_module == NULL) { /* Fallback to the C implementation if we cannot get the Python implementation */ - PyErr_Clear(); + if (PyErr_ExceptionMatches(PyExc_ImportError)) { + PyErr_Clear(); + } return NULL; } } @@ -62,13 +67,11 @@ get_warnings_attr(const char *attr, int try_import) return NULL; } - if (!PyObject_HasAttrString(warnings_module, attr)) { - Py_DECREF(warnings_module); - return NULL; - } - - obj = PyObject_GetAttrString(warnings_module, attr); + obj = _PyObject_GetAttrId(warnings_module, attr_id); Py_DECREF(warnings_module); + if (obj == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } return obj; } @@ -77,21 +80,24 @@ static PyObject * get_once_registry(void) { PyObject *registry; + _Py_IDENTIFIER(onceregistry); - registry = get_warnings_attr("onceregistry", 0); + registry = get_warnings_attr(&PyId_onceregistry, 0); if (registry == NULL) { if (PyErr_Occurred()) return NULL; + assert(_PyRuntime.warnings.once_registry); return _PyRuntime.warnings.once_registry; } if (!PyDict_Check(registry)) { - PyErr_SetString(PyExc_TypeError, - "warnings.onceregistry must be a dict"); + PyErr_Format(PyExc_TypeError, + MODULE_NAME ".onceregistry must be a dict, " + "not '%.200s'", + Py_TYPE(registry)->tp_name); Py_DECREF(registry); return NULL; } - Py_DECREF(_PyRuntime.warnings.once_registry); - _PyRuntime.warnings.once_registry = registry; + Py_SETREF(_PyRuntime.warnings.once_registry, registry); return registry; } @@ -100,12 +106,14 @@ static PyObject * get_default_action(void) { PyObject *default_action; + _Py_IDENTIFIER(defaultaction); - default_action = get_warnings_attr("defaultaction", 0); + default_action = get_warnings_attr(&PyId_defaultaction, 0); if (default_action == NULL) { if (PyErr_Occurred()) { return NULL; } + assert(_PyRuntime.warnings.default_action); return _PyRuntime.warnings.default_action; } if (!PyUnicode_Check(default_action)) { @@ -116,8 +124,7 @@ get_default_action(void) Py_DECREF(default_action); return NULL; } - Py_DECREF(_PyRuntime.warnings.default_action); - _PyRuntime.warnings.default_action = default_action; + Py_SETREF(_PyRuntime.warnings.default_action, default_action); return default_action; } @@ -130,15 +137,15 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, PyObject *action; Py_ssize_t i; PyObject *warnings_filters; + _Py_IDENTIFIER(filters); - warnings_filters = get_warnings_attr("filters", 0); + warnings_filters = get_warnings_attr(&PyId_filters, 0); if (warnings_filters == NULL) { if (PyErr_Occurred()) return NULL; } else { - Py_DECREF(_PyRuntime.warnings.filters); - _PyRuntime.warnings.filters = warnings_filters; + Py_SETREF(_PyRuntime.warnings.filters, warnings_filters); } PyObject *filters = _PyRuntime.warnings.filters; @@ -388,11 +395,13 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, PyObject *sourceline, PyObject *source) { PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL; + _Py_IDENTIFIER(_showwarnmsg); + _Py_IDENTIFIER(WarningMessage); /* If the source parameter is set, try to get the Python implementation. The Python implementation is able to log the traceback where the source was allocated, whereas the C implementation doesn't. */ - show_fn = get_warnings_attr("_showwarnmsg", source != NULL); + show_fn = get_warnings_attr(&PyId__showwarnmsg, source != NULL); if (show_fn == NULL) { if (PyErr_Occurred()) return -1; @@ -406,10 +415,12 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message, goto error; } - warnmsg_cls = get_warnings_attr("WarningMessage", 0); + warnmsg_cls = get_warnings_attr(&PyId_WarningMessage, 0); if (warnmsg_cls == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "unable to get warnings.WarningMessage"); + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, + "unable to get warnings.WarningMessage"); + } goto error; } @@ -517,16 +528,21 @@ warn_explicit(PyObject *category, PyObject *message, goto cleanup; } + if (_PyUnicode_EqualToASCIIString(action, "ignore")) { + goto return_none; + } + /* Store in the registry that we've been here, *except* when the action is "always". */ rc = 0; if (!_PyUnicode_EqualToASCIIString(action, "always")) { if (registry != NULL && registry != Py_None && - PyDict_SetItem(registry, key, Py_True) < 0) + PyDict_SetItem(registry, key, Py_True) < 0) + { goto cleanup; - else if (_PyUnicode_EqualToASCIIString(action, "ignore")) - goto return_none; - else if (_PyUnicode_EqualToASCIIString(action, "once")) { + } + + if (_PyUnicode_EqualToASCIIString(action, "once")) { if (registry == NULL || registry == Py_None) { registry = get_once_registry(); if (registry == NULL) @@ -837,6 +853,68 @@ warnings_warn_impl(PyObject *module, PyObject *message, PyObject *category, return do_warn(message, category, stacklevel, source); } +static PyObject * +get_source_line(PyObject *module_globals, int lineno) +{ + _Py_IDENTIFIER(get_source); + _Py_IDENTIFIER(__loader__); + _Py_IDENTIFIER(__name__); + PyObject *loader; + PyObject *module_name; + PyObject *get_source; + PyObject *source; + PyObject *source_list; + PyObject *source_line; + + /* Check/get the requisite pieces needed for the loader. */ + loader = _PyDict_GetItemIdWithError(module_globals, &PyId___loader__); + if (loader == NULL) { + return NULL; + } + Py_INCREF(loader); + module_name = _PyDict_GetItemIdWithError(module_globals, &PyId___name__); + if (!module_name) { + Py_DECREF(loader); + return NULL; + } + Py_INCREF(module_name); + + /* Make sure the loader implements the optional get_source() method. */ + get_source = _PyObject_GetAttrId(loader, &PyId_get_source); + Py_DECREF(loader); + if (!get_source) { + Py_DECREF(module_name); + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } + return NULL; + } + /* Call get_source() to get the source code. */ + source = PyObject_CallFunctionObjArgs(get_source, module_name, NULL); + Py_DECREF(get_source); + Py_DECREF(module_name); + if (!source) { + return NULL; + } + if (source == Py_None) { + Py_DECREF(source); + return NULL; + } + + /* Split the source into lines. */ + source_list = PyUnicode_Splitlines(source, 0); + Py_DECREF(source); + if (!source_list) { + return NULL; + } + + /* Get the source line. */ + source_line = PyList_GetItem(source_list, lineno-1); + Py_XINCREF(source_line); + Py_DECREF(source_list); + return source_line; +} + static PyObject * warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) { @@ -851,6 +929,8 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) PyObject *registry = NULL; PyObject *module_globals = NULL; PyObject *sourceobj = NULL; + PyObject *source_line = NULL; + PyObject *returned; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOOO:warn_explicit", kwd_list, &message, &category, &filename, &lineno, &module, @@ -858,61 +938,15 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) return NULL; if (module_globals) { - _Py_IDENTIFIER(get_source); - PyObject *tmp; - PyObject *loader; - PyObject *module_name; - PyObject *source; - PyObject *source_list; - PyObject *source_line; - PyObject *returned; - - if ((tmp = _PyUnicode_FromId(&PyId_get_source)) == NULL) - return NULL; - - /* Check/get the requisite pieces needed for the loader. */ - loader = PyDict_GetItemString(module_globals, "__loader__"); - module_name = PyDict_GetItemString(module_globals, "__name__"); - - if (loader == NULL || module_name == NULL) - goto standard_call; - - /* Make sure the loader implements the optional get_source() method. */ - if (!_PyObject_HasAttrId(loader, &PyId_get_source)) - goto standard_call; - /* Call get_source() to get the source code. */ - source = PyObject_CallMethodObjArgs(loader, PyId_get_source.object, - module_name, NULL); - if (!source) - return NULL; - else if (source == Py_None) { - Py_DECREF(Py_None); - goto standard_call; - } - - /* Split the source into lines. */ - source_list = PyUnicode_Splitlines(source, 0); - Py_DECREF(source); - if (!source_list) - return NULL; - - /* Get the source line. */ - source_line = PyList_GetItem(source_list, lineno-1); - if (!source_line) { - Py_DECREF(source_list); + source_line = get_source_line(module_globals, lineno); + if (source_line == NULL && PyErr_Occurred()) { return NULL; } - - /* Handle the warning. */ - returned = warn_explicit(category, message, filename, lineno, module, - registry, source_line, sourceobj); - Py_DECREF(source_list); - return returned; } - - standard_call: - return warn_explicit(category, message, filename, lineno, module, - registry, NULL, sourceobj); + returned = warn_explicit(category, message, filename, lineno, module, + registry, source_line, sourceobj); + Py_XDECREF(source_line); + return returned; } static PyObject * @@ -1123,89 +1157,74 @@ static PyMethodDef warnings_functions[] = { static PyObject * -create_filter(PyObject *category, const char *action) +create_filter(PyObject *category, _Py_Identifier *id) { - static PyObject *ignore_str = NULL; - static PyObject *error_str = NULL; - static PyObject *default_str = NULL; - static PyObject *always_str = NULL; - PyObject *action_obj = NULL; - - if (!strcmp(action, "ignore")) { - if (ignore_str == NULL) { - ignore_str = PyUnicode_InternFromString("ignore"); - if (ignore_str == NULL) - return NULL; - } - action_obj = ignore_str; - } - else if (!strcmp(action, "error")) { - if (error_str == NULL) { - error_str = PyUnicode_InternFromString("error"); - if (error_str == NULL) - return NULL; - } - action_obj = error_str; - } - else if (!strcmp(action, "default")) { - if (default_str == NULL) { - default_str = PyUnicode_InternFromString("default"); - if (default_str == NULL) - return NULL; - } - action_obj = default_str; - } - else if (!strcmp(action, "always")) { - if (always_str == NULL) { - always_str = PyUnicode_InternFromString("always"); - if (always_str == NULL) - return NULL; - } - action_obj = always_str; - } - else { - Py_FatalError("unknown action"); + PyObject *action_str = _PyUnicode_FromId(id); + if (action_str == NULL) { + return NULL; } /* This assumes the line number is zero for now. */ - return PyTuple_Pack(5, action_obj, Py_None, + return PyTuple_Pack(5, action_str, Py_None, category, Py_None, _PyLong_Zero); } static PyObject * -init_filters(void) +init_filters(const _PyCoreConfig *config) { - PyObject *filters = PyList_New(5); - unsigned int pos = 0; /* Post-incremented in each use. */ - unsigned int x; - const char *bytes_action, *resource_action; + int dev_mode = config->dev_mode; + Py_ssize_t count = 2; + if (dev_mode) { + count++; + } +#ifndef Py_DEBUG + if (!dev_mode) { + count += 3; + } +#endif + PyObject *filters = PyList_New(count); if (filters == NULL) return NULL; - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, "ignore")); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, "ignore")); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, "ignore")); + size_t pos = 0; /* Post-incremented in each use. */ +#ifndef Py_DEBUG + if (!dev_mode) { + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, &PyId_ignore)); + } +#endif + + _Py_Identifier *bytes_action; if (Py_BytesWarningFlag > 1) - bytes_action = "error"; + bytes_action = &PyId_error; else if (Py_BytesWarningFlag) - bytes_action = "default"; + bytes_action = &PyId_default; else - bytes_action = "ignore"; + bytes_action = &PyId_ignore; PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); + + _Py_Identifier *resource_action; /* resource usage warnings are enabled by default in pydebug mode */ #ifdef Py_DEBUG - resource_action = "always"; + resource_action = &PyId_default; #else - resource_action = "ignore"; + resource_action = (dev_mode ? &PyId_default: &PyId_ignore); #endif PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, resource_action)); - for (x = 0; x < pos; x += 1) { + + if (dev_mode) { + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_Warning, &PyId_default)); + } + + for (size_t x = 0; x < pos; x++) { if (PyList_GET_ITEM(filters, x) == NULL) { Py_DECREF(filters); return NULL; @@ -1228,8 +1247,8 @@ static struct PyModuleDef warningsmodule = { }; -PyMODINIT_FUNC -_PyWarnings_Init(void) +PyObject* +_PyWarnings_InitWithConfig(const _PyCoreConfig *config) { PyObject *m; @@ -1238,7 +1257,7 @@ _PyWarnings_Init(void) return NULL; if (_PyRuntime.warnings.filters == NULL) { - _PyRuntime.warnings.filters = init_filters(); + _PyRuntime.warnings.filters = init_filters(config); if (_PyRuntime.warnings.filters == NULL) return NULL; } @@ -1269,3 +1288,12 @@ _PyWarnings_Init(void) _PyRuntime.warnings.filters_version = 0; return m; } + + +PyMODINIT_FUNC +_PyWarnings_Init(void) +{ + PyInterpreterState *interp = PyThreadState_GET()->interp; + const _PyCoreConfig *config = &interp->core_config; + return _PyWarnings_InitWithConfig(config); +} diff --git a/Python/ast.c b/Python/ast.c index 79cef708a00f99..e2092f0f85433e 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -11,6 +11,7 @@ #include "pythonrun.h" #include +#include static int validate_stmts(asdl_seq *); static int validate_exprs(asdl_seq *, expr_context_ty, int); @@ -611,7 +612,7 @@ static stmt_ty ast_for_with_stmt(struct compiling *, const node *, int); static stmt_ty ast_for_for_stmt(struct compiling *, const node *, int); /* Note different signature for ast_for_call */ -static expr_ty ast_for_call(struct compiling *, const node *, expr_ty); +static expr_ty ast_for_call(struct compiling *, const node *, expr_ty, bool); static PyObject *parsenumber(struct compiling *, const char *); static expr_ty parsestrplus(struct compiling *, const node *n); @@ -1545,7 +1546,7 @@ ast_for_decorator(struct compiling *c, const node *n) name_expr = NULL; } else { - d = ast_for_call(c, CHILD(n, 3), name_expr); + d = ast_for_call(c, CHILD(n, 3), name_expr, true); if (!d) return NULL; name_expr = NULL; @@ -2368,7 +2369,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena); else - return ast_for_call(c, CHILD(n, 1), left_expr); + return ast_for_call(c, CHILD(n, 1), left_expr, true); } else if (TYPE(CHILD(n, 0)) == DOT) { PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1)); @@ -2705,14 +2706,14 @@ ast_for_expr(struct compiling *c, const node *n) } static expr_ty -ast_for_call(struct compiling *c, const node *n, expr_ty func) +ast_for_call(struct compiling *c, const node *n, expr_ty func, bool allowgen) { /* arglist: argument (',' argument)* [','] argument: ( test [comp_for] | '*' test | test '=' test | '**' test ) */ - int i, nargs, nkeywords, ngens; + int i, nargs, nkeywords; int ndoublestars; asdl_seq *args; asdl_seq *keywords; @@ -2721,14 +2722,22 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) nargs = 0; nkeywords = 0; - ngens = 0; for (i = 0; i < NCH(n); i++) { node *ch = CHILD(n, i); if (TYPE(ch) == argument) { if (NCH(ch) == 1) nargs++; - else if (TYPE(CHILD(ch, 1)) == comp_for) - ngens++; + else if (TYPE(CHILD(ch, 1)) == comp_for) { + nargs++; + if (!allowgen) { + ast_error(c, ch, "invalid syntax"); + return NULL; + } + if (NCH(n) > 1) { + ast_error(c, ch, "Generator expression must be parenthesized"); + return NULL; + } + } else if (TYPE(CHILD(ch, 0)) == STAR) nargs++; else @@ -2736,13 +2745,8 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func) nkeywords++; } } - if (ngens > 1 || (ngens && (nargs || nkeywords))) { - ast_error(c, n, "Generator expression must be parenthesized " - "if not sole argument"); - return NULL; - } - args = _Py_asdl_seq_new(nargs + ngens, c->c_arena); + args = _Py_asdl_seq_new(nargs, c->c_arena); if (!args) return NULL; keywords = _Py_asdl_seq_new(nkeywords, c->c_arena); @@ -3974,7 +3978,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) if (!dummy_name) return NULL; dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, c->c_arena); - call = ast_for_call(c, CHILD(n, 3), dummy); + call = ast_for_call(c, CHILD(n, 3), dummy, false); if (!call) return NULL; } @@ -4156,18 +4160,19 @@ warn_invalid_escape_sequence(struct compiling *c, const node *n, } if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning, msg, c->c_filename, LINENO(n), - NULL, NULL) < 0 && - PyErr_ExceptionMatches(PyExc_DeprecationWarning)) + NULL, NULL) < 0) { - const char *s; + if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + const char *s; - /* Replace the DeprecationWarning exception with a SyntaxError - to get a more accurate error report */ - PyErr_Clear(); + /* Replace the DeprecationWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); - s = PyUnicode_AsUTF8(msg); - if (s != NULL) { - ast_error(c, n, s); + s = PyUnicode_AsUTF8(msg); + if (s != NULL) { + ast_error(c, n, s); + } } Py_DECREF(msg); return -1; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 6215a638c94b71..0bda04344cc1a5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -46,14 +46,86 @@ _Py_IDENTIFIER(stderr); #include "clinic/bltinmodule.c.h" +static PyObject* +update_bases(PyObject* bases, PyObject** args, int nargs, int* modified_bases) +{ + int i, j, ind, tot_extra = 0; + PyObject *base, *meth, *new_base, *new_sub_base, *new_bases; + PyObject *stack[1] = {bases}; + assert(PyTuple_Check(bases)); + + /* We have a separate cycle to calculate replacements with the idea that in + most cases we just scroll quickly though it and return original bases */ + for (i = 2; i < nargs; i++){ + base = args[i]; + if (PyType_Check(base)) { + continue; + } + if (!(meth = PyObject_GetAttrString(base, "__mro_entries__"))) { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + return NULL; + } + PyErr_Clear(); + continue; + } + if (!PyCallable_Check(meth)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must be callable"); + Py_DECREF(meth); + return NULL; + } + if (!(new_base = _PyObject_FastCall(meth, stack, 1))){ + Py_DECREF(meth); + return NULL; + } + if (PyTuple_Check(new_base)) { + tot_extra += PyTuple_Size(new_base) - 1; + } + else { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(meth); + return NULL; + } + Py_DECREF(base); + args[i] = new_base; + *modified_bases = 1; + Py_DECREF(meth); + } + if (!*modified_bases){ + return bases; + } + new_bases = PyTuple_New(nargs - 2 + tot_extra); + ind = 0; + for (i = 2; i < nargs; i++) { + new_base = args[i]; + if (!PyTuple_Check(new_base)) { + Py_INCREF(new_base); + PyTuple_SET_ITEM(new_bases, ind, new_base); + ind++; + } + else { + for (j = 0; j < PyTuple_Size(new_base); j++) { + new_sub_base = PyTuple_GET_ITEM(new_base, j); + Py_INCREF(new_sub_base); + PyTuple_SET_ITEM(new_bases, ind, new_sub_base); + ind++; + } + } + } + return new_bases; +} + /* AC: cannot convert yet, waiting for *args support */ static PyObject * builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *func, *name, *bases, *mkw, *meta, *winner, *prep, *ns; + PyObject *new_bases, *old_bases = NULL; PyObject *cls = NULL, *cell = NULL; int isclass = 0; /* initialize to prevent gcc warning */ + int modified_bases = 0; if (nargs < 2) { PyErr_SetString(PyExc_TypeError, @@ -76,6 +148,16 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, if (bases == NULL) return NULL; + new_bases = update_bases(bases, args, nargs, &modified_bases); + if (new_bases == NULL) { + Py_DECREF(bases); + return NULL; + } + else { + old_bases = bases; + bases = new_bases; + } + if (kwnames == NULL) { meta = NULL; mkw = NULL; @@ -168,6 +250,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, NULL, 0, NULL, 0, NULL, 0, NULL, PyFunction_GET_CLOSURE(func)); if (cell != NULL) { + if (modified_bases){ + PyMapping_SetItemString(ns, "__orig_bases__", old_bases); + } PyObject *margs[3] = {name, bases, ns}; cls = _PyObject_FastCallDict(meta, margs, 3, mkw); if (cls != NULL && PyType_Check(cls) && PyCell_Check(cell)) { @@ -206,6 +291,9 @@ builtin___build_class__(PyObject *self, PyObject **args, Py_ssize_t nargs, Py_DECREF(meta); Py_XDECREF(mkw); Py_DECREF(bases); + if (modified_bases) { + Py_DECREF(old_bases); + } return cls; } @@ -1942,7 +2030,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) /* If we're interactive, use (GNU) readline */ if (tty) { PyObject *po = NULL; - char *promptstr; + const char *promptstr; char *s = NULL; PyObject *stdin_encoding = NULL, *stdin_errors = NULL; PyObject *stdout_encoding = NULL, *stdout_errors = NULL; @@ -2079,19 +2167,23 @@ builtin_repr(PyObject *module, PyObject *obj) } -/* AC: cannot convert yet, as needs PEP 457 group support in inspect - * or a semantic change to accept None for "ndigits" - */ +/*[clinic input] +round as builtin_round + + number: object + ndigits: object = NULL + +Round a number to a given precision in decimal digits. + +The return value is an integer if ndigits is omitted or None. Otherwise +the return value has the same type as the number. ndigits may be negative. +[clinic start generated code]*/ + static PyObject * -builtin_round(PyObject *self, PyObject *args, PyObject *kwds) +builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) +/*[clinic end generated code: output=ff0d9dd176c02ede input=854bc3a217530c3d]*/ { - PyObject *ndigits = NULL; - static char *kwlist[] = {"number", "ndigits", 0}; - PyObject *number, *round, *result; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:round", - kwlist, &number, &ndigits)) - return NULL; + PyObject *round, *result; if (Py_TYPE(number)->tp_dict == NULL) { if (PyType_Ready(Py_TYPE(number)) < 0) @@ -2115,13 +2207,6 @@ builtin_round(PyObject *self, PyObject *args, PyObject *kwds) return result; } -PyDoc_STRVAR(round_doc, -"round(number[, ndigits]) -> number\n\ -\n\ -Round a number to a given precision in decimal digits (default 0 digits).\n\ -This returns an int when called with one argument, otherwise the\n\ -same type as the number. ndigits may be negative."); - /*AC: we need to keep the kwds dict intact to easily call into the * list.sort method, which isn't currently supported in AC. So we just use @@ -2679,7 +2764,7 @@ static PyMethodDef builtin_methods[] = { BUILTIN_POW_METHODDEF {"print", (PyCFunction)builtin_print, METH_FASTCALL | METH_KEYWORDS, print_doc}, BUILTIN_REPR_METHODDEF - {"round", (PyCFunction)builtin_round, METH_VARARGS | METH_KEYWORDS, round_doc}, + BUILTIN_ROUND_METHODDEF BUILTIN_SETATTR_METHODDEF BUILTIN_SORTED_METHODDEF BUILTIN_SUM_METHODDEF diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 807d6023ce4292..610541d810b4c9 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -561,15 +561,16 @@ int Py_ReadHashSeed(char *seed_text, return 0; } -static void +static _PyInitError init_hash_secret(int use_hash_seed, unsigned long hash_seed) { void *secret = &_Py_HashSecret; Py_ssize_t secret_size = sizeof(_Py_HashSecret_t); - if (_Py_HashSecret_Initialized) - return; + if (_Py_HashSecret_Initialized) { + return _Py_INIT_OK(); + } _Py_HashSecret_Initialized = 1; if (use_hash_seed) { @@ -593,12 +594,14 @@ init_hash_secret(int use_hash_seed, pyurandom() is non-blocking mode (blocking=0): see the PEP 524. */ res = pyurandom(secret, secret_size, 0, 0); if (res < 0) { - Py_FatalError("failed to get random numbers to initialize Python"); + return _Py_INIT_USER_ERR("failed to get random numbers " + "to initialize Python"); } } + return _Py_INIT_OK(); } -void +_PyInitError _Py_HashRandomization_Init(_PyCoreConfig *core_config) { char *seed_text; @@ -608,13 +611,13 @@ _Py_HashRandomization_Init(_PyCoreConfig *core_config) if (use_hash_seed < 0) { seed_text = Py_GETENV("PYTHONHASHSEED"); if (Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) { - Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer " - "in range [0; 4294967295]"); + return _Py_INIT_USER_ERR("PYTHONHASHSEED must be \"random\" " + "or an integer in range [0; 4294967295]"); } core_config->use_hash_seed = use_hash_seed; core_config->hash_seed = hash_seed; } - init_hash_secret(use_hash_seed, hash_seed); + return init_hash_secret(use_hash_seed, hash_seed); } void diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index fa327da0ffc352..3647e62c0d3ce5 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -573,6 +573,40 @@ PyDoc_STRVAR(builtin_repr__doc__, #define BUILTIN_REPR_METHODDEF \ {"repr", (PyCFunction)builtin_repr, METH_O, builtin_repr__doc__}, +PyDoc_STRVAR(builtin_round__doc__, +"round($module, /, number, ndigits=None)\n" +"--\n" +"\n" +"Round a number to a given precision in decimal digits.\n" +"\n" +"The return value is an integer if ndigits is omitted or None. Otherwise\n" +"the return value has the same type as the number. ndigits may be negative."); + +#define BUILTIN_ROUND_METHODDEF \ + {"round", (PyCFunction)builtin_round, METH_FASTCALL|METH_KEYWORDS, builtin_round__doc__}, + +static PyObject * +builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits); + +static PyObject * +builtin_round(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"number", "ndigits", NULL}; + static _PyArg_Parser _parser = {"O|O:round", _keywords, 0}; + PyObject *number; + PyObject *ndigits = NULL; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &number, &ndigits)) { + goto exit; + } + return_value = builtin_round_impl(module, number, ndigits); + +exit: + return return_value; +} + PyDoc_STRVAR(builtin_sum__doc__, "sum($module, iterable, start=0, /)\n" "--\n" @@ -676,4 +710,4 @@ builtin_issubclass(PyObject *module, PyObject **args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=09752daa8cdd6ec7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d46a224ac804eef1 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 58a708ce23eb02..a3fcd53e9a24cb 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -4846,7 +4846,7 @@ compiler_visit_nested_slice(struct compiler *c, slice_ty s, static int compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx) { - char * kindname = NULL; + const char * kindname = NULL; switch (s->kind) { case Index_kind: kindname = "index"; @@ -5273,11 +5273,6 @@ compute_code_flags(struct compiler *c) /* (Only) inherit compilerflags in PyCF_MASK */ flags |= (c->c_flags->cf_flags & PyCF_MASK); - if (!PyDict_GET_SIZE(c->u->u_freevars) && - !PyDict_GET_SIZE(c->u->u_cellvars)) { - flags |= CO_NOFREE; - } - return flags; } diff --git a/Python/dup2.c b/Python/dup2.c index 2579afd443dca8..7c6bbfce11dbf8 100644 --- a/Python/dup2.c +++ b/Python/dup2.c @@ -19,13 +19,13 @@ int dup2(int fd1, int fd2) { - if (fd1 != fd2) { - if (fcntl(fd1, F_GETFL) < 0) - return BADEXIT; - if (fcntl(fd2, F_GETFL) >= 0) - close(fd2); - if (fcntl(fd1, F_DUPFD, fd2) < 0) - return BADEXIT; - } - return fd2; + if (fd1 != fd2) { + if (fcntl(fd1, F_GETFL) < 0) + return BADEXIT; + if (fcntl(fd2, F_GETFL) >= 0) + close(fd2); + if (fcntl(fd1, F_DUPFD, fd2) < 0) + return BADEXIT; + } + return fd2; } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 769b33d0ee2f79..77602d70fa89ff 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -2,6 +2,7 @@ /* Python interpreter main program for frozen scripts */ #include "Python.h" +#include "internal/pystate.h" #include #ifdef MS_WINDOWS @@ -15,6 +16,13 @@ extern int PyInitFrozenExtensions(void); int Py_FrozenMain(int argc, char **argv) { + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + fprintf(stderr, "Fatal Python error: %s\n", err.msg); + fflush(stderr); + exit(1); + } + char *p; int i, n, sts = 1; int inspect = 0; diff --git a/Python/getplatform.c b/Python/getplatform.c index 68991402b5b000..81a0f7ac5378bd 100644 --- a/Python/getplatform.c +++ b/Python/getplatform.c @@ -8,5 +8,5 @@ const char * Py_GetPlatform(void) { - return PLATFORM; + return PLATFORM; } diff --git a/Python/getversion.c b/Python/getversion.c index 7bd6efd0a01547..c32b6f9d60d4b7 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -8,8 +8,8 @@ const char * Py_GetVersion(void) { - static char version[250]; - PyOS_snprintf(version, sizeof(version), "%.80s (%.80s) %.80s", - PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); - return version; + static char version[250]; + PyOS_snprintf(version, sizeof(version), "%.80s (%.80s) %.80s", + PY_VERSION, Py_GetBuildInfo(), Py_GetCompiler()); + return version; } diff --git a/Python/import.c b/Python/import.c index 950c872ccb88d3..57521e4920715c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -42,19 +42,23 @@ module _imp /* Initialize things */ -void +_PyInitError _PyImport_Init(void) { PyInterpreterState *interp = PyThreadState_Get()->interp; initstr = PyUnicode_InternFromString("__init__"); - if (initstr == NULL) - Py_FatalError("Can't initialize import variables"); + if (initstr == NULL) { + return _Py_INIT_ERR("Can't initialize import variables"); + } + interp->builtins_copy = PyDict_Copy(interp->builtins); - if (interp->builtins_copy == NULL) - Py_FatalError("Can't backup builtins dict"); + if (interp->builtins_copy == NULL) { + return _Py_INIT_ERR("Can't backup builtins dict"); + } + return _Py_INIT_OK(); } -void +_PyInitError _PyImportHooks_Init(void) { PyObject *v, *path_hooks = NULL; @@ -80,15 +84,18 @@ _PyImportHooks_Init(void) goto error; err = PySys_SetObject("path_hooks", path_hooks); if (err) { - error: - PyErr_Print(); - Py_FatalError("initializing sys.meta_path, sys.path_hooks, " - "or path_importer_cache failed"); + goto error; } Py_DECREF(path_hooks); + return _Py_INIT_OK(); + + error: + PyErr_Print(); + return _Py_INIT_ERR("initializing sys.meta_path, sys.path_hooks, " + "or path_importer_cache failed"); } -void +_PyInitError _PyImportZip_Init(void) { PyObject *path_hooks, *zimpimport; @@ -133,11 +140,11 @@ _PyImportZip_Init(void) } } - return; + return _Py_INIT_OK(); error: PyErr_Print(); - Py_FatalError("initializing zipimport failed"); + return _Py_INIT_ERR("initializing zipimport failed"); } /* Locking primitives to prevent parallel imports of the same module @@ -1582,12 +1589,67 @@ resolve_name(PyObject *name, PyObject *globals, int level) return NULL; } +static PyObject * +import_find_and_load(PyObject *abs_name) +{ + _Py_IDENTIFIER(_find_and_load); + PyObject *mod = NULL; + PyInterpreterState *interp = PyThreadState_GET()->interp; + int import_time = interp->core_config.import_time; + static int import_level; + static _PyTime_t accumulated; + + _PyTime_t t1 = 0, accumulated_copy = accumulated; + + /* XOptions is initialized after first some imports. + * So we can't have negative cache before completed initialization. + * Anyway, importlib._find_and_load is much slower than + * _PyDict_GetItemIdWithError(). + */ + if (import_time) { + static int header = 1; + if (header) { + fputs("import time: self [us] | cumulative | imported package\n", + stderr); + header = 0; + } + + import_level++; + t1 = _PyTime_GetPerfCounter(); + accumulated = 0; + } + + if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) + PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); + + mod = _PyObject_CallMethodIdObjArgs(interp->importlib, + &PyId__find_and_load, abs_name, + interp->import_func, NULL); + + if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) + PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), + mod != NULL); + + if (import_time) { + _PyTime_t cum = _PyTime_GetPerfCounter() - t1; + + import_level--; + fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", + (long)_PyTime_AsMicroseconds(cum - accumulated, _PyTime_ROUND_CEILING), + (long)_PyTime_AsMicroseconds(cum, _PyTime_ROUND_CEILING), + import_level*2, "", PyUnicode_AsUTF8(abs_name)); + + accumulated = accumulated_copy + cum; + } + + return mod; +} + PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - _Py_IDENTIFIER(_find_and_load); _Py_IDENTIFIER(_handle_fromlist); PyObject *abs_name = NULL; PyObject *final_mod = NULL; @@ -1667,75 +1729,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } } else { - /* 1 -- true, 0 -- false, -1 -- not initialized */ - static int ximporttime = -1; - static int import_level; - static _PyTime_t accumulated; - _Py_IDENTIFIER(importtime); - - _PyTime_t t1 = 0, accumulated_copy = accumulated; - - /* XOptions is initialized after first some imports. - * So we can't have negative cache before completed initialization. - * Anyway, importlib._find_and_load is much slower than - * _PyDict_GetItemIdWithError(). - */ - if (ximporttime < 0) { - const char *envoption = Py_GETENV("PYTHONPROFILEIMPORTTIME"); - if (envoption != NULL && *envoption != '\0') { - ximporttime = 1; - } - else { - PyObject *xoptions = PySys_GetXOptions(); - PyObject *value = NULL; - if (xoptions) { - value = _PyDict_GetItemIdWithError( - xoptions, &PyId_importtime); - } - if (value == NULL && PyErr_Occurred()) { - goto error; - } - if (value != NULL || Py_IsInitialized()) { - ximporttime = (value == Py_True); - } - } - if (ximporttime > 0) { - fputs("import time: self [us] | cumulative | imported package\n", - stderr); - } - } - - if (ximporttime > 0) { - import_level++; - t1 = _PyTime_GetPerfCounter(); - accumulated = 0; - } - - Py_XDECREF(mod); - - if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) - PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - - mod = _PyObject_CallMethodIdObjArgs(interp->importlib, - &PyId__find_and_load, abs_name, - interp->import_func, NULL); - - if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) - PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), - mod != NULL); - - if (ximporttime > 0) { - _PyTime_t cum = _PyTime_GetPerfCounter() - t1; - - import_level--; - fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", - (long)_PyTime_AsMicroseconds(cum - accumulated, _PyTime_ROUND_CEILING), - (long)_PyTime_AsMicroseconds(cum, _PyTime_ROUND_CEILING), - import_level*2, "", PyUnicode_AsUTF8(abs_name)); - - accumulated = accumulated_copy + cum; - } - + mod = import_find_and_load(abs_name); if (mod == NULL) { goto error; } diff --git a/Python/marshal.c b/Python/marshal.c index 7b583eef1a1ffa..e23daf6497b455 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -39,6 +39,9 @@ module marshal #define TYPE_STOPITER 'S' #define TYPE_ELLIPSIS '.' #define TYPE_INT 'i' +/* TYPE_INT64 is not generated anymore. + Supported for backward compatibility only. */ +#define TYPE_INT64 'I' #define TYPE_FLOAT 'f' #define TYPE_BINARY_FLOAT 'g' #define TYPE_COMPLEX 'x' @@ -783,6 +786,19 @@ r_long(RFILE *p) return x; } +/* r_long64 deals with the TYPE_INT64 code. */ +static PyObject * +r_long64(RFILE *p) +{ + const unsigned char *buffer = (const unsigned char *) r_string(8, p); + if (buffer == NULL) { + return NULL; + } + return _PyLong_FromByteArray(buffer, 8, + 1 /* little endian */, + 1 /* signed */); +} + static PyObject * r_PyLong(RFILE *p) { @@ -982,6 +998,11 @@ r_object(RFILE *p) R_REF(retval); break; + case TYPE_INT64: + retval = r_long64(p); + R_REF(retval); + break; + case TYPE_LONG: retval = r_PyLong(p); R_REF(retval); diff --git a/Python/pathconfig.c b/Python/pathconfig.c new file mode 100644 index 00000000000000..6a03f7dca1ba7a --- /dev/null +++ b/Python/pathconfig.c @@ -0,0 +1,266 @@ +/* Path configuration like module_search_path (sys.path) */ + +#include "Python.h" +#include "osdefs.h" +#include "internal/pystate.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +_PyPathConfig _Py_path_config = _PyPathConfig_INIT; + + +void +_PyPathConfig_Clear(_PyPathConfig *config) +{ + /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, + since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be + called before Py_Initialize() which can changes the memory allocator. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->prefix); + CLEAR(config->program_full_path); +#ifdef MS_WINDOWS + CLEAR(config->dll_path); +#else + CLEAR(config->exec_prefix); +#endif + CLEAR(config->module_search_path); + CLEAR(config->home); + CLEAR(config->program_name); +#undef CLEAR + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +/* Initialize paths for Py_GetPath(), Py_GetPrefix(), Py_GetExecPrefix() + and Py_GetProgramFullPath() */ +_PyInitError +_PyPathConfig_Init(const _PyMainInterpreterConfig *main_config) +{ + if (_Py_path_config.module_search_path) { + /* Already initialized */ + return _Py_INIT_OK(); + } + + _PyInitError err; + _PyPathConfig new_config = _PyPathConfig_INIT; + + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + /* Calculate program_full_path, prefix, exec_prefix (Unix) + or dll_path (Windows), and module_search_path */ + err = _PyPathConfig_Calculate(&new_config, main_config); + if (_Py_INIT_FAILED(err)) { + _PyPathConfig_Clear(&new_config); + goto done; + } + + /* Copy home and program_name from main_config */ + if (main_config->home != NULL) { + new_config.home = _PyMem_RawWcsdup(main_config->home); + if (new_config.home == NULL) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + } + else { + new_config.home = NULL; + } + + new_config.program_name = _PyMem_RawWcsdup(main_config->program_name); + if (new_config.program_name == NULL) { + err = _Py_INIT_NO_MEMORY(); + goto done; + } + + _PyPathConfig_Clear(&_Py_path_config); + _Py_path_config = new_config; + + err = _Py_INIT_OK(); + +done: + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return err; +} + + +static void +pathconfig_global_init(void) +{ + if (_Py_path_config.module_search_path) { + /* Already initialized */ + return; + } + + _PyInitError err; + _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + err = _PyMainInterpreterConfig_Read(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + err = _PyPathConfig_Init(&config); + if (_Py_INIT_FAILED(err)) { + goto error; + } + + _PyMainInterpreterConfig_Clear(&config); + return; + +error: + _PyMainInterpreterConfig_Clear(&config); + _Py_FatalInitError(err); +} + + +/* External interface */ + +void +Py_SetPath(const wchar_t *path) +{ + if (path == NULL) { + _PyPathConfig_Clear(&_Py_path_config); + return; + } + + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + _PyPathConfig new_config; + new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); + new_config.prefix = _PyMem_RawWcsdup(L""); +#ifdef MS_WINDOWS + new_config.dll_path = _PyMem_RawWcsdup(L""); +#else + new_config.exec_prefix = _PyMem_RawWcsdup(L""); +#endif + new_config.module_search_path = _PyMem_RawWcsdup(path); + + /* steal the home and program_name values (to leave them unchanged) */ + new_config.home = _Py_path_config.home; + _Py_path_config.home = NULL; + new_config.program_name = _Py_path_config.program_name; + _Py_path_config.program_name = NULL; + + _PyPathConfig_Clear(&_Py_path_config); + _Py_path_config = new_config; + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); +} + + +void +Py_SetPythonHome(wchar_t *home) +{ + if (home == NULL) { + return; + } + + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + PyMem_RawFree(_Py_path_config.home); + _Py_path_config.home = _PyMem_RawWcsdup(home); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_Py_path_config.home == NULL) { + Py_FatalError("Py_SetPythonHome() failed: out of memory"); + } +} + + +void +Py_SetProgramName(wchar_t *program_name) +{ + if (program_name == NULL || program_name[0] == L'\0') { + return; + } + + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + PyMem_RawFree(_Py_path_config.program_name); + _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + if (_Py_path_config.program_name == NULL) { + Py_FatalError("Py_SetProgramName() failed: out of memory"); + } +} + + +wchar_t * +Py_GetPath(void) +{ + pathconfig_global_init(); + return _Py_path_config.module_search_path; +} + + +wchar_t * +Py_GetPrefix(void) +{ + pathconfig_global_init(); + return _Py_path_config.prefix; +} + + +wchar_t * +Py_GetExecPrefix(void) +{ +#ifdef MS_WINDOWS + return Py_GetPrefix(); +#else + pathconfig_global_init(); + return _Py_path_config.exec_prefix; +#endif +} + + +wchar_t * +Py_GetProgramFullPath(void) +{ + pathconfig_global_init(); + return _Py_path_config.program_full_path; +} + + +wchar_t* +Py_GetPythonHome(void) +{ + pathconfig_global_init(); + return _Py_path_config.home; +} + + +wchar_t * +Py_GetProgramName(void) +{ + pathconfig_global_init(); + return _Py_path_config.program_name; +} + + +#ifdef __cplusplus +} +#endif diff --git a/Python/pyfpe.c b/Python/pyfpe.c index ab0ef83ead094a..925fa4654d9fc0 100644 --- a/Python/pyfpe.c +++ b/Python/pyfpe.c @@ -19,5 +19,5 @@ int PyFPE_counter = 0; double PyFPE_dummy(void *dummy) { - return 1.0; + return 1.0; } diff --git a/Python/pyhash.c b/Python/pyhash.c index 8a6bd60c105b6f..aa49eeb35bb325 100644 --- a/Python/pyhash.c +++ b/Python/pyhash.c @@ -196,7 +196,7 @@ _PyHash_Fini(void) #ifdef Py_HASH_STATS int i; Py_ssize_t total = 0; - char *fmt = "%2i %8" PY_FORMAT_SIZE_T "d %8" PY_FORMAT_SIZE_T "d\n"; + const char *fmt = "%2i %8" PY_FORMAT_SIZE_T "d %8" PY_FORMAT_SIZE_T "d\n"; fprintf(stderr, "len calls total\n"); for (i = 1; i <= Py_HASH_STATS_MAX; i++) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 4e8bbb662e971e..b615c799989364 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -48,16 +48,14 @@ _Py_IDENTIFIER(threading); extern "C" { #endif -extern wchar_t *Py_GetPath(void); - extern grammar _PyParser_Grammar; /* From graminit.c */ /* Forward */ -static void initmain(PyInterpreterState *interp); -static int initfsencoding(PyInterpreterState *interp); -static void initsite(void); -static int initstdio(void); -static void initsigs(void); +static _PyInitError add_main_module(PyInterpreterState *interp); +static _PyInitError initfsencoding(PyInterpreterState *interp); +static _PyInitError initsite(void); +static _PyInitError init_sys_streams(void); +static _PyInitError initsigs(void); static void call_py_exitfuncs(void); static void wait_for_thread_shutdown(void); static void call_ll_exitfuncs(void); @@ -66,19 +64,19 @@ extern int _PyStructSequence_Init(void); extern void _PyUnicode_Fini(void); extern int _PyLong_Init(void); extern void PyLong_Fini(void); -extern int _PyFaulthandler_Init(void); +extern _PyInitError _PyFaulthandler_Init(int enable); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); -extern int _PyTraceMalloc_Init(void); +extern int _PyTraceMalloc_Init(int enable); extern int _PyTraceMalloc_Fini(void); extern void _Py_ReadyTypes(void); extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); extern void _PyGILState_Fini(void); -_PyRuntimeState _PyRuntime = {0, 0}; +_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT; -void +_PyInitError _PyRuntime_Initialize(void) { /* XXX We only initialize once in the process, which aligns with @@ -88,10 +86,12 @@ _PyRuntime_Initialize(void) This is because the runtime state is not properly finalized currently. */ static int initialized = 0; - if (initialized) - return; + if (initialized) { + return _Py_INIT_OK(); + } initialized = 1; - _PyRuntimeState_Init(&_PyRuntime); + + return _PyRuntimeState_Init(&_PyRuntime); } void @@ -116,7 +116,6 @@ int Py_InspectFlag; /* Needed to determine whether to exit at SystemExit */ int Py_OptimizeFlag = 0; /* Needed by compile.c */ int Py_NoSiteFlag; /* Suppress 'import site' */ int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ -int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.pyc) */ @@ -217,20 +216,6 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) */ -static void -set_flag(int *flag, const char *envs) -{ - /* Helper to set flag variables from environment variables: - * - uses the higher of the two values if they're both set - * - otherwise sets the flag to 1 - */ - int env = atoi(envs); - if (*flag < env) - *flag = env; - if (*flag < 1) - *flag = 1; -} - static char* get_codec_name(const char *encoding) { @@ -282,43 +267,43 @@ get_locale_encoding(void) #endif } -static void +static _PyInitError initimport(PyInterpreterState *interp, PyObject *sysmod) { PyObject *importlib; PyObject *impmod; PyObject *value; + _PyInitError err; /* Import _importlib through its frozen version, _frozen_importlib. */ if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) { - Py_FatalError("Py_Initialize: can't import _frozen_importlib"); + return _Py_INIT_ERR("can't import _frozen_importlib"); } else if (Py_VerboseFlag) { PySys_FormatStderr("import _frozen_importlib # frozen\n"); } importlib = PyImport_AddModule("_frozen_importlib"); if (importlib == NULL) { - Py_FatalError("Py_Initialize: couldn't get _frozen_importlib from " - "sys.modules"); + return _Py_INIT_ERR("couldn't get _frozen_importlib from sys.modules"); } interp->importlib = importlib; Py_INCREF(interp->importlib); interp->import_func = PyDict_GetItemString(interp->builtins, "__import__"); if (interp->import_func == NULL) - Py_FatalError("Py_Initialize: __import__ not found"); + return _Py_INIT_ERR("__import__ not found"); Py_INCREF(interp->import_func); /* Import the _imp module */ impmod = PyInit_imp(); if (impmod == NULL) { - Py_FatalError("Py_Initialize: can't import _imp"); + return _Py_INIT_ERR("can't import _imp"); } else if (Py_VerboseFlag) { PySys_FormatStderr("import _imp # builtin\n"); } if (_PyImport_SetModuleString("_imp", impmod) < 0) { - Py_FatalError("Py_Initialize: can't save _imp to sys.modules"); + return _Py_INIT_ERR("can't save _imp to sys.modules"); } /* Install importlib as the implementation of import */ @@ -330,15 +315,20 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) } if (value == NULL) { PyErr_Print(); - Py_FatalError("Py_Initialize: importlib install failed"); + return _Py_INIT_ERR("importlib install failed"); } Py_DECREF(value); Py_DECREF(impmod); - _PyImportZip_Init(); + err = _PyImportZip_Init(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + return _Py_INIT_OK(); } -static void +static _PyInitError initexternalimport(PyInterpreterState *interp) { PyObject *value; @@ -346,9 +336,10 @@ initexternalimport(PyInterpreterState *interp) "_install_external_importers", ""); if (value == NULL) { PyErr_Print(); - Py_FatalError("Py_EndInitialization: external importer setup failed"); + return _Py_INIT_ERR("external importer setup failed"); } Py_DECREF(value); + return _Py_INIT_OK(); } /* Helper functions to better handle the legacy C locale @@ -459,7 +450,7 @@ _coerce_default_locale_settings(const _LocaleCoercionTarget *target) const char *newloc = target->locale_name; /* Reset locale back to currently configured defaults */ - setlocale(LC_ALL, ""); + _Py_SetLocaleFromEnv(LC_ALL); /* Set the relevant locale environment variable */ if (setenv("LC_CTYPE", newloc, 1)) { @@ -472,7 +463,7 @@ _coerce_default_locale_settings(const _LocaleCoercionTarget *target) } /* Reconfigure with the overridden environment variables */ - setlocale(LC_ALL, ""); + _Py_SetLocaleFromEnv(LC_ALL); } #endif @@ -503,13 +494,14 @@ _Py_CoerceLegacyLocale(void) const char *new_locale = setlocale(LC_CTYPE, target->locale_name); if (new_locale != NULL) { -#if !defined(__APPLE__) && defined(HAVE_LANGINFO_H) && defined(CODESET) +#if !defined(__APPLE__) && !defined(__ANDROID__) && \ + defined(HAVE_LANGINFO_H) && defined(CODESET) /* Also ensure that nl_langinfo works in this locale */ char *codeset = nl_langinfo(CODESET); if (!codeset || *codeset == '\0') { /* CODESET is not set or empty, so skip coercion */ new_locale = NULL; - setlocale(LC_CTYPE, ""); + _Py_SetLocaleFromEnv(LC_CTYPE); continue; } #endif @@ -524,6 +516,65 @@ _Py_CoerceLegacyLocale(void) #endif } +/* _Py_SetLocaleFromEnv() is a wrapper around setlocale(category, "") to + * isolate the idiosyncrasies of different libc implementations. It reads the + * appropriate environment variable and uses its value to select the locale for + * 'category'. */ +char * +_Py_SetLocaleFromEnv(int category) +{ +#ifdef __ANDROID__ + const char *locale; + const char **pvar; +#ifdef PY_COERCE_C_LOCALE + const char *coerce_c_locale; +#endif + const char *utf8_locale = "C.UTF-8"; + const char *env_var_set[] = { + "LC_ALL", + "LC_CTYPE", + "LANG", + NULL, + }; + + /* Android setlocale(category, "") doesn't check the environment variables + * and incorrectly sets the "C" locale at API 24 and older APIs. We only + * check the environment variables listed in env_var_set. */ + for (pvar=env_var_set; *pvar; pvar++) { + locale = getenv(*pvar); + if (locale != NULL && *locale != '\0') { + if (strcmp(locale, utf8_locale) == 0 || + strcmp(locale, "en_US.UTF-8") == 0) { + return setlocale(category, utf8_locale); + } + return setlocale(category, "C"); + } + } + + /* Android uses UTF-8, so explicitly set the locale to C.UTF-8 if none of + * LC_ALL, LC_CTYPE, or LANG is set to a non-empty string. + * Quote from POSIX section "8.2 Internationalization Variables": + * "4. If the LANG environment variable is not set or is set to the empty + * string, the implementation-defined default locale shall be used." */ + +#ifdef PY_COERCE_C_LOCALE + coerce_c_locale = getenv("PYTHONCOERCECLOCALE"); + if (coerce_c_locale == NULL || strcmp(coerce_c_locale, "0") != 0) { + /* Some other ported code may check the environment variables (e.g. in + * extension modules), so we make sure that they match the locale + * configuration */ + if (setenv("LC_CTYPE", utf8_locale, 1)) { + fprintf(stderr, "Warning: failed setting the LC_CTYPE " + "environment variable to %s\n", utf8_locale); + } + } +#endif + return setlocale(category, utf8_locale); +#else /* __ANDROID__ */ + return setlocale(category, ""); +#endif /* __ANDROID__ */ +} + /* Global initializations. Can be undone by Py_Finalize(). Don't call this twice without an intervening Py_Finalize() call. @@ -555,30 +606,34 @@ _Py_CoerceLegacyLocale(void) * safe to call without calling Py_Initialize first) */ -/* TODO: Progressively move functionality from Py_BeginInitialization to - * Py_ReadConfig and Py_EndInitialization - */ - -void _Py_InitializeCore(const _PyCoreConfig *config) +_PyInitError +_Py_InitializeCore(const _PyCoreConfig *config) { PyInterpreterState *interp; PyThreadState *tstate; PyObject *bimod, *sysmod, *pstderr; - char *p; _PyCoreConfig core_config = _PyCoreConfig_INIT; _PyMainInterpreterConfig preinit_config = _PyMainInterpreterConfig_INIT; + _PyInitError err; - _PyRuntime_Initialize(); + err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } if (config != NULL) { core_config = *config; } + if (_PyMem_SetupAllocators(core_config.allocator) < 0) { + return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); + } + if (_PyRuntime.initialized) { - Py_FatalError("Py_InitializeCore: main interpreter already initialized"); + return _Py_INIT_ERR("main interpreter already initialized"); } if (_PyRuntime.core_initialized) { - Py_FatalError("Py_InitializeCore: runtime core already initialized"); + return _Py_INIT_ERR("runtime core already initialized"); } /* Py_Finalize leaves _Py_Finalizing set in order to help daemon @@ -592,69 +647,38 @@ void _Py_InitializeCore(const _PyCoreConfig *config) */ _PyRuntime.finalizing = NULL; - if (_PyMem_SetupAllocators(core_config.allocator) < 0) { - fprintf(stderr, - "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", - core_config.allocator); - exit(1); - } - -#ifdef __ANDROID__ - /* Passing "" to setlocale() on Android requests the C locale rather - * than checking environment variables, so request C.UTF-8 explicitly - */ - setlocale(LC_CTYPE, "C.UTF-8"); -#else #ifndef MS_WINDOWS /* Set up the LC_CTYPE locale, so we can obtain the locale's charset without having to switch locales. */ - setlocale(LC_CTYPE, ""); + _Py_SetLocaleFromEnv(LC_CTYPE); _emit_stderr_warning_for_legacy_locale(); -#endif #endif - if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0') - set_flag(&Py_DebugFlag, p); - if ((p = Py_GETENV("PYTHONVERBOSE")) && *p != '\0') - set_flag(&Py_VerboseFlag, p); - if ((p = Py_GETENV("PYTHONOPTIMIZE")) && *p != '\0') - set_flag(&Py_OptimizeFlag, p); - if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') - set_flag(&Py_InspectFlag, p); - if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0') - set_flag(&Py_DontWriteBytecodeFlag, p); - if ((p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0') - set_flag(&Py_NoUserSiteDirectory, p); - if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') - set_flag(&Py_UnbufferedStdioFlag, p); - /* The variable is only tested for existence here; - _Py_HashRandomization_Init will check its value further. */ - if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0') - set_flag(&Py_HashRandomizationFlag, p); -#ifdef MS_WINDOWS - if ((p = Py_GETENV("PYTHONLEGACYWINDOWSFSENCODING")) && *p != '\0') - set_flag(&Py_LegacyWindowsFSEncodingFlag, p); - if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0') - set_flag(&Py_LegacyWindowsStdioFlag, p); -#endif + err = _Py_HashRandomization_Init(&core_config); + if (_Py_INIT_FAILED(err)) { + return err; + } - _Py_HashRandomization_Init(&core_config); if (!core_config.use_hash_seed || core_config.hash_seed) { /* Random or non-zero hash seed */ Py_HashRandomizationFlag = 1; } - _PyInterpreterState_Enable(&_PyRuntime); + err = _PyInterpreterState_Enable(&_PyRuntime); + if (_Py_INIT_FAILED(err)) { + return err; + } + interp = PyInterpreterState_New(); if (interp == NULL) - Py_FatalError("Py_InitializeCore: can't make main interpreter"); + return _Py_INIT_ERR("can't make main interpreter"); interp->core_config = core_config; interp->config = preinit_config; tstate = PyThreadState_New(interp); if (tstate == NULL) - Py_FatalError("Py_InitializeCore: can't make first thread"); + return _Py_INIT_ERR("can't make first thread"); (void) PyThreadState_Swap(tstate); /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because @@ -669,46 +693,50 @@ void _Py_InitializeCore(const _PyCoreConfig *config) _Py_ReadyTypes(); if (!_PyFrame_Init()) - Py_FatalError("Py_InitializeCore: can't init frames"); + return _Py_INIT_ERR("can't init frames"); if (!_PyLong_Init()) - Py_FatalError("Py_InitializeCore: can't init longs"); + return _Py_INIT_ERR("can't init longs"); if (!PyByteArray_Init()) - Py_FatalError("Py_InitializeCore: can't init bytearray"); + return _Py_INIT_ERR("can't init bytearray"); if (!_PyFloat_Init()) - Py_FatalError("Py_InitializeCore: can't init float"); + return _Py_INIT_ERR("can't init float"); PyObject *modules = PyDict_New(); if (modules == NULL) - Py_FatalError("Py_InitializeCore: can't make modules dictionary"); + return _Py_INIT_ERR("can't make modules dictionary"); interp->modules = modules; - sysmod = _PySys_BeginInit(); - if (sysmod == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys"); + err = _PySys_BeginInit(&sysmod); + if (_Py_INIT_FAILED(err)) { + return err; + } + interp->sysdict = PyModule_GetDict(sysmod); - if (interp->sysdict == NULL) - Py_FatalError("Py_InitializeCore: can't initialize sys dict"); + if (interp->sysdict == NULL) { + return _Py_INIT_ERR("can't initialize sys dict"); + } + Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); _PyImport_FixupBuiltin(sysmod, "sys", modules); /* Init Unicode implementation; relies on the codec registry */ if (_PyUnicode_Init() < 0) - Py_FatalError("Py_InitializeCore: can't initialize unicode"); + return _Py_INIT_ERR("can't initialize unicode"); if (_PyStructSequence_Init() < 0) - Py_FatalError("Py_InitializeCore: can't initialize structseq"); + return _Py_INIT_ERR("can't initialize structseq"); bimod = _PyBuiltin_Init(); if (bimod == NULL) - Py_FatalError("Py_InitializeCore: can't initialize builtins modules"); + return _Py_INIT_ERR("can't initialize builtins modules"); _PyImport_FixupBuiltin(bimod, "builtins", modules); interp->builtins = PyModule_GetDict(bimod); if (interp->builtins == NULL) - Py_FatalError("Py_InitializeCore: can't initialize builtins dict"); + return _Py_INIT_ERR("can't initialize builtins dict"); Py_INCREF(interp->builtins); /* initialize builtin exceptions */ @@ -718,25 +746,37 @@ void _Py_InitializeCore(const _PyCoreConfig *config) infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); if (pstderr == NULL) - Py_FatalError("Py_InitializeCore: can't set preliminary stderr"); + return _Py_INIT_ERR("can't set preliminary stderr"); _PySys_SetObjectId(&PyId_stderr, pstderr); PySys_SetObject("__stderr__", pstderr); Py_DECREF(pstderr); - _PyImport_Init(); + err = _PyImport_Init(); + if (_Py_INIT_FAILED(err)) { + return err; + } - _PyImportHooks_Init(); + err = _PyImportHooks_Init(); + if (_Py_INIT_FAILED(err)) { + return err; + } /* Initialize _warnings. */ - _PyWarnings_Init(); + if (_PyWarnings_InitWithConfig(&interp->core_config) == NULL) { + return _Py_INIT_ERR("can't initialize warnings"); + } /* This call sets up builtin and frozen import support */ if (!interp->core_config._disable_importlib) { - initimport(interp, sysmod); + err = initimport(interp, sysmod); + if (_Py_INIT_FAILED(err)) { + return err; + } } /* Only when we get here is the runtime core fully initialized */ _PyRuntime.core_initialized = 1; + return _Py_INIT_OK(); } /* Read configuration settings from standard locations @@ -751,16 +791,46 @@ void _Py_InitializeCore(const _PyCoreConfig *config) * this function multiple times with various preconfigured settings. */ -int _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) +_PyInitError +_PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config) { /* Signal handlers are installed by default */ if (config->install_signal_handlers < 0) { config->install_signal_handlers = 1; } - return 0; + if (config->program_name == NULL) { +#ifdef MS_WINDOWS + const wchar_t *program_name = L"python"; +#else + const wchar_t *program_name = L"python3"; +#endif + config->program_name = _PyMem_RawWcsdup(program_name); + if (config->program_name == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + return _Py_INIT_OK(); +} + + +void +_PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config) +{ +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->module_search_path_env); + CLEAR(config->home); + CLEAR(config->program_name); +#undef CLEAR } + /* Update interpreter state based on supplied configuration settings * * After calling this function, most of the restrictions on the interpreter @@ -772,25 +842,27 @@ int _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) * Other errors should be reported as normal Python exceptions with a * non-zero return code. */ -int _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) +_PyInitError +_Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) { PyInterpreterState *interp; PyThreadState *tstate; + _PyInitError err; if (!_PyRuntime.core_initialized) { - Py_FatalError("Py_InitializeMainInterpreter: runtime core not initialized"); + return _Py_INIT_ERR("runtime core not initialized"); } if (_PyRuntime.initialized) { - Py_FatalError("Py_InitializeMainInterpreter: main interpreter already initialized"); + return _Py_INIT_ERR("main interpreter already initialized"); } /* Get current thread state and interpreter pointer */ tstate = PyThreadState_GET(); if (!tstate) - Py_FatalError("Py_InitializeMainInterpreter: failed to read thread state"); + return _Py_INIT_ERR("failed to read thread state"); interp = tstate->interp; if (!interp) - Py_FatalError("Py_InitializeMainInterpreter: failed to get interpreter"); + return _Py_INIT_ERR("failed to get interpreter"); /* Now finish configuring the main interpreter */ interp->config = *config; @@ -802,39 +874,61 @@ int _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) * or pure Python code in the standard library won't work. */ _PyRuntime.initialized = 1; - return 0; + return _Py_INIT_OK(); } - /* TODO: Report exceptions rather than fatal errors below here */ - if (_PyTime_Init() < 0) - Py_FatalError("Py_InitializeMainInterpreter: can't initialize time"); + if (_PyTime_Init() < 0) { + return _Py_INIT_ERR("can't initialize time"); + } - /* Finish setting up the sys module and import system */ /* GetPath may initialize state that _PySys_EndInit locks in, and so has to be called first. */ - /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */ - PySys_SetPath(Py_GetPath()); + err = _PyPathConfig_Init(&interp->config); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *sys_path = Py_GetPath(); + + /* Finish setting up the sys module and import system */ + PySys_SetPath(sys_path); if (_PySys_EndInit(interp->sysdict) < 0) - Py_FatalError("Py_InitializeMainInterpreter: can't finish initializing sys"); - initexternalimport(interp); + return _Py_INIT_ERR("can't finish initializing sys"); + + err = initexternalimport(interp); + if (_Py_INIT_FAILED(err)) { + return err; + } /* initialize the faulthandler module */ - if (_PyFaulthandler_Init()) - Py_FatalError("Py_InitializeMainInterpreter: can't initialize faulthandler"); + err = _PyFaulthandler_Init(interp->core_config.faulthandler); + if (_Py_INIT_FAILED(err)) { + return err; + } - if (initfsencoding(interp) < 0) - Py_FatalError("Py_InitializeMainInterpreter: unable to load the file system codec"); + err = initfsencoding(interp); + if (_Py_INIT_FAILED(err)) { + return err; + } - if (config->install_signal_handlers) - initsigs(); /* Signal handling stuff, including initintr() */ + if (interp->config.install_signal_handlers) { + err = initsigs(); /* Signal handling stuff, including initintr() */ + if (_Py_INIT_FAILED(err)) { + return err; + } + } - if (_PyTraceMalloc_Init() < 0) - Py_FatalError("Py_InitializeMainInterpreter: can't initialize tracemalloc"); + if (_PyTraceMalloc_Init(interp->core_config.tracemalloc) < 0) + return _Py_INIT_ERR("can't initialize tracemalloc"); - initmain(interp); /* Module __main__ */ - if (initstdio() < 0) - Py_FatalError( - "Py_InitializeMainInterpreter: can't initialize sys standard streams"); + err = add_main_module(interp); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = init_sys_streams(); + if (_Py_INIT_FAILED(err)) { + return err; + } /* Initialize warnings. */ if (PySys_HasWarnOptions()) { @@ -848,37 +942,64 @@ int _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) _PyRuntime.initialized = 1; - if (!Py_NoSiteFlag) - initsite(); /* Module site */ + if (!Py_NoSiteFlag) { + err = initsite(); /* Module site */ + if (_Py_INIT_FAILED(err)) { + return err; + } + } - return 0; + return _Py_INIT_OK(); } #undef _INIT_DEBUG_PRINT -void +_PyInitError _Py_InitializeEx_Private(int install_sigs, int install_importlib) { _PyCoreConfig core_config = _PyCoreConfig_INIT; _PyMainInterpreterConfig config = _PyMainInterpreterConfig_INIT; + _PyInitError err; - /* TODO: Moar config options! */ core_config.ignore_environment = Py_IgnoreEnvironmentFlag; core_config._disable_importlib = !install_importlib; config.install_signal_handlers = install_sigs; - _Py_InitializeCore(&core_config); - /* TODO: Print any exceptions raised by these operations */ - if (_Py_ReadMainInterpreterConfig(&config)) - Py_FatalError("Py_Initialize: Py_ReadMainInterpreterConfig failed"); - if (_Py_InitializeMainInterpreter(&config)) - Py_FatalError("Py_Initialize: Py_InitializeMainInterpreter failed"); + + err = _Py_InitializeCore(&core_config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _PyMainInterpreterConfig_ReadEnv(&config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _PyMainInterpreterConfig_Read(&config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _Py_InitializeMainInterpreter(&config); + if (_Py_INIT_FAILED(err)) { + goto done; + } + + err = _Py_INIT_OK(); + +done: + _PyMainInterpreterConfig_Clear(&config); + return err; } void Py_InitializeEx(int install_sigs) { - _Py_InitializeEx_Private(install_sigs, 1); + _PyInitError err = _Py_InitializeEx_Private(install_sigs, 1); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } } void @@ -1017,10 +1138,6 @@ Py_FinalizeEx(void) /* nothing */; #endif -#ifdef Py_REF_DEBUG - PyObject *showrefcount = _PyDebug_XOptionShowRefCount(); -#endif - /* Destroy all modules */ PyImport_Cleanup(); @@ -1069,8 +1186,9 @@ Py_FinalizeEx(void) _PyHash_Fini(); #ifdef Py_REF_DEBUG - if (showrefcount == Py_True) - _PyDebug_PrintTotalRefs(); + if (interp->core_config.show_ref_count) { + _PyDebug_PrintTotalRefs(); + } #endif #ifdef Py_TRACE_REFS @@ -1154,6 +1272,7 @@ Py_FinalizeEx(void) #endif call_ll_exitfuncs(); + _PyRuntime_Finalize(); return status; } @@ -1177,28 +1296,33 @@ Py_Finalize(void) */ -PyThreadState * -Py_NewInterpreter(void) +static _PyInitError +new_interpreter(PyThreadState **tstate_p) { PyInterpreterState *interp; PyThreadState *tstate, *save_tstate; PyObject *bimod, *sysmod; + _PyInitError err; - if (!_PyRuntime.initialized) - Py_FatalError("Py_NewInterpreter: call Py_Initialize first"); + if (!_PyRuntime.initialized) { + return _Py_INIT_ERR("Py_Initialize must be called first"); + } /* Issue #10915, #15751: The GIL API doesn't work with multiple interpreters: disable PyGILState_Check(). */ _PyGILState_check_enabled = 0; interp = PyInterpreterState_New(); - if (interp == NULL) - return NULL; + if (interp == NULL) { + *tstate_p = NULL; + return _Py_INIT_OK(); + } tstate = PyThreadState_New(interp); if (tstate == NULL) { PyInterpreterState_Delete(interp); - return NULL; + *tstate_p = NULL; + return _Py_INIT_OK(); } save_tstate = PyThreadState_Swap(tstate); @@ -1214,11 +1338,17 @@ Py_NewInterpreter(void) interp->config = main_interp->config; } - /* XXX The following is lax in error checking */ + err = _PyPathConfig_Init(&interp->config); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *sys_path = Py_GetPath(); + /* XXX The following is lax in error checking */ PyObject *modules = PyDict_New(); - if (modules == NULL) - Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); + if (modules == NULL) { + return _Py_INIT_ERR("can't make modules dictionary"); + } interp->modules = modules; sysmod = _PyImport_FindBuiltin("sys", modules); @@ -1228,7 +1358,7 @@ Py_NewInterpreter(void) goto handle_error; Py_INCREF(interp->sysdict); PyDict_SetItemString(interp->sysdict, "modules", modules); - PySys_SetPath(Py_GetPath()); + PySys_SetPath(sys_path); _PySys_EndInit(interp->sysdict); } @@ -1249,30 +1379,57 @@ Py_NewInterpreter(void) /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); - if (pstderr == NULL) - Py_FatalError("Py_NewInterpreter: can't set preliminary stderr"); + if (pstderr == NULL) { + return _Py_INIT_ERR("can't set preliminary stderr"); + } _PySys_SetObjectId(&PyId_stderr, pstderr); PySys_SetObject("__stderr__", pstderr); Py_DECREF(pstderr); - _PyImportHooks_Init(); + err = _PyImportHooks_Init(); + if (_Py_INIT_FAILED(err)) { + return err; + } - initimport(interp, sysmod); - initexternalimport(interp); + err = initimport(interp, sysmod); + if (_Py_INIT_FAILED(err)) { + return err; + } - if (initfsencoding(interp) < 0) - goto handle_error; + err = initexternalimport(interp); + if (_Py_INIT_FAILED(err)) { + return err; + } - if (initstdio() < 0) - Py_FatalError( - "Py_NewInterpreter: can't initialize sys standard streams"); - initmain(interp); - if (!Py_NoSiteFlag) - initsite(); + err = initfsencoding(interp); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = init_sys_streams(); + if (_Py_INIT_FAILED(err)) { + return err; + } + + err = add_main_module(interp); + if (_Py_INIT_FAILED(err)) { + return err; + } + + if (!Py_NoSiteFlag) { + err = initsite(); + if (_Py_INIT_FAILED(err)) { + return err; + } + } + } + + if (PyErr_Occurred()) { + goto handle_error; } - if (!PyErr_Occurred()) - return tstate; + *tstate_p = tstate; + return _Py_INIT_OK(); handle_error: /* Oops, it didn't work. Undo it all. */ @@ -1283,7 +1440,20 @@ Py_NewInterpreter(void) PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); - return NULL; + *tstate_p = NULL; + return _Py_INIT_OK(); +} + +PyThreadState * +Py_NewInterpreter(void) +{ + PyThreadState *tstate; + _PyInitError err = new_interpreter(&tstate); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + return tstate; + } /* Delete an interpreter and its last thread. This requires that the @@ -1319,77 +1489,35 @@ Py_EndInterpreter(PyThreadState *tstate) PyInterpreterState_Delete(interp); } -#ifdef MS_WINDOWS -static wchar_t *progname = L"python"; -#else -static wchar_t *progname = L"python3"; -#endif - -void -Py_SetProgramName(wchar_t *pn) -{ - if (pn && *pn) - progname = pn; -} - -wchar_t * -Py_GetProgramName(void) -{ - return progname; -} - -static wchar_t *default_home = NULL; -static wchar_t env_home[MAXPATHLEN+1]; - -void -Py_SetPythonHome(wchar_t *home) -{ - default_home = home; -} - -wchar_t * -Py_GetPythonHome(void) -{ - wchar_t *home = default_home; - if (home == NULL && !Py_IgnoreEnvironmentFlag) { - char* chome = Py_GETENV("PYTHONHOME"); - if (chome) { - size_t size = Py_ARRAY_LENGTH(env_home); - size_t r = mbstowcs(env_home, chome, size); - if (r != (size_t)-1 && r < size) - home = env_home; - } +/* Add the __main__ module */ - } - return home; -} - -/* Create __main__ module */ - -static void -initmain(PyInterpreterState *interp) +static _PyInitError +add_main_module(PyInterpreterState *interp) { PyObject *m, *d, *loader, *ann_dict; m = PyImport_AddModule("__main__"); if (m == NULL) - Py_FatalError("can't create __main__ module"); + return _Py_INIT_ERR("can't create __main__ module"); + d = PyModule_GetDict(m); ann_dict = PyDict_New(); if ((ann_dict == NULL) || (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) { - Py_FatalError("Failed to initialize __main__.__annotations__"); + return _Py_INIT_ERR("Failed to initialize __main__.__annotations__"); } Py_DECREF(ann_dict); + if (PyDict_GetItemString(d, "__builtins__") == NULL) { PyObject *bimod = PyImport_ImportModule("builtins"); if (bimod == NULL) { - Py_FatalError("Failed to retrieve builtins module"); + return _Py_INIT_ERR("Failed to retrieve builtins module"); } if (PyDict_SetItemString(d, "__builtins__", bimod) < 0) { - Py_FatalError("Failed to initialize __main__.__builtins__"); + return _Py_INIT_ERR("Failed to initialize __main__.__builtins__"); } Py_DECREF(bimod); } + /* Main is a little special - imp.is_builtin("__main__") will return * False, but BuiltinImporter is still the most appropriate initial * setting for its __loader__ attribute. A more suitable value will @@ -1401,41 +1529,40 @@ initmain(PyInterpreterState *interp) PyObject *loader = PyObject_GetAttrString(interp->importlib, "BuiltinImporter"); if (loader == NULL) { - Py_FatalError("Failed to retrieve BuiltinImporter"); + return _Py_INIT_ERR("Failed to retrieve BuiltinImporter"); } if (PyDict_SetItemString(d, "__loader__", loader) < 0) { - Py_FatalError("Failed to initialize __main__.__loader__"); + return _Py_INIT_ERR("Failed to initialize __main__.__loader__"); } Py_DECREF(loader); } + return _Py_INIT_OK(); } -static int +static _PyInitError initfsencoding(PyInterpreterState *interp) { PyObject *codec; #ifdef MS_WINDOWS - if (Py_LegacyWindowsFSEncodingFlag) - { + if (Py_LegacyWindowsFSEncodingFlag) { Py_FileSystemDefaultEncoding = "mbcs"; Py_FileSystemDefaultEncodeErrors = "replace"; } - else - { + else { Py_FileSystemDefaultEncoding = "utf-8"; Py_FileSystemDefaultEncodeErrors = "surrogatepass"; } #else - if (Py_FileSystemDefaultEncoding == NULL) - { + if (Py_FileSystemDefaultEncoding == NULL) { Py_FileSystemDefaultEncoding = get_locale_encoding(); - if (Py_FileSystemDefaultEncoding == NULL) - Py_FatalError("Py_Initialize: Unable to get the locale encoding"); + if (Py_FileSystemDefaultEncoding == NULL) { + return _Py_INIT_ERR("Unable to get the locale encoding"); + } Py_HasFileSystemDefaultEncoding = 0; interp->fscodec_initialized = 1; - return 0; + return _Py_INIT_OK(); } #endif @@ -1445,29 +1572,25 @@ initfsencoding(PyInterpreterState *interp) /* Such error can only occurs in critical situations: no more * memory, import a module of the standard library failed, * etc. */ - return -1; + return _Py_INIT_ERR("unable to load the file system codec"); } Py_DECREF(codec); interp->fscodec_initialized = 1; - return 0; + return _Py_INIT_OK(); } /* Import the site module (not into __main__ though) */ -static void +static _PyInitError initsite(void) { PyObject *m; m = PyImport_ImportModule("site"); if (m == NULL) { - fprintf(stderr, "Failed to import the site module\n"); - PyErr_Print(); - Py_Finalize(); - exit(1); - } - else { - Py_DECREF(m); + return _Py_INIT_USER_ERR("Failed to import the site module"); } + Py_DECREF(m); + return _Py_INIT_OK(); } /* Check if a file descriptor is valid or not. @@ -1621,16 +1744,17 @@ create_stdio(PyObject* io, } /* Initialize sys.stdin, stdout, stderr and builtins.open */ -static int -initstdio(void) +static _PyInitError +init_sys_streams(void) { PyObject *iomod = NULL, *wrapper; PyObject *bimod = NULL; PyObject *m; PyObject *std = NULL; - int status = 0, fd; + int fd; PyObject * encoding_attr; char *pythonioencoding = NULL, *encoding, *errors; + _PyInitError res = _Py_INIT_OK(); /* Hack to avoid a nasty recursion issue when Python is invoked in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ @@ -1744,11 +1868,12 @@ initstdio(void) Py_DECREF(std); #endif - if (0) { - error: - status = -1; - } + goto done; +error: + res = _Py_INIT_ERR("can't initialize sys standard streams"); + +done: /* We won't need them anymore. */ if (_Py_StandardStreamEncoding) { PyMem_RawFree(_Py_StandardStreamEncoding); @@ -1761,7 +1886,7 @@ initstdio(void) PyMem_Free(pythonioencoding); Py_XDECREF(bimod); Py_XDECREF(iomod); - return status; + return res; } @@ -1873,8 +1998,8 @@ fatal_output_debug(const char *msg) } #endif -void -Py_FatalError(const char *msg) +static void _Py_NO_RETURN +fatal_error(const char *prefix, const char *msg, int status) { const int fd = fileno(stderr); static int reentrant = 0; @@ -1886,7 +2011,18 @@ Py_FatalError(const char *msg) } reentrant = 1; - fprintf(stderr, "Fatal Python error: %s\n", msg); + fprintf(stderr, "Fatal Python error: "); + if (prefix) { + fputs(prefix, stderr); + fputs(": ", stderr); + } + if (msg) { + fputs(msg, stderr); + } + else { + fprintf(stderr, ""); + } + fputs("\n", stderr); fflush(stderr); /* it helps in Windows debug build */ /* Print the exception (if an exception is set) with its traceback, @@ -1912,10 +2048,30 @@ Py_FatalError(const char *msg) #endif /* MS_WINDOWS */ exit: + if (status < 0) { #if defined(MS_WINDOWS) && defined(_DEBUG) - DebugBreak(); + DebugBreak(); #endif - abort(); + abort(); + } + else { + exit(status); + } +} + +void +Py_FatalError(const char *msg) +{ + fatal_error(NULL, msg, -1); +} + +void +_Py_FatalInitError(_PyInitError err) +{ + /* On "user" error: exit with status 1. + For all other errors, call abort(). */ + int status = err.user_err ? 1 : -1; + fatal_error(err.prefix, err.msg, status); } /* Clean up and exit */ @@ -1992,7 +2148,7 @@ Py_Exit(int sts) exit(sts); } -static void +static _PyInitError initsigs(void) { #ifdef SIGPIPE @@ -2006,8 +2162,9 @@ initsigs(void) #endif PyOS_InitInterrupts(); /* May imply initsignal() */ if (PyErr_Occurred()) { - Py_FatalError("Py_Initialize: can't import signal"); + return _Py_INIT_ERR("can't import signal"); } + return _Py_INIT_OK(); } diff --git a/Python/pystate.c b/Python/pystate.c index 55ff64951b0b7e..500f96768752f4 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -35,37 +35,57 @@ to avoid the expense of doing their own locking). extern "C" { #endif -void -_PyRuntimeState_Init(_PyRuntimeState *runtime) +static _PyInitError +_PyRuntimeState_Init_impl(_PyRuntimeState *runtime) { memset(runtime, 0, sizeof(*runtime)); - _PyObject_Initialize(&runtime->obj); - _PyMem_Initialize(&runtime->mem); _PyGC_Initialize(&runtime->gc); _PyEval_Initialize(&runtime->ceval); runtime->gilstate.check_enabled = 1; + /* A TSS key must be initialized with Py_tss_NEEDS_INIT in accordance with the specification. */ - { - Py_tss_t initial = Py_tss_NEEDS_INIT; - runtime->gilstate.autoTSSkey = initial; - } + Py_tss_t initial = Py_tss_NEEDS_INIT; + runtime->gilstate.autoTSSkey = initial; runtime->interpreters.mutex = PyThread_allocate_lock(); - if (runtime->interpreters.mutex == NULL) - Py_FatalError("Can't initialize threads for interpreter"); + if (runtime->interpreters.mutex == NULL) { + return _Py_INIT_ERR("Can't initialize threads for interpreter"); + } + runtime->interpreters.next_id = -1; + return _Py_INIT_OK(); +} + +_PyInitError +_PyRuntimeState_Init(_PyRuntimeState *runtime) +{ + /* Force default allocator, since _PyRuntimeState_Fini() must + use the same allocator than this function. */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + + _PyInitError err = _PyRuntimeState_Init_impl(runtime); + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + return err; } void _PyRuntimeState_Fini(_PyRuntimeState *runtime) { + /* Force the allocator used by _PyRuntimeState_Init(). */ + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + if (runtime->interpreters.mutex != NULL) { PyThread_free_lock(runtime->interpreters.mutex); runtime->interpreters.mutex = NULL; } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } #define HEAD_LOCK() PyThread_acquire_lock(_PyRuntime.interpreters.mutex, \ @@ -74,7 +94,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) static void _PyGILState_NoteThreadState(PyThreadState* tstate); -void +_PyInitError _PyInterpreterState_Enable(_PyRuntimeState *runtime) { runtime->interpreters.next_id = 0; @@ -83,9 +103,11 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) initialized here. */ if (runtime->interpreters.mutex == NULL) { runtime->interpreters.mutex = PyThread_allocate_lock(); - if (runtime->interpreters.mutex == NULL) - Py_FatalError("Can't initialize threads for interpreter"); + if (runtime->interpreters.mutex == NULL) { + return _Py_INIT_ERR("Can't initialize threads for interpreter"); + } } + return _Py_INIT_OK(); } PyInterpreterState * @@ -94,55 +116,60 @@ PyInterpreterState_New(void) PyInterpreterState *interp = (PyInterpreterState *) PyMem_RawMalloc(sizeof(PyInterpreterState)); - if (interp != NULL) { - interp->modules = NULL; - interp->modules_by_index = NULL; - interp->sysdict = NULL; - interp->builtins = NULL; - interp->builtins_copy = NULL; - interp->tstate_head = NULL; - interp->check_interval = 100; - interp->num_threads = 0; - interp->pythread_stacksize = 0; - interp->codec_search_path = NULL; - interp->codec_search_cache = NULL; - interp->codec_error_registry = NULL; - interp->codecs_initialized = 0; - interp->fscodec_initialized = 0; - interp->importlib = NULL; - interp->import_func = NULL; - interp->eval_frame = _PyEval_EvalFrameDefault; - interp->co_extra_user_count = 0; + if (interp == NULL) { + return NULL; + } + + + interp->modules = NULL; + interp->modules_by_index = NULL; + interp->sysdict = NULL; + interp->builtins = NULL; + interp->builtins_copy = NULL; + interp->tstate_head = NULL; + interp->check_interval = 100; + interp->num_threads = 0; + interp->pythread_stacksize = 0; + interp->codec_search_path = NULL; + interp->codec_search_cache = NULL; + interp->codec_error_registry = NULL; + interp->codecs_initialized = 0; + interp->fscodec_initialized = 0; + interp->core_config = _PyCoreConfig_INIT; + interp->config = _PyMainInterpreterConfig_INIT; + interp->importlib = NULL; + interp->import_func = NULL; + interp->eval_frame = _PyEval_EvalFrameDefault; + interp->co_extra_user_count = 0; #ifdef HAVE_DLOPEN #if HAVE_DECL_RTLD_NOW - interp->dlopenflags = RTLD_NOW; + interp->dlopenflags = RTLD_NOW; #else - interp->dlopenflags = RTLD_LAZY; + interp->dlopenflags = RTLD_LAZY; #endif #endif #ifdef HAVE_FORK - interp->before_forkers = NULL; - interp->after_forkers_parent = NULL; - interp->after_forkers_child = NULL; + interp->before_forkers = NULL; + interp->after_forkers_parent = NULL; + interp->after_forkers_child = NULL; #endif - HEAD_LOCK(); - interp->next = _PyRuntime.interpreters.head; - if (_PyRuntime.interpreters.main == NULL) { - _PyRuntime.interpreters.main = interp; - } - _PyRuntime.interpreters.head = interp; - if (_PyRuntime.interpreters.next_id < 0) { - /* overflow or Py_Initialize() not called! */ - PyErr_SetString(PyExc_RuntimeError, - "failed to get an interpreter ID"); - interp = NULL; - } else { - interp->id = _PyRuntime.interpreters.next_id; - _PyRuntime.interpreters.next_id += 1; - } - HEAD_UNLOCK(); + HEAD_LOCK(); + interp->next = _PyRuntime.interpreters.head; + if (_PyRuntime.interpreters.main == NULL) { + _PyRuntime.interpreters.main = interp; + } + _PyRuntime.interpreters.head = interp; + if (_PyRuntime.interpreters.next_id < 0) { + /* overflow or Py_Initialize() not called! */ + PyErr_SetString(PyExc_RuntimeError, + "failed to get an interpreter ID"); + interp = NULL; + } else { + interp->id = _PyRuntime.interpreters.next_id; + _PyRuntime.interpreters.next_id += 1; } + HEAD_UNLOCK(); return interp; } @@ -895,6 +922,8 @@ PyGILState_Ensure(void) { int current; PyThreadState *tcur; + int need_init_threads = 0; + /* Note that we do not auto-init Python here - apart from potential races with 2 threads auto-initializing, pep-311 spells out other issues. Embedders are expected to have @@ -902,12 +931,10 @@ PyGILState_Ensure(void) */ /* Py_Initialize() hasn't been called! */ assert(_PyRuntime.gilstate.autoInterpreterState); + tcur = (PyThreadState *)PyThread_tss_get(&_PyRuntime.gilstate.autoTSSkey); if (tcur == NULL) { - /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is - called from a new thread for the first time, we need the create the - GIL. */ - PyEval_InitThreads(); + need_init_threads = 1; /* Create a new thread state for this thread */ tcur = PyThreadState_New(_PyRuntime.gilstate.autoInterpreterState); @@ -918,16 +945,28 @@ PyGILState_Ensure(void) tcur->gilstate_counter = 0; current = 0; /* new thread state is never current */ } - else + else { current = PyThreadState_IsCurrent(tcur); - if (current == 0) + } + + if (current == 0) { PyEval_RestoreThread(tcur); + } + /* Update our counter in the thread-state - no need for locks: - tcur will remain valid as we hold the GIL. - the counter is safe as we are the only thread "allowed" to modify this value */ ++tcur->gilstate_counter; + + if (need_init_threads) { + /* At startup, Python has no concrete GIL. If PyGILState_Ensure() is + called from a new thread for the first time, we need the create the + GIL. */ + PyEval_InitThreads(); + } + return current ? PyGILState_LOCKED : PyGILState_UNLOCKED; } diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 1259ed12dffe5c..6dbf32dcc4f92d 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -16,14 +16,14 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, if (return_bytes) { /* If _PyBytes_FromSize() were public we could avoid malloc+copy. */ retbuf = (Py_UCS1*) PyMem_Malloc(arglen*2); - if (!retbuf) - return PyErr_NoMemory(); + if (!retbuf) + return PyErr_NoMemory(); retval = NULL; /* silence a compiler warning, assigned later. */ } else { - retval = PyUnicode_New(arglen*2, 127); - if (!retval) - return NULL; - retbuf = PyUnicode_1BYTE_DATA(retval); + retval = PyUnicode_New(arglen*2, 127); + if (!retval) + return NULL; + retbuf = PyUnicode_1BYTE_DATA(retval); } /* make hex version of string, taken from shamodule.c */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index f19d2399b38ab2..9bf93638621038 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -597,7 +597,8 @@ Py_LOCAL_INLINE(char *) ensure_decimal_point(char* buffer, size_t buf_size, int precision) { int digit_count, insert_count = 0, convert_to_exp = 0; - char *chars_to_insert, *digits_start; + const char *chars_to_insert; + char *digits_start; /* search for the first non-digit character */ char *p = buffer; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 17ec182b74cc3f..b5285754ab0be5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -65,6 +65,7 @@ static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); static void err_free(perrdetail *); +static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *); /* Parse input from a file and execute it */ int @@ -89,6 +90,10 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags * PyObject *filename, *v; int ret, err; PyCompilerFlags local_flags; + int nomem_count = 0; +#ifdef Py_REF_DEBUG + int show_ref_count = PyThreadState_GET()->interp->core_config.show_ref_count; +#endif filename = PyUnicode_DecodeFSDefault(filename_str); if (filename == NULL) { @@ -110,22 +115,33 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags * _PySys_SetObjectId(&PyId_ps2, v = PyUnicode_FromString("... ")); Py_XDECREF(v); } - err = -1; - for (;;) { - ret = PyRun_InteractiveOneObject(fp, filename, flags); + err = 0; + do { + ret = PyRun_InteractiveOneObjectEx(fp, filename, flags); + if (ret == -1 && PyErr_Occurred()) { + /* Prevent an endless loop after multiple consecutive MemoryErrors + * while still allowing an interactive command to fail with a + * MemoryError. */ + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + if (++nomem_count > 16) { + PyErr_Clear(); + err = -1; + break; + } + } else { + nomem_count = 0; + } + PyErr_Print(); + flush_io(); + } else { + nomem_count = 0; + } #ifdef Py_REF_DEBUG - if (_PyDebug_XOptionShowRefCount() == Py_True) + if (show_ref_count) { _PyDebug_PrintTotalRefs(); -#endif - if (ret == E_EOF) { - err = 0; - break; } - /* - if (ret == E_NOMEM) - break; - */ - } +#endif + } while (ret != E_EOF); Py_DECREF(filename); return err; } @@ -154,8 +170,11 @@ static int PARSER_FLAGS(PyCompilerFlags *flags) PyPARSE_WITH_IS_KEYWORD : 0)) : 0) #endif -int -PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) +/* A PyRun_InteractiveOneObject() auxiliary function that does not print the + * error on failure. */ +static int +PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, + PyCompilerFlags *flags) { PyObject *m, *d, *v, *w, *oenc = NULL, *mod_name; mod_ty mod; @@ -167,7 +186,6 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) mod_name = _PyUnicode_FromId(&PyId___main__); /* borrowed */ if (mod_name == NULL) { - PyErr_Print(); return -1; } @@ -227,7 +245,6 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) PyErr_Clear(); return E_EOF; } - PyErr_Print(); return -1; } m = PyImport_AddModuleObject(mod_name); @@ -239,8 +256,6 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) v = run_mod(mod, filename, d, d, flags, arena); PyArena_Free(arena); if (v == NULL) { - PyErr_Print(); - flush_io(); return -1; } Py_DECREF(v); @@ -248,6 +263,19 @@ PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) return 0; } +int +PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) +{ + int res; + + res = PyRun_InteractiveOneObjectEx(fp, filename, flags); + if (res == -1) { + PyErr_Print(); + flush_io(); + } + return res; +} + int PyRun_InteractiveOneFlags(FILE *fp, const char *filename_str, PyCompilerFlags *flags) { @@ -1300,7 +1328,7 @@ err_input(perrdetail *err) { PyObject *v, *w, *errtype, *errtext; PyObject *msg_obj = NULL; - char *msg = NULL; + const char *msg = NULL; int offset = err->offset; errtype = PyExc_SyntaxError; diff --git a/Python/strdup.c b/Python/strdup.c index 769d3db130988c..99dc77417bd6d1 100644 --- a/Python/strdup.c +++ b/Python/strdup.c @@ -5,10 +5,10 @@ char * strdup(const char *str) { - if (str != NULL) { - char *copy = malloc(strlen(str) + 1); - if (copy != NULL) - return strcpy(copy, str); - } - return NULL; + if (str != NULL) { + char *copy = malloc(strlen(str) + 1); + if (copy != NULL) + return strcpy(copy, str); + } + return NULL; } diff --git a/Python/symtable.c b/Python/symtable.c index 55815c91cc9cab..bbac25cf3767aa 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1734,7 +1734,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, e->lineno, e->col_offset)) { return 0; } - st->st_cur->ste_generator = is_generator; if (outermost->is_async) { st->st_cur->ste_coroutine = 1; } @@ -1754,6 +1753,36 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (value) VISIT(st, expr, value); VISIT(st, expr, elt); + if (st->st_cur->ste_generator) { + PyObject *msg = PyUnicode_FromString( + (e->kind == ListComp_kind) ? "'yield' inside list comprehension" : + (e->kind == SetComp_kind) ? "'yield' inside set comprehension" : + (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" : + "'yield' inside generator expression"); + if (msg == NULL) { + symtable_exit_block(st, (void *)e); + return 0; + } + if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning, + msg, st->st_filename, st->st_cur->ste_lineno, + NULL, NULL) == -1) + { + if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + /* Replace the DeprecationWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + PyErr_SetObject(PyExc_SyntaxError, msg); + PyErr_SyntaxLocationObject(st->st_filename, + st->st_cur->ste_lineno, + st->st_cur->ste_col_offset); + } + Py_DECREF(msg); + symtable_exit_block(st, (void *)e); + return 0; + } + Py_DECREF(msg); + } + st->st_cur->ste_generator |= is_generator; return symtable_exit_block(st, (void *)e); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6dc8e08be7d997..64bc14e9c3d733 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1582,13 +1582,23 @@ PySys_ResetWarnOptions(void) PyList_SetSlice(warnoptions, 0, PyList_GET_SIZE(warnoptions), NULL); } -void -PySys_AddWarnOptionUnicode(PyObject *unicode) +int +_PySys_AddWarnOptionWithError(PyObject *option) { PyObject *warnoptions = get_warnoptions(); - if (warnoptions == NULL) - return; - PyList_Append(warnoptions, unicode); + if (warnoptions == NULL) { + return -1; + } + if (PyList_Append(warnoptions, option)) { + return -1; + } + return 0; +} + +void +PySys_AddWarnOptionUnicode(PyObject *option) +{ + (void)_PySys_AddWarnOptionWithError(option); } void @@ -1627,18 +1637,17 @@ get_xoptions(void) return xoptions; } -void -PySys_AddXOption(const wchar_t *s) +int +_PySys_AddXOptionWithError(const wchar_t *s) { - PyObject *opts; PyObject *name = NULL, *value = NULL; - const wchar_t *name_end; - opts = get_xoptions(); - if (opts == NULL) + PyObject *opts = get_xoptions(); + if (opts == NULL) { goto error; + } - name_end = wcschr(s, L'='); + const wchar_t *name_end = wcschr(s, L'='); if (!name_end) { name = PyUnicode_FromWideChar(s, -1); value = Py_True; @@ -1648,19 +1657,30 @@ PySys_AddXOption(const wchar_t *s) name = PyUnicode_FromWideChar(s, name_end - s); value = PyUnicode_FromWideChar(name_end + 1, -1); } - if (name == NULL || value == NULL) + if (name == NULL || value == NULL) { + goto error; + } + if (PyDict_SetItem(opts, name, value) < 0) { goto error; - PyDict_SetItem(opts, name, value); + } Py_DECREF(name); Py_DECREF(value); - return; + return 0; error: Py_XDECREF(name); Py_XDECREF(value); - /* No return value, therefore clear error state if possible */ - if (_PyThreadState_UncheckedGet()) { - PyErr_Clear(); + return -1; +} + +void +PySys_AddXOption(const wchar_t *s) +{ + if (_PySys_AddXOptionWithError(s) < 0) { + /* No return value, therefore clear error state if possible */ + if (_PyThreadState_UncheckedGet()) { + PyErr_Clear(); + } } } @@ -1794,6 +1814,7 @@ static PyStructSequence_Field flags_fields[] = { {"quiet", "-q"}, {"hash_randomization", "-R"}, {"isolated", "-I"}, + {"dev_mode", "-X dev"}, {0} }; @@ -1801,7 +1822,7 @@ static PyStructSequence_Desc flags_desc = { "sys.flags", /* name */ flags__doc__, /* doc */ flags_fields, /* fields */ - 13 + 14 }; static PyObject* @@ -1809,6 +1830,7 @@ make_flags(void) { int pos = 0; PyObject *seq; + _PyCoreConfig *core_config = &_PyGILState_GetInterpreterStateUnsafe()->core_config; seq = PyStructSequence_New(&FlagsType); if (seq == NULL) @@ -1833,6 +1855,7 @@ make_flags(void) SetFlag(Py_HashRandomizationFlag); SetFlag(Py_IsolatedFlag); #undef SetFlag + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(core_config->dev_mode)); if (PyErr_Occurred()) { Py_DECREF(seq); @@ -1999,50 +2022,52 @@ static struct PyModuleDef sysmodule = { #define SET_SYS_FROM_STRING_BORROW(key, value) \ do { \ PyObject *v = (value); \ - if (v == NULL) \ - return NULL; \ + if (v == NULL) { \ + goto err_occurred; \ + } \ res = PyDict_SetItemString(sysdict, key, v); \ if (res < 0) { \ - return NULL; \ + goto err_occurred; \ } \ } while (0) #define SET_SYS_FROM_STRING(key, value) \ do { \ PyObject *v = (value); \ - if (v == NULL) \ - return NULL; \ + if (v == NULL) { \ + goto err_occurred; \ + } \ res = PyDict_SetItemString(sysdict, key, v); \ Py_DECREF(v); \ if (res < 0) { \ - return NULL; \ + goto err_occurred; \ } \ } while (0) -PyObject * -_PySys_BeginInit(void) + +_PyInitError +_PySys_BeginInit(PyObject **sysmod) { PyObject *m, *sysdict, *version_info; int res; m = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION); - if (m == NULL) - return NULL; + if (m == NULL) { + return _Py_INIT_ERR("failed to create a module object"); + } sysdict = PyModule_GetDict(m); /* Check that stdin is not a directory - Using shell redirection, you can redirect stdin to a directory, - crashing the Python interpreter. Catch this common mistake here - and output a useful error message. Note that under MS Windows, - the shell already prevents that. */ -#if !defined(MS_WINDOWS) + Using shell redirection, you can redirect stdin to a directory, + crashing the Python interpreter. Catch this common mistake here + and output a useful error message. Note that under MS Windows, + the shell already prevents that. */ +#ifndef MS_WINDOWS { struct _Py_stat_struct sb; if (_Py_fstat_noraise(fileno(stdin), &sb) == 0 && S_ISDIR(sb.st_mode)) { - /* There's nothing more we can do. */ - /* Py_FatalError() will core dump, so just exit. */ - PySys_WriteStderr("Python error: is a directory, cannot continue\n"); - exit(EXIT_FAILURE); + return _Py_INIT_USER_ERR(" is a directory, " + "cannot continue"); } } #endif @@ -2078,8 +2103,9 @@ _PySys_BeginInit(void) PyLong_GetInfo()); /* initialize hash_info */ if (Hash_InfoType.tp_name == NULL) { - if (PyStructSequence_InitType2(&Hash_InfoType, &hash_info_desc) < 0) - return NULL; + if (PyStructSequence_InitType2(&Hash_InfoType, &hash_info_desc) < 0) { + goto type_init_failed; + } } SET_SYS_FROM_STRING("hash_info", get_hash_info()); @@ -2109,8 +2135,9 @@ _PySys_BeginInit(void) /* version_info */ if (VersionInfoType.tp_name == NULL) { if (PyStructSequence_InitType2(&VersionInfoType, - &version_info_desc) < 0) - return NULL; + &version_info_desc) < 0) { + goto type_init_failed; + } } version_info = make_version_info(); SET_SYS_FROM_STRING("version_info", version_info); @@ -2126,8 +2153,9 @@ _PySys_BeginInit(void) /* flags */ if (FlagsType.tp_name == 0) { - if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) - return NULL; + if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) { + goto type_init_failed; + } } /* Set flags to their default values */ SET_SYS_FROM_STRING("flags", make_flags()); @@ -2136,14 +2164,17 @@ _PySys_BeginInit(void) /* getwindowsversion */ if (WindowsVersionType.tp_name == 0) if (PyStructSequence_InitType2(&WindowsVersionType, - &windows_version_desc) < 0) - return NULL; + &windows_version_desc) < 0) { + goto type_init_failed; + } /* prevent user from creating new instances */ WindowsVersionType.tp_init = NULL; WindowsVersionType.tp_new = NULL; + assert(!PyErr_Occurred()); res = PyDict_DelItemString(WindowsVersionType.tp_dict, "__new__"); - if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) + if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_Clear(); + } #endif /* float repr style: 0.03 (short) vs 0.029999999999999999 (legacy) */ @@ -2161,13 +2192,22 @@ _PySys_BeginInit(void) if (AsyncGenHooksType.tp_name == NULL) { if (PyStructSequence_InitType2( &AsyncGenHooksType, &asyncgen_hooks_desc) < 0) { - return NULL; + goto type_init_failed; } } - if (PyErr_Occurred()) - return NULL; - return m; + if (PyErr_Occurred()) { + goto err_occurred; + } + + *sysmod = m; + return _Py_INIT_OK(); + +type_init_failed: + return _Py_INIT_ERR("failed to initialize a type"); + +err_occurred: + return _Py_INIT_ERR("can't initialize sys module"); } #undef SET_SYS_FROM_STRING diff --git a/Python/traceback.c b/Python/traceback.c index 21b36b1471935c..831b4f26249c93 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -414,57 +414,68 @@ tb_displayline(PyObject *f, PyObject *filename, int lineno, PyObject *name) return err; } +static int +tb_print_line_repeated(PyObject *f, long cnt) +{ + int err; + PyObject *line = PyUnicode_FromFormat( + " [Previous line repeated %ld more times]\n", cnt-3); + if (line == NULL) { + return -1; + } + err = PyFile_WriteObject(line, f, Py_PRINT_RAW); + Py_DECREF(line); + return err; +} + static int tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit) { int err = 0; - long depth = 0; + Py_ssize_t depth = 0; PyObject *last_file = NULL; int last_line = -1; PyObject *last_name = NULL; long cnt = 0; - PyObject *line; PyTracebackObject *tb1 = tb; while (tb1 != NULL) { depth++; tb1 = tb1->tb_next; } + while (tb != NULL && depth > limit) { + depth--; + tb = tb->tb_next; + } while (tb != NULL && err == 0) { - if (depth <= limit) { - if (last_file != NULL && - tb->tb_frame->f_code->co_filename == last_file && - last_line != -1 && tb->tb_lineno == last_line && - last_name != NULL && - tb->tb_frame->f_code->co_name == last_name) { - cnt++; - } else { - if (cnt > 3) { - line = PyUnicode_FromFormat( - " [Previous line repeated %d more times]\n", cnt-3); - err = PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_DECREF(line); - } - last_file = tb->tb_frame->f_code->co_filename; - last_line = tb->tb_lineno; - last_name = tb->tb_frame->f_code->co_name; - cnt = 0; - } - if (cnt < 3) - err = tb_displayline(f, - tb->tb_frame->f_code->co_filename, - tb->tb_lineno, - tb->tb_frame->f_code->co_name); + if (last_file != NULL && + tb->tb_frame->f_code->co_filename == last_file && + last_line != -1 && tb->tb_lineno == last_line && + last_name != NULL && tb->tb_frame->f_code->co_name == last_name) + { + cnt++; + } + else { + if (cnt > 3) { + err = tb_print_line_repeated(f, cnt); + } + last_file = tb->tb_frame->f_code->co_filename; + last_line = tb->tb_lineno; + last_name = tb->tb_frame->f_code->co_name; + cnt = 0; + } + if (err == 0 && cnt < 3) { + err = tb_displayline(f, + tb->tb_frame->f_code->co_filename, + tb->tb_lineno, + tb->tb_frame->f_code->co_name); + if (err == 0) { + err = PyErr_CheckSignals(); + } } - depth--; tb = tb->tb_next; - if (err == 0) - err = PyErr_CheckSignals(); } - if (cnt > 3) { - line = PyUnicode_FromFormat( - " [Previous line repeated %d more times]\n", cnt-3); - err = PyFile_WriteObject(line, f, Py_PRINT_RAW); - Py_DECREF(line); + if (err == 0 && cnt > 3) { + err = tb_print_line_repeated(f, cnt); } return err; } @@ -485,26 +496,15 @@ PyTraceBack_Print(PyObject *v, PyObject *f) return -1; } limitv = PySys_GetObject("tracebacklimit"); - if (limitv) { - PyObject *exc_type, *exc_value, *exc_tb; - - PyErr_Fetch(&exc_type, &exc_value, &exc_tb); - limit = PyLong_AsLong(limitv); - if (limit == -1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) { - limit = PyTraceBack_LIMIT; - } - else { - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_tb); - return 0; - } + if (limitv && PyLong_Check(limitv)) { + int overflow; + limit = PyLong_AsLongAndOverflow(limitv, &overflow); + if (overflow > 0) { + limit = LONG_MAX; } else if (limit <= 0) { - limit = PyTraceBack_LIMIT; + return 0; } - PyErr_Restore(exc_type, exc_value, exc_tb); } err = PyFile_WriteString("Traceback (most recent call last):\n", f); if (!err) diff --git a/Tools/c-globals/ignored-globals.txt b/Tools/c-globals/ignored-globals.txt index 7b5add865c166c..ce6d1d805147b6 100644 --- a/Tools/c-globals/ignored-globals.txt +++ b/Tools/c-globals/ignored-globals.txt @@ -393,7 +393,6 @@ Py_NoUserSiteDirectory Py_OptimizeFlag Py_QuietFlag Py_UnbufferedStdioFlag -Py_UseClassExceptionsFlag Py_VerboseFlag diff --git a/Tools/scripts/README b/Tools/scripts/README index c6b2282edfdfcd..d4ac2ade25ef84 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -61,9 +61,7 @@ rgrep.py Reverse grep through a file (useful for big logfiles) run_tests.py Run the test suite with more sensible default options serve.py Small wsgiref-based web server, used in make serve in Doc suff.py Sort a list of files by suffix -svneol.py Set svn:eol-style on all files in directory texi2html.py Convert GNU texinfo files into HTML -treesync.py Synchronize source trees (very idiosyncratic) untabify.py Replace tabs with spaces in argument files which.py Find a program in $PATH win_add2path.py Add Python to the search path on Windows diff --git a/Tools/scripts/svneol.py b/Tools/scripts/svneol.py deleted file mode 100755 index 6c70da9692221d..00000000000000 --- a/Tools/scripts/svneol.py +++ /dev/null @@ -1,114 +0,0 @@ -#! /usr/bin/env python3 - -r""" -SVN helper script. - -Try to set the svn:eol-style property to "native" on every .py, .txt, .c and -.h file in the directory tree rooted at the current directory. - -Files with the svn:eol-style property already set (to anything) are skipped. - -svn will itself refuse to set this property on a file that's not under SVN -control, or that has a binary mime-type property set. This script inherits -that behavior, and passes on whatever warning message the failing "svn -propset" command produces. - -In the Python project, it's safe to invoke this script from the root of -a checkout. - -No output is produced for files that are ignored. For a file that gets -svn:eol-style set, output looks like: - - property 'svn:eol-style' set on 'Lib\ctypes\__init__.py' - -For a file not under version control: - - svn: warning: 'patch-finalizer.txt' is not under version control - -and for a file with a binary mime-type property: - - svn: File 'Lib\test\test_pep263.py' has binary mime type property -""" - -import re -import os -import sys -import subprocess - - -def propfiles(root, fn): - default = os.path.join(root, ".svn", "props", fn + ".svn-work") - try: - format = int(open(os.path.join(root, ".svn", "format")).read().strip()) - except IOError: - return [] - if format in (8, 9): - # In version 8 and 9, committed props are stored in prop-base, local - # modifications in props - return [os.path.join(root, ".svn", "prop-base", fn + ".svn-base"), - os.path.join(root, ".svn", "props", fn + ".svn-work")] - raise ValueError("Unknown repository format") - - -def proplist(root, fn): - """Return a list of property names for file fn in directory root.""" - result = [] - for path in propfiles(root, fn): - try: - f = open(path) - except IOError: - # no properties file: not under version control, - # or no properties set - continue - while True: - # key-value pairs, of the form - # K - # NL - # V length - # NL - # END - line = f.readline() - if line.startswith("END"): - break - assert line.startswith("K ") - L = int(line.split()[1]) - key = f.read(L) - result.append(key) - f.readline() - line = f.readline() - assert line.startswith("V ") - L = int(line.split()[1]) - value = f.read(L) - f.readline() - f.close() - return result - - -def set_eol_native(path): - cmd = 'svn propset svn:eol-style native "{}"'.format(path) - propset = subprocess.Popen(cmd, shell=True) - propset.wait() - - -possible_text_file = re.compile(r"\.([hc]|py|txt|sln|vcproj)$").search - - -def main(): - for arg in sys.argv[1:] or [os.curdir]: - if os.path.isfile(arg): - root, fn = os.path.split(arg) - if 'svn:eol-style' not in proplist(root, fn): - set_eol_native(arg) - elif os.path.isdir(arg): - for root, dirs, files in os.walk(arg): - if '.svn' in dirs: - dirs.remove('.svn') - for fn in files: - if possible_text_file(fn): - if 'svn:eol-style' not in proplist(root, fn): - path = os.path.join(root, fn) - set_eol_native(path) - - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/treesync.py b/Tools/scripts/treesync.py deleted file mode 100755 index 652d3940a49a40..00000000000000 --- a/Tools/scripts/treesync.py +++ /dev/null @@ -1,210 +0,0 @@ -#! /usr/bin/env python3 - -"""Script to synchronize two source trees. - -Invoke with two arguments: - -python treesync.py slave master - -The assumption is that "master" contains CVS administration while -slave doesn't. All files in the slave tree that have a CVS/Entries -entry in the master tree are synchronized. This means: - - If the files differ: - if the slave file is newer: - normalize the slave file - if the files still differ: - copy the slave to the master - else (the master is newer): - copy the master to the slave - - normalizing the slave means replacing CRLF with LF when the master - doesn't use CRLF - -""" - -import os, sys, stat, getopt - -# Interactivity options -default_answer = "ask" -create_files = "yes" -create_directories = "no" -write_slave = "ask" -write_master = "ask" - -def main(): - global always_no, always_yes - global create_directories, write_master, write_slave - opts, args = getopt.getopt(sys.argv[1:], "nym:s:d:f:a:") - for o, a in opts: - if o == '-y': - default_answer = "yes" - if o == '-n': - default_answer = "no" - if o == '-s': - write_slave = a - if o == '-m': - write_master = a - if o == '-d': - create_directories = a - if o == '-f': - create_files = a - if o == '-a': - create_files = create_directories = write_slave = write_master = a - try: - [slave, master] = args - except ValueError: - print("usage: python", sys.argv[0] or "treesync.py", end=' ') - print("[-n] [-y] [-m y|n|a] [-s y|n|a] [-d y|n|a] [-f n|y|a]", end=' ') - print("slavedir masterdir") - return - process(slave, master) - -def process(slave, master): - cvsdir = os.path.join(master, "CVS") - if not os.path.isdir(cvsdir): - print("skipping master subdirectory", master) - print("-- not under CVS") - return - print("-"*40) - print("slave ", slave) - print("master", master) - if not os.path.isdir(slave): - if not okay("create slave directory %s?" % slave, - answer=create_directories): - print("skipping master subdirectory", master) - print("-- no corresponding slave", slave) - return - print("creating slave directory", slave) - try: - os.mkdir(slave) - except OSError as msg: - print("can't make slave directory", slave, ":", msg) - return - else: - print("made slave directory", slave) - cvsdir = None - subdirs = [] - names = os.listdir(master) - for name in names: - mastername = os.path.join(master, name) - slavename = os.path.join(slave, name) - if name == "CVS": - cvsdir = mastername - else: - if os.path.isdir(mastername) and not os.path.islink(mastername): - subdirs.append((slavename, mastername)) - if cvsdir: - entries = os.path.join(cvsdir, "Entries") - for e in open(entries).readlines(): - words = e.split('/') - if words[0] == '' and words[1:]: - name = words[1] - s = os.path.join(slave, name) - m = os.path.join(master, name) - compare(s, m) - for (s, m) in subdirs: - process(s, m) - -def compare(slave, master): - try: - sf = open(slave, 'r') - except IOError: - sf = None - try: - mf = open(master, 'rb') - except IOError: - mf = None - if not sf: - if not mf: - print("Neither master nor slave exists", master) - return - print("Creating missing slave", slave) - copy(master, slave, answer=create_files) - return - if not mf: - print("Not updating missing master", master) - return - if sf and mf: - if identical(sf, mf): - return - sft = mtime(sf) - mft = mtime(mf) - if mft > sft: - # Master is newer -- copy master to slave - sf.close() - mf.close() - print("Master ", master) - print("is newer than slave", slave) - copy(master, slave, answer=write_slave) - return - # Slave is newer -- copy slave to master - print("Slave is", sft-mft, "seconds newer than master") - # But first check what to do about CRLF - mf.seek(0) - fun = funnychars(mf) - mf.close() - sf.close() - if fun: - print("***UPDATING MASTER (BINARY COPY)***") - copy(slave, master, "rb", answer=write_master) - else: - print("***UPDATING MASTER***") - copy(slave, master, "r", answer=write_master) - -BUFSIZE = 16*1024 - -def identical(sf, mf): - while 1: - sd = sf.read(BUFSIZE) - md = mf.read(BUFSIZE) - if sd != md: return 0 - if not sd: break - return 1 - -def mtime(f): - st = os.fstat(f.fileno()) - return st[stat.ST_MTIME] - -def funnychars(f): - while 1: - buf = f.read(BUFSIZE) - if not buf: break - if '\r' in buf or '\0' in buf: return 1 - return 0 - -def copy(src, dst, rmode="rb", wmode="wb", answer='ask'): - print("copying", src) - print(" to", dst) - if not okay("okay to copy? ", answer): - return - f = open(src, rmode) - g = open(dst, wmode) - while 1: - buf = f.read(BUFSIZE) - if not buf: break - g.write(buf) - f.close() - g.close() - -def raw_input(prompt): - sys.stdout.write(prompt) - sys.stdout.flush() - return sys.stdin.readline() - -def okay(prompt, answer='ask'): - answer = answer.strip().lower() - if not answer or answer[0] not in 'ny': - answer = input(prompt) - answer = answer.strip().lower() - if not answer: - answer = default_answer - if answer[:1] == 'y': - return 1 - if answer[:1] == 'n': - return 0 - print("Yes or No please -- try again:") - return okay(prompt) - -if __name__ == '__main__': - main() diff --git a/configure b/configure index 0e74828a408ca4..d02675742d27d9 100755 --- a/configure +++ b/configure @@ -778,7 +778,6 @@ infodir docdir oldincludedir includedir -runstatedir localstatedir sharedstatedir sysconfdir @@ -890,7 +889,6 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' -runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1143,15 +1141,6 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; - -runstatedir | --runstatedir | --runstatedi | --runstated \ - | --runstate | --runstat | --runsta | --runst | --runs \ - | --run | --ru | --r) - ac_prev=runstatedir ;; - -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ - | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ - | --run=* | --ru=* | --r=*) - runstatedir=$ac_optarg ;; - -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1289,7 +1278,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir runstatedir + libdir localedir mandir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1442,7 +1431,6 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] - --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -5623,7 +5611,6 @@ $as_echo "$ac_cv_safe_to_define___extensions__" >&6; } $as_echo_n "checking for the Android API level... " >&6; } cat >> conftest.c < android_api = __ANDROID_API__ arm_arch = __ARM_ARCH #else @@ -5636,6 +5623,10 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then _arm_arch=`sed -n -e '/__ARM_ARCH/d' -e 's/^arm_arch = //p' conftest.out` { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ANDROID_API_LEVEL" >&5 $as_echo "$ANDROID_API_LEVEL" >&6; } + if test -z "$ANDROID_API_LEVEL"; then + echo 'Fatal: you must define __ANDROID_API__' + exit 1 + fi cat >>confdefs.h <<_ACEOF #define ANDROID_API_LEVEL $ANDROID_API_LEVEL @@ -9542,6 +9533,9 @@ $as_echo "no" >&6; } fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +# 'Real Time' functions on Solaris +# posix4 on Solaris 2.6 +# pthread (first!) on Linux { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing sem_init" >&5 $as_echo_n "checking for library containing sem_init... " >&6; } if ${ac_cv_search_sem_init+:} false; then : @@ -9597,9 +9591,7 @@ if test "$ac_res" != no; then : test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" fi - # 'Real Time' functions on Solaris - # posix4 on Solaris 2.6 - # pthread (first!) on Linux + # check if we need libintl for locale functions { $as_echo "$as_me:${as_lineno-$LINENO}: checking for textdomain in -lintl" >&5 @@ -11154,7 +11146,8 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ futimens futimes gai_strerror getentropy \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ - initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ + if_nameindex \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ @@ -12584,80 +12577,6 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -# On Android API level 24 with android-ndk-r13, if_nameindex() is available, -# but the if_nameindex structure is not defined. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for if_nameindex" >&5 -$as_echo_n "checking for if_nameindex... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif - -int -main () -{ -struct if_nameindex *ni = if_nameindex(); int x = ni[0].if_index; - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - -$as_echo "#define HAVE_IF_NAMEINDEX 1" >>confdefs.h - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - -fi -rm -f core conftest.err conftest.$ac_objext \ - conftest$ac_exeext conftest.$ac_ext - -# Issue #28762: lockf() is available on Android API level 24, but the F_LOCK -# macro is not defined in android-ndk-r13. -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for lockf" >&5 -$as_echo_n "checking for lockf... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include -int -main () -{ -lockf(0, F_LOCK, 0); - ; - return 0; -} -_ACEOF -if ac_fn_c_try_link "$LINENO"; then : - -$as_echo "#define HAVE_LOCKF 1" >>confdefs.h - - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext diff --git a/configure.ac b/configure.ac index 1a309c813adb96..68a95c3be6af4a 100644 --- a/configure.ac +++ b/configure.ac @@ -885,7 +885,6 @@ AC_USE_SYSTEM_EXTENSIONS AC_MSG_CHECKING([for the Android API level]) cat >> conftest.c < android_api = __ANDROID_API__ arm_arch = __ARM_ARCH #else @@ -897,6 +896,10 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then ANDROID_API_LEVEL=`sed -n -e '/__ANDROID_API__/d' -e 's/^android_api = //p' conftest.out` _arm_arch=`sed -n -e '/__ARM_ARCH/d' -e 's/^arm_arch = //p' conftest.out` AC_MSG_RESULT([$ANDROID_API_LEVEL]) + if test -z "$ANDROID_API_LEVEL"; then + echo 'Fatal: you must define __ANDROID_API__' + exit 1 + fi AC_DEFINE_UNQUOTED(ANDROID_API_LEVEL, $ANDROID_API_LEVEL, [The Android API level.]) AC_MSG_CHECKING([for the Android arm ABI]) @@ -2688,9 +2691,10 @@ void *x = uuid_generate_time_safe [AC_MSG_RESULT(no)] ) -AC_SEARCH_LIBS(sem_init, pthread rt posix4) # 'Real Time' functions on Solaris - # posix4 on Solaris 2.6 - # pthread (first!) on Linux +# 'Real Time' functions on Solaris +# posix4 on Solaris 2.6 +# pthread (first!) on Linux +AC_SEARCH_LIBS(sem_init, pthread rt posix4) # check if we need libintl for locale functions AC_CHECK_LIB(intl, textdomain, @@ -3410,7 +3414,8 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \ futimens futimes gai_strerror getentropy \ getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \ getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \ - initgroups kill killpg lchmod lchown linkat lstat lutimes mmap \ + if_nameindex \ + initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \ memrchr mbrtowc mkdirat mkfifo \ mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \ posix_fallocate posix_fadvise pread \ @@ -3759,40 +3764,6 @@ AC_LINK_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(no) ]) -# On Android API level 24 with android-ndk-r13, if_nameindex() is available, -# but the if_nameindex structure is not defined. -AC_MSG_CHECKING(for if_nameindex) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[ -#include -#ifdef STDC_HEADERS -# include -# include -#else -# ifdef HAVE_STDLIB_H -# include -# endif -#endif -#ifdef HAVE_SYS_SOCKET_H -# include -#endif -#ifdef HAVE_NET_IF_H -# include -#endif -]], [[struct if_nameindex *ni = if_nameindex(); int x = ni[0].if_index;]])], - [AC_DEFINE(HAVE_IF_NAMEINDEX, 1, Define to 1 if you have the 'if_nameindex' function.) - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) -]) - -# Issue #28762: lockf() is available on Android API level 24, but the F_LOCK -# macro is not defined in android-ndk-r13. -AC_MSG_CHECKING(for lockf) -AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]],[[lockf(0, F_LOCK, 0);]])], - [AC_DEFINE(HAVE_LOCKF, 1, Define to 1 if you have the 'lockf' function and the F_LOCK macro.) - AC_MSG_RESULT(yes)], - [AC_MSG_RESULT(no) -]) - # On OSF/1 V5.1, getaddrinfo is available, but a define # for [no]getaddrinfo in netdb.h. AC_MSG_CHECKING(for getaddrinfo) diff --git a/pyconfig.h.in b/pyconfig.h.in index 6e0f3e8eeb37b5..66b9e888274500 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -601,7 +601,7 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_VM_SOCKETS_H -/* Define to 1 if you have the 'lockf' function and the F_LOCK macro. */ +/* Define to 1 if you have the `lockf' function. */ #undef HAVE_LOCKF /* Define to 1 if you have the `log1p' function. */ diff --git a/setup.py b/setup.py index f284301ea4ccaa..c22de17f953396 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,31 @@ def add_dir_to_list(dirlist, dir): return dirlist.insert(0, dir) +def sysroot_paths(make_vars, subdirs): + """Get the paths of sysroot sub-directories. + + * make_vars: a sequence of names of variables of the Makefile where + sysroot may be set. + * subdirs: a sequence of names of subdirectories used as the location for + headers or libraries. + """ + + dirs = [] + for var_name in make_vars: + var = sysconfig.get_config_var(var_name) + if var is not None: + m = re.search(r'--sysroot=([^"]\S*|"[^"]+")', var) + if m is not None: + sysroot = m.group(1).strip('"') + for subdir in subdirs: + if os.path.isabs(subdir): + subdir = subdir[1:] + path = os.path.join(sysroot, subdir) + if os.path.isdir(path): + dirs.append(path) + break + return dirs + def macosx_sdk_root(): """ Return the directory of the current OSX SDK, @@ -501,13 +526,6 @@ def add_gcc_paths(self): finally: os.unlink(tmpfile) - def detect_math_libs(self): - # Check for MacOS X, which doesn't need libm.a at all - if host_platform == 'darwin': - return [] - else: - return ['m'] - def detect_modules(self): # Ensure that /usr/local is always used, but the local build # directories (i.e. '.' and 'Include') must be first. See issue @@ -566,18 +584,23 @@ def detect_modules(self): add_dir_to_list(self.compiler.include_dirs, sysconfig.get_config_var("INCLUDEDIR")) + system_lib_dirs = ['/lib64', '/usr/lib64', '/lib', '/usr/lib'] + system_include_dirs = ['/usr/include'] # lib_dirs and inc_dirs are used to search for files; # if a file is found in one of those directories, it can # be assumed that no additional -I,-L directives are needed. if not cross_compiling: - lib_dirs = self.compiler.library_dirs + [ - '/lib64', '/usr/lib64', - '/lib', '/usr/lib', - ] - inc_dirs = self.compiler.include_dirs + ['/usr/include'] + lib_dirs = self.compiler.library_dirs + system_lib_dirs + inc_dirs = self.compiler.include_dirs + system_include_dirs else: - lib_dirs = self.compiler.library_dirs[:] - inc_dirs = self.compiler.include_dirs[:] + # Add the sysroot paths. 'sysroot' is a compiler option used to + # set the logical path of the standard system headers and + # libraries. + lib_dirs = (self.compiler.library_dirs + + sysroot_paths(('LDFLAGS', 'CC'), system_lib_dirs)) + inc_dirs = (self.compiler.include_dirs + + sysroot_paths(('CPPFLAGS', 'CFLAGS', 'CC'), + system_include_dirs)) exts = [] missing = [] @@ -613,8 +636,6 @@ def detect_modules(self): if item.startswith('-L'): lib_dirs.append(item[2:]) - math_libs = self.detect_math_libs() - # # The following modules are all pretty straightforward, and compile # on pretty much any POSIXish platform. @@ -628,12 +649,12 @@ def detect_modules(self): exts.append( Extension('cmath', ['cmathmodule.c'], extra_objects=[shared_math], depends=['_math.h', shared_math], - libraries=math_libs) ) + libraries=['m']) ) # math library functions, e.g. sin() exts.append( Extension('math', ['mathmodule.c'], extra_objects=[shared_math], depends=['_math.h', shared_math], - libraries=math_libs) ) + libraries=['m']) ) # time libraries: librt may be needed for clock_gettime() time_libs = [] @@ -644,10 +665,10 @@ def detect_modules(self): # time operations and variables exts.append( Extension('time', ['timemodule.c'], libraries=time_libs) ) - # math_libs is needed by delta_new() that uses round() and by accum() - # that uses modf(). + # libm is needed by delta_new() that uses round() and by accum() that + # uses modf(). exts.append( Extension('_datetime', ['_datetimemodule.c'], - libraries=math_libs) ) + libraries=['m']) ) # random number generator implemented in C exts.append( Extension("_random", ["_randommodule.c"]) ) # bisect @@ -732,9 +753,9 @@ def detect_modules(self): # According to #993173, this one should actually work fine on # 64-bit platforms. # - # audioop needs math_libs for floor() in multiple functions. + # audioop needs libm for floor() in multiple functions. exts.append( Extension('audioop', ['audioop.c'], - libraries=math_libs) ) + libraries=['m']) ) # readline do_readline = self.compiler.find_library_file(lib_dirs, 'readline') @@ -1659,12 +1680,11 @@ class db_found(Exception): pass # Build the _uuid module if possible uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"]) - if uuid_incs: + if uuid_incs is not None: if self.compiler.find_library_file(lib_dirs, 'uuid'): uuid_libs = ['uuid'] else: uuid_libs = [] - if uuid_incs: self.extensions.append(Extension('_uuid', ['_uuidmodule.c'], libraries=uuid_libs, include_dirs=uuid_incs)) @@ -1972,7 +1992,6 @@ def detect_ctypes(self, inc_dirs, lib_dirs): '_ctypes/stgdict.c', '_ctypes/cfield.c'] depends = ['_ctypes/ctypes.h'] - math_libs = self.detect_math_libs() if host_platform == 'darwin': sources.append('_ctypes/malloc_closure.c') @@ -2003,10 +2022,10 @@ def detect_ctypes(self, inc_dirs, lib_dirs): libraries=[], sources=sources, depends=depends) - # function my_sqrt() needs math library for sqrt() + # function my_sqrt() needs libm for sqrt() ext_test = Extension('_ctypes_test', sources=['_ctypes/_ctypes_test.c'], - libraries=math_libs) + libraries=['m']) self.extensions.extend([ext, ext_test]) if host_platform == 'darwin': @@ -2050,7 +2069,6 @@ def _decimal_ext(self): 'Modules', '_decimal', 'libmpdec'))] - libraries = self.detect_math_libs() sources = [ '_decimal/_decimal.c', '_decimal/libmpdec/basearith.c', @@ -2146,7 +2164,7 @@ def _decimal_ext(self): ext = Extension ( '_decimal', include_dirs=include_dirs, - libraries=libraries, + libraries=['m'], define_macros=define_macros, undef_macros=undef_macros, extra_compile_args=extra_compile_args,