Merge branch 'cmaloney/systrace_helper_wip' into cmaloney/pyio_merged

Author: cmaloney

Doc/c-api/long.rst

@@ -405,14 +405,13 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
 
    Passing zero to *n_bytes* will return the size of a buffer that would
    be large enough to hold the value. This may be larger than technically
-   necessary, but not unreasonably so.
+   necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be
+   ``NULL``.
 
    .. note::
 
       Passing *n_bytes=0* to this function is not an accurate way to determine
-      the bit length of a value.
-
-   If *n_bytes=0*, *buffer* may be ``NULL``.
+      the bit length of the value.
 
    To get at the entire Python value of an unknown size, the function can be
    called twice: first to determine the buffer size, then to fill it::
@@ -462,6 +461,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN   ``3``
    .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4``
    .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8``
+   .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX     ``16``
    ============================================= ======
 
    Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian
@@ -483,6 +483,13 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    provided there is enough space for at least one sign bit, regardless of
    whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified.
 
+   If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is
+   passed, its :meth:`~object.__index__` method will be called first. This may
+   result in Python code executing and other threads being allowed to run, which
+   could cause changes to other objects or values in use. When *flags* is
+   ``-1``, this option is not set, and non-integer values will raise
+   :exc:`TypeError`.
+
    .. note::
 
       With the default *flags* (``-1``, or *UNSIGNED_BUFFER*  without

Doc/whatsnew/3.14.rst

@@ -125,6 +125,14 @@ symtable
 Optimizations
 =============
 
+asyncio
+-------
+
+* :mod:`asyncio` now uses double linked list implementation for native tasks
+  which speeds up execution by 10% on standard pyperformance benchmarks and
+  reduces memory usage.
+  (Contributed by Kumar Aditya in :gh:`107803`.)
+
 
 
 

Include/cpython/longobject.h

@@ -10,6 +10,7 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base);
 #define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3
 #define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4
 #define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8
+#define Py_ASNATIVEBYTES_ALLOW_INDEX 16
 
 /* PyLong_AsNativeBytes: Copy the integer value to a native variable.
    buffer points to the first byte of the variable.
@@ -20,8 +21,10 @@ PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base);
    * 2 - native endian
    * 4 - unsigned destination (e.g. don't reject copying 255 into one byte)
    * 8 - raise an exception for negative inputs
-   If flags is -1 (all bits set), native endian is used and value truncation
-   behaves most like C (allows negative inputs and allow MSB set).
+   * 16 - call __index__ on non-int types
+   If flags is -1 (all bits set), native endian is used, value truncation
+   behaves most like C (allows negative inputs and allow MSB set), and non-int
+   objects will raise a TypeError.
    Big endian mode will write the most significant byte into the address
    directly referenced by buffer; little endian will write the least significant
    byte into that address.

Include/pymacro.h

@@ -46,7 +46,8 @@
 /* Argument must be a char or an int in [-128, 127] or [0, 255]. */
 #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff))
 
-#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
+     && !defined(_MSC_VER))
 #  define Py_BUILD_ASSERT_EXPR(cond) \
     ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \
      0)

Lib/argparse.py

@@ -1791,7 +1791,7 @@ def _get_kwargs(self):
     # ==================================
     def add_subparsers(self, **kwargs):
         if self._subparsers is not None:
-            self.error(_('cannot have multiple subparser arguments'))
+            raise ArgumentError(None, _('cannot have multiple subparser arguments'))
 
         # add the parser class to the arguments if it's not present
         kwargs.setdefault('parser_class', type(self))
@@ -1846,7 +1846,8 @@ def parse_args(self, args=None, namespace=None):
             msg = _('unrecognized arguments: %s') % ' '.join(argv)
             if self.exit_on_error:
                 self.error(msg)
-            raise ArgumentError(None, msg)
+            else:
+                raise ArgumentError(None, msg)
         return args
 
     def parse_known_args(self, args=None, namespace=None):
@@ -2135,7 +2136,7 @@ def consume_positionals(start_index):
                                 self._get_value(action, action.default))
 
         if required_actions:
-            self.error(_('the following arguments are required: %s') %
+            raise ArgumentError(None, _('the following arguments are required: %s') %
                        ', '.join(required_actions))
 
         # make sure all required groups had one option present
@@ -2151,7 +2152,7 @@ def consume_positionals(start_index):
                              for action in group._group_actions
                              if action.help is not SUPPRESS]
                     msg = _('one of the arguments %s is required')
-                    self.error(msg % ' '.join(names))
+                    raise ArgumentError(None, msg % ' '.join(names))
 
         # return the updated namespace and the extra arguments
         return namespace, extras
@@ -2178,7 +2179,7 @@ def _read_args_from_files(self, arg_strings):
                         arg_strings = self._read_args_from_files(arg_strings)
                         new_arg_strings.extend(arg_strings)
                 except OSError as err:
-                    self.error(str(err))
+                    raise ArgumentError(None, str(err))
 
         # return the modified argument list
         return new_arg_strings
@@ -2258,7 +2259,7 @@ def _parse_optional(self, arg_string):
                 for action, option_string, sep, explicit_arg in option_tuples])
             args = {'option': arg_string, 'matches': options}
             msg = _('ambiguous option: %(option)s could match %(matches)s')
-            self.error(msg % args)
+            raise ArgumentError(None, msg % args)
 
         # if exactly one action matched, this segmentation is good,
         # so return the parsed action
@@ -2318,7 +2319,7 @@ def _get_option_tuples(self, option_string):
 
         # shouldn't ever get here
         else:
-            self.error(_('unexpected option string: %s') % option_string)
+            raise ArgumentError(None, _('unexpected option string: %s') % option_string)
 
         # return the collected option tuples
         return result
@@ -2375,8 +2376,11 @@ def _get_nargs_pattern(self, action):
     def parse_intermixed_args(self, args=None, namespace=None):
         args, argv = self.parse_known_intermixed_args(args, namespace)
         if argv:
-            msg = _('unrecognized arguments: %s')
-            self.error(msg % ' '.join(argv))
+            msg = _('unrecognized arguments: %s') % ' '.join(argv)
+            if self.exit_on_error:
+                self.error(msg)
+            else:
+                raise ArgumentError(None, msg)
         return args
 
     def parse_known_intermixed_args(self, args=None, namespace=None):

Lib/asyncio/unix_events.py

@@ -867,9 +867,6 @@ class _PidfdChildWatcher:
     recent (5.3+) kernels.
     """
 
-    def is_active(self):
-        return True
-
     def add_child_handler(self, pid, callback, *args):
         loop = events.get_running_loop()
         pidfd = os.pidfd_open(pid)
@@ -911,9 +908,6 @@ def __init__(self):
         self._pid_counter = itertools.count(0)
         self._threads = {}
 
-    def is_active(self):
-        return True
-
     def __del__(self, _warn=warnings.warn):
         threads = [thread for thread in list(self._threads.values())
                    if thread.is_alive()]

Lib/test/support/strace_helper.py

@@ -0,0 +1,173 @@
+import re
+import sys
+import textwrap
+import unittest
+from dataclasses import dataclass
+from test import support
+from test.support.script_helper import run_python_until_end
+from typing import Dict, List
+
+_strace_binary = "/usr/bin/strace"
+_syscall_regex = re.compile(
+    r"(?P<syscall>[^(]*)\((?P<args>[^)]*)\)\s*[=]\s*(?P<returncode>.+)")
+_returncode_regex = re.compile(
+    r"\+\+\+ exited with (?P<returncode>\d+) \+\+\+")
+
+# Cached value of whether or not there is a compatible strace binary
+_strace_working: bool | None = None
+
+
+@dataclass
+class StraceEvent:
+    syscall: str
+    args: List[str]
+    returncode: str
+
+
+@dataclass
+class StraceResult:
+    strace_returncode: int
+    python_returncode: int
+    _raw_events: str
+    stdout: str
+    stderr: str
+
+    def events(self) -> List[StraceEvent]:
+        """Extract the call list information from _raw_events"""
+        matches = [
+            _syscall_regex.match(event)
+            for event in self._raw_events.splitlines()
+        ]
+        return [
+            StraceEvent(match["syscall"],
+                        [arg.strip() for arg in (match["args"].split(","))],
+                        match["returncode"]) for match in matches if match
+        ]
+
+    def sections(self) -> Dict[str:List[StraceEvent]]:
+        """Find all "MARK <X>" writes and use them to make groups of events.
+
+        This is useful to avoid variable / overhead strace events, like that
+        at interpreter startup, so a test can just check does the small case
+        under study work."""
+        current_section = "__startup"
+        sections = {current_section: []}
+        for event in self.events():
+            if event.syscall == 'write' and len(
+                    event.args) > 2 and event.args[1].startswith("\"MARK "):
+                # Found a new section, don't include the write in the section
+                # but all events until next mark should be in that section
+                current_section = event.args[1].split(
+                    " ", 1)[1].removesuffix('\\n"')
+                if current_section not in sections:
+                    sections[current_section] = list()
+            else:
+                sections[current_section].append(event)
+
+        return sections
+
+
+@support.requires_subprocess()
+def strace_python(code: str,
+                  strace_flags: List[str],
+                  check: bool = True) -> StraceResult:
+    """Run strace and return the trace.
+
+    Sets strace_returncode and python_returncode to `-1` on error
+    """
+    res = None
+
+    def _make_error(reason, details):
+        return StraceResult(
+            strace_returncode=-1,
+            python_returncode=-1,
+            _raw_events=f"error({reason},details={details}) = -1",
+            stdout=res.out if res else "",
+            stderr=res.err if res else "")
+
+    # Run strace, and get out the raw text
+    try:
+        res, cmd_line = run_python_until_end(
+            "-c",
+            textwrap.dedent(code),
+            __run_using_command=[_strace_binary] + strace_flags)
+    except OSError as err:
+        return _make_error("Caught OSError", err)
+
+    if check and res.rc:
+        res.fail(cmd_line)
+
+    # Get out program returncode
+    decoded = res.err.decode().strip()
+
+    output = decoded.rsplit("\n", 1)
+    if len(output) != 2:
+        return _make_error("Expected strace events and exit code line",
+                           decoded[-50:])
+
+    returncode_match = _returncode_regex.match(output[1])
+    if not returncode_match:
+        return _make_error("Expected to find returncode in last line.",
+                           output[1][:50])
+
+    python_returncode = int(returncode_match["returncode"])
+    if check and python_returncode:
+        res.fail(cmd_line)
+
+    return StraceResult(strace_returncode=res.rc,
+                        python_returncode=python_returncode,
+                        _raw_events=output[0],
+                        stdout=res.out,
+                        stderr=res.err)
+
+
+def _get_events(code: str, strace_flags: List[str], prelude: str,
+                cleanup: str) -> List[StraceEvent]:
+    # NOTE: The flush is currently required to prevent the prints from getting
+    # buffered and done all at once at exit
+    prelude = textwrap.dedent(prelude)
+    code = textwrap.dedent(code)
+    cleanup = textwrap.dedent(cleanup)
+    to_run = f"""
+print("MARK prelude", flush=True)
+{prelude}
+print("MARK code", flush=True)
+{code}
+print("MARK cleanup", flush=True)
+{cleanup}
+print("MARK __shutdown", flush=True)
+    """
+    trace = strace_python(to_run, strace_flags)
+    all_sections = trace.sections()
+    return all_sections['code']
+
+
+def get_syscalls(code: str,
+                 strace_flags: List[str],
+                 prelude: str = "",
+                 cleanup: str = "") -> List[str]:
+    """Get the syscalls which a given chunk of python code generates"""
+    events = _get_events(code, strace_flags, prelude=prelude, cleanup=cleanup)
+    return [ev.syscall for ev in events]
+
+
+def _can_strace():
+    res = strace_python("import sys; sys.exit(0)", [], check=False)
+    assert res.events(), "Should have parsed multiple calls"
+
+    global _strace_working
+    _strace_working = res.strace_returncode == 0 and res.python_returncode == 0
+
+
+def requires_strace():
+    if sys.platform != "linux":
+        return unittest.skip("Linux only, requires strace.")
+    # Moderately expensive (spawns a subprocess), so share results when possible.
+    if _strace_working is None:
+        _can_strace()
+
+    assert _strace_working is not None, "Should have been set by _can_strace"
+    return unittest.skipUnless(_strace_working, "Requires working strace")
+
+
+__all__ = ["requires_strace", "strace_python", "StraceResult", "StraceEvent"]