gh-115816: Improve internal symbols API in optimizer

Doc/library/ast.rst

@@ -103,20 +103,15 @@ Node classes
    For example, to create and populate an :class:`ast.UnaryOp` node, you could
    use ::
 
-      node = ast.UnaryOp()
-      node.op = ast.USub()
-      node.operand = ast.Constant()
-      node.operand.value = 5
-      node.operand.lineno = 0
-      node.operand.col_offset = 0
-      node.lineno = 0
-      node.col_offset = 0
-
-   or the more compact ::
-
       node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0),
                          lineno=0, col_offset=0)
 
+   If a field that is optional in the grammar is omitted from the constructor,
+   it defaults to ``None``. If a list field is omitted, it defaults to the empty
+   list. If any other field is omitted, a :exc:`DeprecationWarning` is raised
+   and the AST node will not have this field. In Python 3.15, this condition will
+   raise an error.
+
 .. versionchanged:: 3.8
 
    Class :class:`ast.Constant` is now used for all constants.
@@ -140,6 +135,14 @@ Node classes
    In the meantime, instantiating them will return an instance of
    a different class.
 
+.. deprecated-removed:: 3.13 3.15
+
+   Previous versions of Python allowed the creation of AST nodes that were missing
+   required fields. Similarly, AST node constructors allowed arbitrary keyword
+   arguments that were set as attributes of the AST node, even if they did not
+   match any of the fields of the AST node. This behavior is deprecated and will
+   be removed in Python 3.15.
+
 .. note::
     The descriptions of the specific node classes displayed here
     were initially adapted from the fantastic `Green Tree

Doc/whatsnew/3.13.rst

@@ -206,6 +206,21 @@ array
 ast
 ---
 
+* The constructors of node types in the :mod:`ast` module are now stricter
+  in the arguments they accept, and have more intuitive behaviour when
+  arguments are omitted.
+
+  If an optional field on an AST node is not included as an argument when
+  constructing an instance, the field will now be set to ``None``. Similarly,
+  if a list field is omitted, that field will now be set to an empty list.
+  (Previously, in both cases, the attribute would be missing on the newly
+  constructed AST node instance.)
+
+  If other arguments are omitted, a :exc:`DeprecationWarning` is emitted.
+  This will cause an exception in Python 3.15. Similarly, passing a keyword
+  argument that does not map to a field on the AST node is now deprecated,
+  and will raise an exception in Python 3.15.
+
 * :func:`ast.parse` now accepts an optional argument ``optimize``
   which is passed on to the :func:`compile` built-in. This makes it
   possible to obtain an optimized ``AST``.

Include/internal/pycore_global_strings.h

@@ -242,6 +242,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(_check_retval_)
         STRUCT_FOR_ID(_dealloc_warn)
         STRUCT_FOR_ID(_feature_version)
+        STRUCT_FOR_ID(_field_types)
         STRUCT_FOR_ID(_fields_)
         STRUCT_FOR_ID(_finalizing)
         STRUCT_FOR_ID(_find_and_load)

Include/internal/pycore_optimizer.h

@@ -27,12 +27,12 @@ extern PyTypeObject _PyUOpExecutor_Type;
 extern PyTypeObject _PyUOpOptimizer_Type;
 
 /* Symbols */
+/* See explanation in optimizer_symbols.c */
 
 struct _Py_UopsSymbol {
-    int flags;
-    PyTypeObject *typ;
-    // constant propagated value (might be NULL)
-    PyObject *const_val;
+    int flags;  // 0 bits: Top; 2 or more bits: Bottom
+    PyTypeObject *typ;  // Borrowed reference
+    PyObject *const_val;  // Owned reference (!)
 };
 
 // Holds locals, stack, locals, stack ... co_consts (in that order)
@@ -92,7 +92,9 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *con
 extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx);
 extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ);
 extern void _Py_uop_sym_set_null(_Py_UopsSymbol *sym);
-extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *tp);
+extern void _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym);
+extern void _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ);
+extern void _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val);
 
 extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx);
 extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx);

Lib/asyncio/events.py

@@ -54,7 +54,8 @@ def _repr_info(self):
             info.append('cancelled')
         if self._callback is not None:
             info.append(format_helpers._format_callback_source(
-                self._callback, self._args))
+                self._callback, self._args,
+                debug=self._loop.get_debug()))
         if self._source_traceback:
             frame = self._source_traceback[-1]
             info.append(f'created at {frame[0]}:{frame[1]}')
@@ -90,7 +91,8 @@ def _run(self):
             raise
         except BaseException as exc:
             cb = format_helpers._format_callback_source(
-                self._callback, self._args)
+                self._callback, self._args,
+                debug=self._loop.get_debug())
             msg = f'Exception in callback {cb}'
             context = {
                 'message': msg,

Lib/asyncio/format_helpers.py

@@ -19,19 +19,26 @@ def _get_function_source(func):
     return None
 
 
-def _format_callback_source(func, args):
-    func_repr = _format_callback(func, args, None)
+def _format_callback_source(func, args, *, debug=False):
+    func_repr = _format_callback(func, args, None, debug=debug)
     source = _get_function_source(func)
     if source:
         func_repr += f' at {source[0]}:{source[1]}'
     return func_repr
 
 
-def _format_args_and_kwargs(args, kwargs):
+def _format_args_and_kwargs(args, kwargs, *, debug=False):
     """Format function arguments and keyword arguments.
 
     Special case for a single parameter: ('hello',) is formatted as ('hello').
+
+    Note that this function only returns argument details when
+    debug=True is specified, as arguments may contain sensitive
+    information.
     """
+    if not debug:
+        return '()'
+
     # use reprlib to limit the length of the output
     items = []
     if args:
@@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs):
     return '({})'.format(', '.join(items))
 
 
-def _format_callback(func, args, kwargs, suffix=''):
+def _format_callback(func, args, kwargs, *, debug=False, suffix=''):
     if isinstance(func, functools.partial):
-        suffix = _format_args_and_kwargs(args, kwargs) + suffix
-        return _format_callback(func.func, func.args, func.keywords, suffix)
+        suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix
+        return _format_callback(func.func, func.args, func.keywords,
+                                debug=debug, suffix=suffix)
 
     if hasattr(func, '__qualname__') and func.__qualname__:
         func_repr = func.__qualname__
@@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''):
     else:
         func_repr = repr(func)
 
-    func_repr += _format_args_and_kwargs(args, kwargs)
+    func_repr += _format_args_and_kwargs(args, kwargs, debug=debug)
     if suffix:
         func_repr += suffix
     return func_repr

Lib/asyncio/streams.py

@@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None):
             # is established.
             self._strong_reader = stream_reader
         self._reject_connection = False
-        self._stream_writer = None
         self._task = None
         self._transport = None
         self._client_connected_cb = client_connected_cb
@@ -214,10 +213,8 @@ def _stream_reader(self):
             return None
         return self._stream_reader_wr()
 
-    def _replace_writer(self, writer):
+    def _replace_transport(self, transport):
         loop = self._loop
-        transport = writer.transport
-        self._stream_writer = writer
         self._transport = transport
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
 
@@ -239,11 +236,8 @@ def connection_made(self, transport):
             reader.set_transport(transport)
         self._over_ssl = transport.get_extra_info('sslcontext') is not None
         if self._client_connected_cb is not None:
-            self._stream_writer = StreamWriter(transport, self,
-                                               reader,
-                                               self._loop)
-            res = self._client_connected_cb(reader,
-                                            self._stream_writer)
+            writer = StreamWriter(transport, self, reader, self._loop)
+            res = self._client_connected_cb(reader, writer)
             if coroutines.iscoroutine(res):
                 def callback(task):
                     if task.cancelled():
@@ -405,7 +399,7 @@ async def start_tls(self, sslcontext, *,
             ssl_handshake_timeout=ssl_handshake_timeout,
             ssl_shutdown_timeout=ssl_shutdown_timeout)
         self._transport = new_transport
-        protocol._replace_writer(self)
+        protocol._replace_transport(new_transport)
 
     def __del__(self, warnings=warnings):
         if not self._transport.is_closing():