Merge remote-tracking branch 'upstream/main' into pythongh-116168-remove-extra-_check_stack_space-ops

Author: lazorchakp

Doc/library/datetime.rst

@@ -1079,6 +1079,24 @@ Other constructors, all class methods:
    time tuple.  See also :ref:`strftime-strptime-behavior` and
    :meth:`datetime.fromisoformat`.
 
+   .. versionchanged:: 3.13
+
+      If *format* specifies a day of month without a year a
+      :exc:`DeprecationWarning` is now emitted.  This is to avoid a quadrennial
+      leap year bug in code seeking to parse only a month and day as the
+      default year used in absence of one in the format is not a leap year.
+      Such *format* values may raise an error as of Python 3.15.  The
+      workaround is to always include a year in your *format*.  If parsing
+      *date_string* values that do not have a year, explicitly add a year that
+      is a leap year before parsing:
+
+      .. doctest::
+
+         >>> from datetime import datetime
+         >>> date_string = "02/29"
+         >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y")  # Avoids leap year bug.
+         >>> when.strftime("%B %d")  # doctest: +SKIP
+         'February 29'
 
 
 Class attributes:
@@ -2657,6 +2675,25 @@ Notes:
    for  formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``,
    ``%W``, and ``%V``. Format ``%y`` does require a leading zero.
 
+(10)
+   When parsing a month and day using :meth:`~.datetime.strptime`, always
+   include a year in the format.  If the value you need to parse lacks a year,
+   append an explicit dummy leap year.  Otherwise your code will raise an
+   exception when it encounters leap day because the default year used by the
+   parser is not a leap year.  Users run into this bug every four years...
+
+   .. doctest::
+
+      >>> month_day = "02/29"
+      >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y")  # No leap year bug.
+      datetime.datetime(1984, 2, 29, 0, 0)
+
+   .. deprecated-removed:: 3.13 3.15
+      :meth:`~.datetime.strptime` calls using a format string containing
+      a day of month without a year now emit a
+      :exc:`DeprecationWarning`. In 3.15 or later we may change this into
+      an error or change the default year to a leap year. See :gh:`70647`.
+
 .. rubric:: Footnotes
 
 .. [#] If, that is, we ignore the effects of Relativity

Doc/library/importlib.resources.abc.rst

@@ -109,13 +109,35 @@
 
        Return True if self is a file.
 
-    .. abstractmethod:: joinpath(child)
+    .. abstractmethod:: joinpath(*pathsegments)
 
-       Return Traversable child in self.
+       Traverse directories according to *pathsegments* and return
+       the result as :class:`!Traversable`.
+
+       Each *pathsegments* argument may contain multiple names separated by
+       forward slashes (``/``, ``posixpath.sep`` ).
+       For example, the following are equivalent::
+
+           files.joinpath('subdir', 'subsuddir', 'file.txt')
+           files.joinpath('subdir/subsuddir/file.txt')
+
+       Note that some :class:`!Traversable` implementations
+       might not be updated to the latest version of the protocol.
+       For compatibility with such implementations, provide a single argument
+       without path separators to each call to ``joinpath``. For example::
+
+           files.joinpath('subdir').joinpath('subsubdir').joinpath('file.txt')
+
+       .. versionchanged:: 3.11
+
+          ``joinpath`` accepts multiple *pathsegments*, and these segments
+          may contain forward slashes as path separators.
+          Previously, only a single *child* argument was accepted.
 
     .. abstractmethod:: __truediv__(child)
 
        Return Traversable child in self.
+       Equivalent to ``joinpath(child)``.
 
     .. abstractmethod:: open(mode='r', *args, **kwargs)
 

Doc/library/unittest.rst

@@ -1880,8 +1880,8 @@ Loading and running tests
       Python identifiers) will be loaded.
 
       All test modules must be importable from the top level of the project. If
-      the start directory is not the top level directory then the top level
-      directory must be specified separately.
+      the start directory is not the top level directory then *top_level_dir*
+      must be specified separately.
 
       If importing a module fails, for example due to a syntax error, then
       this will be recorded as a single error and discovery will continue.  If
@@ -1901,9 +1901,11 @@ Loading and running tests
       package.
 
       The pattern is deliberately not stored as a loader attribute so that
-      packages can continue discovery themselves. *top_level_dir* is stored so
-      ``load_tests`` does not need to pass this argument in to
-      ``loader.discover()``.
+      packages can continue discovery themselves.
+
+      *top_level_dir* is stored internally, and used as a default to any
+      nested calls to ``discover()``. That is, if a package's ``load_tests``
+      calls ``loader.discover()``, it does not need to pass this argument.
 
       *start_dir* can be a dotted module name as well as a directory.
 
@@ -1930,6 +1932,9 @@ Loading and running tests
          *start_dir* can not be a :term:`namespace packages <namespace package>`.
          It has been broken since Python 3.7 and Python 3.11 officially remove it.
 
+      .. versionchanged:: 3.13
+         *top_level_dir* is only stored for the duration of *discover* call.
+
 
    The following attributes of a :class:`TestLoader` can be configured either by
    subclassing or assignment on an instance:

Grammar/python.gram

@@ -1013,6 +1013,7 @@ kwargs[asdl_seq*]:
 starred_expression[expr_ty]:
     | invalid_starred_expression
     | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) }
+    | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") }
 
 kwarg_or_starred[KeywordOrStarred*]:
     | invalid_kwarg
@@ -1133,8 +1134,8 @@ func_type_comment[Token*]:
 
 # From here on, there are rules for invalid syntax with specialised error messages
 invalid_arguments:
-    | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' b='*' {
-        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(b, "iterable argument unpacking follows keyword argument unpacking") }
+    | ((','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) a=',' ','.(starred_expression !'=')+ {
+        RAISE_SYNTAX_ERROR_STARTING_FROM(a, "iterable argument unpacking follows keyword argument unpacking") }
     | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] {
         RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, _PyPegen_get_last_comprehension_item(PyPegen_last_item(b, comprehension_ty)), "Generator expression must be parenthesized") }
     | a=NAME b='=' expression for_if_clauses {
@@ -1396,6 +1397,7 @@ invalid_kvpair:
     | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") }
 invalid_starred_expression:
     | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") }
+
 invalid_replacement_field:
     | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") }
     | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") }

Include/cpython/compile.h

@@ -32,28 +32,8 @@ typedef struct {
 #define _PyCompilerFlags_INIT \
     (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION}
 
-/* source location information */
-typedef struct {
-    int lineno;
-    int end_lineno;
-    int col_offset;
-    int end_col_offset;
-} _PyCompilerSrcLocation;
-
-#define SRC_LOCATION_FROM_AST(n) \
-    (_PyCompilerSrcLocation){ \
-               .lineno = (n)->lineno, \
-               .end_lineno = (n)->end_lineno, \
-               .col_offset = (n)->col_offset, \
-               .end_col_offset = (n)->end_col_offset }
-
 /* Future feature support */
 
-typedef struct {
-    int ff_features;                    /* flags set by future statements */
-    _PyCompilerSrcLocation ff_location; /* location of last future statement */
-} PyFutureFeatures;
-
 #define FUTURE_NESTED_SCOPES "nested_scopes"
 #define FUTURE_GENERATORS "generators"
 #define FUTURE_DIVISION "division"

Include/cpython/pystats.h

@@ -77,12 +77,11 @@ typedef struct _object_stats {
     uint64_t frees;
     uint64_t to_freelist;
     uint64_t from_freelist;
-    uint64_t new_values;
+    uint64_t inline_values;
     uint64_t dict_materialized_on_request;
     uint64_t dict_materialized_new_key;
     uint64_t dict_materialized_too_big;
     uint64_t dict_materialized_str_subclass;
-    uint64_t dict_dematerialized;
     uint64_t type_cache_hits;
     uint64_t type_cache_misses;
     uint64_t type_cache_dunder_hits;

Include/internal/pycore_bytes_methods.h

@@ -32,8 +32,12 @@ extern PyObject *_Py_bytes_rfind(const char *str, Py_ssize_t len, PyObject *args
 extern PyObject *_Py_bytes_rindex(const char *str, Py_ssize_t len, PyObject *args);
 extern PyObject *_Py_bytes_count(const char *str, Py_ssize_t len, PyObject *args);
 extern int _Py_bytes_contains(const char *str, Py_ssize_t len, PyObject *arg);
-extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len, PyObject *args);
-extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len, PyObject *args);
+extern PyObject *_Py_bytes_startswith(const char *str, Py_ssize_t len,
+                                      PyObject *subobj, Py_ssize_t start,
+                                      Py_ssize_t end);
+extern PyObject *_Py_bytes_endswith(const char *str, Py_ssize_t len,
+                                    PyObject *subobj, Py_ssize_t start,
+                                    Py_ssize_t end);
 
 /* The maketrans() static method. */
 extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to);

Include/internal/pycore_code.h

@@ -79,7 +79,10 @@ typedef struct {
 typedef struct {
     uint16_t counter;
     uint16_t type_version[2];
-    uint16_t keys_version[2];
+    union {
+        uint16_t keys_version[2];
+        uint16_t dict_offset;
+    };
     uint16_t descr[4];
 } _PyLoadMethodCache;
 

Include/internal/pycore_compile.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_symtable.h"  // _Py_SourceLocation
+
 struct _arena;   // Type defined in pycore_pyarena.h
 struct _mod;     // Type defined in pycore_ast.h
 
@@ -27,7 +29,7 @@ extern int _PyCompile_AstOptimize(
     int optimize,
     struct _arena *arena);
 
-static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1};
+struct _Py_SourceLocation;
 
 extern int _PyAST_Optimize(
     struct _mod *,
@@ -44,7 +46,7 @@ typedef struct {
 typedef struct {
     int i_opcode;
     int i_oparg;
-    _PyCompilerSrcLocation i_loc;
+    _Py_SourceLocation i_loc;
     _PyCompile_ExceptHandlerInfo i_except_handler_info;
 
     /* Used by the assembler */
@@ -65,7 +67,7 @@ typedef struct {
 int _PyCompile_InstructionSequence_UseLabel(_PyCompile_InstructionSequence *seq, int lbl);
 int _PyCompile_InstructionSequence_Addop(_PyCompile_InstructionSequence *seq,
                                          int opcode, int oparg,
-                                         _PyCompilerSrcLocation loc);
+                                         _Py_SourceLocation loc);
 int _PyCompile_InstructionSequence_ApplyLabelMap(_PyCompile_InstructionSequence *seq);
 
 typedef struct {

Include/internal/pycore_dict.h

@@ -11,7 +11,7 @@ extern "C" {
 
 #include "pycore_freelist.h"      // _PyFreeListState
 #include "pycore_identifier.h"    // _Py_Identifier
-#include "pycore_object.h"        // PyDictOrValues
+#include "pycore_object.h"        // PyManagedDictPointer
 
 // Unsafe flavor of PyDict_GetItemWithError(): no error checking
 extern PyObject* _PyDict_GetItemWithError(PyObject *dp, PyObject *key);
@@ -181,6 +181,10 @@ struct _dictkeysobject {
  * [-1] = prefix size. [-2] = used size. size[-2-n...] = insertion order.
  */
 struct _dictvalues {
+    uint8_t capacity;
+    uint8_t size;
+    uint8_t embedded;
+    uint8_t valid;
     PyObject *values[1];
 };
 
@@ -196,6 +200,7 @@ static inline void* _DK_ENTRIES(PyDictKeysObject *dk) {
     size_t index = (size_t)1 << dk->dk_log2_index_bytes;
     return (&indices[index]);
 }
+
 static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) {
     assert(dk->dk_kind == DICT_KEYS_GENERAL);
     return (PyDictKeyEntry*)_DK_ENTRIES(dk);
@@ -211,9 +216,6 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) {
 #define DICT_WATCHER_MASK ((1 << DICT_MAX_WATCHERS) - 1)
 #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1)
 
-#define DICT_VALUES_SIZE(values) ((uint8_t *)values)[-1]
-#define DICT_VALUES_USED_SIZE(values) ((uint8_t *)values)[-2]
-
 #ifdef Py_GIL_DISABLED
 #define DICT_NEXT_VERSION(INTERP) \
     (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT)
@@ -246,25 +248,63 @@ _PyDict_NotifyEvent(PyInterpreterState *interp,
     return DICT_NEXT_VERSION(interp) | (mp->ma_version_tag & DICT_WATCHER_AND_MODIFICATION_MASK);
 }
 
-extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values);
-PyAPI_FUNC(bool) _PyObject_MakeInstanceAttributesFromDict(PyObject *obj, PyDictOrValues *dorv);
+extern PyDictObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj);
+
 PyAPI_FUNC(PyObject *)_PyDict_FromItems(
         PyObject *const *keys, Py_ssize_t keys_offset,
         PyObject *const *values, Py_ssize_t values_offset,
         Py_ssize_t length);
 
+static inline uint8_t *
+get_insertion_order_array(PyDictValues *values)
+{
+    return (uint8_t *)&values->values[values->capacity];
+}
+
 static inline void
 _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix)
 {
     assert(ix < SHARED_KEYS_MAX_SIZE);
-    uint8_t *size_ptr = ((uint8_t *)values)-2;
-    int size = *size_ptr;
-    assert(size+2 < DICT_VALUES_SIZE(values));
-    size++;
-    size_ptr[-size] = (uint8_t)ix;
-    *size_ptr = size;
+    int size = values->size;
+    uint8_t *array = get_insertion_order_array(values);
+    assert(size < values->capacity);
+    assert(((uint8_t)ix) == ix);
+    array[size] = (uint8_t)ix;
+    values->size = size+1;
+}
+
+static inline size_t
+shared_keys_usable_size(PyDictKeysObject *keys)
+{
+#ifdef Py_GIL_DISABLED
+    // dk_usable will decrease for each instance that is created and each
+    // value that is added.  dk_nentries will increase for each value that
+    // is added.  We want to always return the right value or larger.
+    // We therefore increase dk_nentries first and we decrease dk_usable
+    // second, and conversely here we read dk_usable first and dk_entries
+    // second (to avoid the case where we read entries before the increment
+    // and read usable after the decrement)
+    return (size_t)(_Py_atomic_load_ssize_acquire(&keys->dk_usable) +
+                    _Py_atomic_load_ssize_acquire(&keys->dk_nentries));
+#else
+    return (size_t)keys->dk_nentries + (size_t)keys->dk_usable;
+#endif
 }
 
+static inline size_t
+_PyInlineValuesSize(PyTypeObject *tp)
+{
+    PyDictKeysObject *keys = ((PyHeapTypeObject*)tp)->ht_cached_keys;
+    assert(keys != NULL);
+    size_t size = shared_keys_usable_size(keys);
+    size_t prefix_size = _Py_SIZE_ROUND_UP(size, sizeof(PyObject *));
+    assert(prefix_size < 256);
+    return prefix_size + (size + 1) * sizeof(PyObject *);
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *dict, PyObject *obj);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_flowgraph.h

@@ -18,7 +18,7 @@ typedef struct {
 struct _PyCfgBuilder;
 
 int _PyCfgBuilder_UseLabel(struct _PyCfgBuilder *g, _PyCfgJumpTargetLabel lbl);
-int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _PyCompilerSrcLocation loc);
+int _PyCfgBuilder_Addop(struct _PyCfgBuilder *g, int opcode, int oparg, _Py_SourceLocation loc);
 
 struct _PyCfgBuilder* _PyCfgBuilder_New(void);
 void _PyCfgBuilder_Free(struct _PyCfgBuilder *g);