gh-117953: Other Cleanups in the Extensions Machinery

Lib/test/test_import/__init__.py

@@ -2285,6 +2285,107 @@ def test_disallowed_reimport(self):
 
 
 class TestSinglePhaseSnapshot(ModuleSnapshot):
+    """A representation of a single-phase init module for testing.
+
+    Fields from ModuleSnapshot:
+
+    * id - id(mod)
+    * module - mod or a SimpleNamespace with __file__ & __spec__
+    * ns - a shallow copy of mod.__dict__
+    * ns_id - id(mod.__dict__)
+    * cached - sys.modules[name] (or None if not there or not snapshotable)
+    * cached_id - id(sys.modules[name]) (or None if not there)
+
+    Extra fields:
+
+    * summed - the result of calling "mod.sum(1, 2)"
+    * lookedup - the result of calling "mod.look_up_self()"
+    * lookedup_id - the object ID of self.lookedup
+    * state_initialized - the result of calling "mod.state_initialized()"
+    * init_count - (optional) the result of calling "mod.initialized_count()"
+
+    Overridden methods from ModuleSnapshot:
+
+    * from_module()
+    * parse()
+
+    Other methods from ModuleSnapshot:
+
+    * build_script()
+    * from_subinterp()
+
+    ----
+
+    There are 5 modules in Modules/_testsinglephase.c:
+
+    * _testsinglephase
+       * has global state
+       * extra loads skip the init function, copy def.m_base.m_copy
+       * counts calls to init function
+    * _testsinglephase_basic_wrapper
+       * _testsinglephase by another name (and separate init function symbol)
+    * _testsinglephase_basic_copy
+       * same as _testsinglephase but with own def (and init func)
+    * _testsinglephase_with_reinit
+       * has no global or module state
+       * mod.state_initialized returns None
+       * an extra load in the main interpreter calls the cached init func
+       * an extra load in legacy subinterpreters does a full load
+    * _testsinglephase_with_state
+       * has module state
+       * an extra load in the main interpreter calls the cached init func
+       * an extra load in legacy subinterpreters does a full load
+
+    (See Modules/_testsinglephase.c for more info.)
+
+    For all those modules, the snapshot after the initial load (not in
+    the global extensions cache) would look like the following:
+
+    * initial load
+       * id: ID of nww module object
+       * ns: exactly what the module init put there
+       * ns_id: ID of new module's __dict__
+       * cached_id: same as self.id
+       * summed: 3  (never changes)
+       * lookedup_id: same as self.id
+       * state_initialized: a timestamp between the time of the load
+         and the time of the snapshot
+       * init_count: 1  (None for _testsinglephase_with_reinit)
+
+    For the other scenarios it varies.
+
+    For the _testsinglephase, _testsinglephase_basic_wrapper, and
+    _testsinglephase_basic_copy modules, the snapshot should look
+    like the following:
+
+    * reloaded
+       * id: no change
+       * ns: matches what the module init function put there,
+         including the IDs of all contained objects,
+         plus any extra attributes added before the reload
+       * ns_id: no change
+       * cached_id: no change
+       * lookedup_id: no change
+       * state_initialized: no change
+       * init_count: no change
+    * already loaded
+       * (same as initial load except for ns and state_initialized)
+       * ns: matches the initial load, incl. IDs of contained objects
+       * state_initialized: no change from initial load
+
+    For _testsinglephase_with_reinit:
+
+    * reloaded: same as initial load (old module & ns is discarded)
+    * already loaded: same as initial load (old module & ns is discarded)
+
+    For _testsinglephase_with_state:
+
+    * reloaded
+       * (same as initial load (old module & ns is discarded),
+         except init_count)
+       * init_count: increase by 1
+    * already loaded: same as reloaded
+    """
 
     @classmethod
     def from_module(cls, mod):
@@ -2901,17 +3002,18 @@ def test_basic_multiple_interpreters_deleted_no_reset(self):
         #  * module's global state was initialized but cleared
 
         # Start with an interpreter that gets destroyed right away.
-        base = self.import_in_subinterp(postscript='''
-            # Attrs set after loading are not in m_copy.
-            mod.spam = 'spam, spam, mash, spam, eggs, and spam'
-        ''')
+        base = self.import_in_subinterp(
+            postscript='''
+                # Attrs set after loading are not in m_copy.
+                mod.spam = 'spam, spam, mash, spam, eggs, and spam'
+                ''')
         self.check_common(base)
         self.check_fresh(base)
 
         # At this point:
         #  * alive in 0 interpreters
         #  * module def in _PyRuntime.imports.extensions
-        #  * mod init func ran again
+        #  * mod init func ran for the first time (since reset)
         #  * m_copy is NULL (claered when the interpreter was destroyed)
         #  * module's global state was initialized, not reset
 
@@ -2923,7 +3025,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self):
         # At this point:
         #  * alive in 1 interpreter (interp1)
         #  * module def still in _PyRuntime.imports.extensions
-        #  * mod init func ran again
+        #  * mod init func ran for the second time (since reset)
         #  * m_copy was copied from interp1 (was NULL)
         #  * module's global state was updated, not reset
 
@@ -2935,7 +3037,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self):
         # At this point:
         #  * alive in 2 interpreters (interp1, interp2)
         #  * module def still in _PyRuntime.imports.extensions
-        #  * mod init func ran again
+        #  * mod init func did not run again
         #  * m_copy was copied from interp2 (was from interp1)
         #  * module's global state was updated, not reset
 

Modules/_testsinglephase.c

@@ -1,6 +1,172 @@
 
 /* Testing module for single-phase initialization of extension modules
- */
+
+This file contains 5 distinct modules, meaning each as its own name
+and its own init function (PyInit_...).  The default import system will
+only find the one matching the filename: _testsinglephase.  To load the
+others you must do so manually.  For example:
+
+```python
+name = '_testsinglephase_base_wrapper'
+filename = _testsinglephase.__file__
+loader = importlib.machinery.ExtensionFileLoader(name, filename)
+spec = importlib.util.spec_from_file_location(name, filename, loader=loader)
+mod = importlib._bootstrap._load(spec)
+```
+
+Here are the 5 modules:
+
+* _testsinglephase
+   * def: _testsinglephase_basic,
+      * m_name: "_testsinglephase"
+      * m_size: -1
+   * state
+      * process-global
+         * <int> initialized_count  (default to -1; will never be 0)
+         * <module_state> module  (see module state below)
+      * module state: no
+      * initial __dict__: see common initial __dict__ below
+   * init function
+      1. create module
+      2. clear <global>.module
+      3. initialize <global>.module: see module state below
+      4. initialize module: set initial __dict__
+      5. increment <global>.initialized_count
+   * functions
+      * (3 common, see below)
+      * initialized_count() - return <global>.module.initialized_count
+   * import system
+      * caches
+         * global extensions cache: yes
+         * def.m_base.m_copy: yes
+         * def.m_base.m_init: no
+         * per-interpreter cache: yes  (all single-phase init modules)
+      * load in main interpreter
+         * initial  (not already in global cache)
+            1. get init function from shared object file
+            2. run init function
+            3. copy __dict__ into def.m_base.m_copy
+            4. set entry in global cache
+            5. set entry in per-interpreter cache
+            6. set entry in sys.modules
+         * reload  (already in sys.modules)
+            1. get def from global cache
+            2. get module from sys.modules
+            3. update module with contents of def.m_base.m_copy
+         * already loaded in other interpreter  (already in global cache)
+            * same as reload, but create new module and update *it*
+         * not in any sys.modules, still in global cache
+            * same as already loaded
+      * load in legacy (non-isolated) interpreter
+         * same as main interpreter
+      * unload: never  (all single-phase init modules)
+* _testsinglephase_basic_wrapper
+   * identical to _testsinglephase except module name
+* _testsinglephase_basic_copy
+   * def: static local variable in init function
+      * m_name: "_testsinglephase_basic_copy"
+      * m_size: -1
+   * state: same as _testsinglephase
+   * init function: same as _testsinglephase
+   * functions: same as _testsinglephase
+   * import system: same as _testsinglephase
+* _testsinglephase_with_reinit
+   * def: _testsinglephase_with_reinit,
+      * m_name: "_testsinglephase_with_reinit"
+      * m_size: 0
+   * state
+      * process-global state: no
+      * module state: no
+      * initial __dict__: see common initial __dict__ below
+   * init function
+      1. create module
+      2. initialize temporary module state (local var): see module state below
+      3. initialize module: set initial __dict__
+   * functions: see common functions below
+   * import system
+      * caches
+         * global extensions cache: only if loaded in main interpreter
+         * def.m_base.m_copy: no
+         * def.m_base.m_init: only if loaded in the main interpreter
+         * per-interpreter cache: yes  (all single-phase init modules)
+      * load in main interpreter
+         * initial  (not already in global cache)
+            * (same as _testsinglephase except step 3)
+            1. get init function from shared object file
+            2. run init function
+            3. set def.m_base.m_init to the init function
+            4. set entry in global cache
+            5. set entry in per-interpreter cache
+            6. set entry in sys.modules
+         * reload  (already in sys.modules)
+            1. get def from global cache
+            2. call def->m_base.m_init to get a new module object
+            3. replace the existing module in sys.modules
+         * already loaded in other interpreter  (already in global cache)
+            * same as reload (since will only be in cache for main interp)
+         * not in any sys.modules, still in global cache
+            * same as already loaded
+      * load in legacy (non-isolated) interpreter
+         * initial  (not already in global cache)
+            * (same as main interpreter except skip steps 3 & 4 there)
+            1. get init function from shared object file
+            2. run init function
+            ...
+            5. set entry in per-interpreter cache
+            6. set entry in sys.modules
+         * reload  (already in sys.modules)
+            * same as initial  (load from scratch)
+         * already loaded in other interpreter  (already in global cache)
+            * same as initial  (load from scratch)
+         * not in any sys.modules, still in global cache
+            * same as initial  (load from scratch)
+      * unload: never  (all single-phase init modules)
+* _testsinglephase_with_state
+   * def: _testsinglephase_with_state,
+      * m_name: "_testsinglephase_with_state"
+      * m_size: sizeof(module_state)
+   * state
+      * process-global: no
+      * module state: see module state below
+      * initial __dict__: see common initial __dict__ below
+   * init function
+      1. create module
+      3. initialize module state: see module state below
+      4. initialize module: set initial __dict__
+      5. increment <global>.initialized_count
+   * functions: see common functions below
+   * import system: same as _testsinglephase_basic_copy
+
+Module state:
+
+* fields
+   * <PyTime_t> initialized - when the module was first initialized
+   * <PyObject> *error
+   * <PyObject> *int_const
+   * <PyObject> *str_const
+* initialization
+   1. set state.initialized to the current time
+   2. set state.error to a new exception class
+   3. set state->int_const to int(1969)
+   4. set state->str_const to "something different"
+
+Common initial __dict__:
+
+* error: state.error
+* int_const: state.int_const
+* str_const: state.str_const
+* _module_initialized: state.initialized
+
+Common functions:
+
+* look_up_self() - return the module from the per-interpreter "by-index" cache
+* sum() - return a + b
+* state_initialized() - return state->initialized (or None if m_size == 0)
+
+See Python/import.c, especially the long comments, for more about
+single-phase init modules.
+*/
+
 #ifndef Py_BUILD_CORE_BUILTIN
 #  define Py_BUILD_CORE_MODULE 1
 #endif