gh-80406: Finalise subinterpreters in Py_FinalizeEx()

Include/internal/pycore_pystate.h

@@ -226,6 +226,7 @@ typedef struct pyruntimestate {
            If that becomes a problem later then we can adjust, e.g. by
            using a Python int. */
         int64_t next_id;
+        int finalizing;
     } interpreters;
     // XXX Remove this field once we have a tp_* slot.
     struct _xidregistry {

Misc/NEWS.d/next/C API/2019-12-14-15-20-51.bpo-36225.z7Nnrq.rst

@@ -0,0 +1 @@
+:func:`Py_FinalizeEx()` now implicitly cleans up subinterpreters, as the C API documentation suggests.

Programs/_testembed.c

@@ -85,6 +85,47 @@ static int test_repeated_init_and_subinterpreters(void)
     return 0;
 }
 
+/* bpo-36225: Implicitly tear down subinterpreters with Py_Finalize() */
+static int test_finalize_subinterps(void)
+{
+    PyThreadState *mainstate;
+    PyGILState_STATE gilstate;
+    int i, j;
+
+    for (i=0; i<15; i++) {
+        printf("--- Pass %d ---\n", i);
+        _testembed_Py_Initialize();
+        mainstate = PyThreadState_Get();
+
+        PyEval_ReleaseThread(mainstate);
+
+        gilstate = PyGILState_Ensure();
+        print_subinterp();
+        PyThreadState_Swap(NULL);
+
+        for (j=0; j<2; j++) {
+            Py_NewInterpreter();
+            print_subinterp();
+        }
+
+        PyThreadState_Swap(mainstate);
+        print_subinterp();
+
+        for (j=0; j<2; j++) {
+            Py_NewInterpreter();
+            print_subinterp();
+        }
+
+        PyThreadState_Swap(mainstate);
+        print_subinterp();
+        PyGILState_Release(gilstate);
+
+        PyEval_RestoreThread(mainstate);
+        Py_Finalize();
+    }
+    return 0;
+}
+
 /*****************************************************
  * Test forcing a particular IO encoding
  *****************************************************/
@@ -1181,10 +1222,14 @@ static int test_audit_subinterpreter(void)
     PySys_AddAuditHook(_audit_subinterpreter_hook, NULL);
     _testembed_Py_Initialize();
 
+    PyThreadState *mainstate = PyThreadState_Get();
+
     Py_NewInterpreter();
     Py_NewInterpreter();
     Py_NewInterpreter();
 
+    // Currently unable to call Py_Finalize from subinterpreter thread, see bpo-37776.
+    PyThreadState_Swap(mainstate);
     Py_Finalize();
 
     switch (_audit_subinterpreter_interpreter_count) {
@@ -1603,6 +1648,7 @@ struct TestCase
 static struct TestCase TestCases[] = {
     {"test_forced_io_encoding", test_forced_io_encoding},
     {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
+    {"test_finalize_subinterps", test_finalize_subinterps},
     {"test_pre_initialization_api", test_pre_initialization_api},
     {"test_pre_initialization_sys_options", test_pre_initialization_sys_options},
     {"test_bpo20891", test_bpo20891},

Python/pylifecycle.c

@@ -1257,6 +1257,20 @@ Py_FinalizeEx(void)
     PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
     PyInterpreterState *interp = tstate->interp;
 
+    // Finalize sub-interpreters.
+    runtime->interpreters.finalizing = 1;
+    PyInterpreterState *subinterp = PyInterpreterState_Head();
+    PyInterpreterState *next_interp;
+    while (subinterp != NULL) {
+        next_interp = PyInterpreterState_Next(subinterp);
+        if (subinterp != PyInterpreterState_Main()) {
+            PyThreadState_Swap(subinterp->tstate_head);
+            Py_EndInterpreter(subinterp->tstate_head);
+        }
+        subinterp = next_interp;
+    }
+    PyThreadState_Swap(tstate);
+
     // Wrap up existing "threading"-module-created, non-daemon threads.
     wait_for_thread_shutdown(tstate);
 
@@ -1454,6 +1468,10 @@ new_interpreter(PyThreadState **tstate_p)
         return _PyStatus_ERR("Py_Initialize must be called first");
     }
 
+    if (runtime->interpreters.finalizing) {
+        return _PyStatus_ERR("Interpreters are being finalized");
+    }
+
     /* Issue #10915, #15751: The GIL API doesn't work with multiple
        interpreters: disable PyGILState_Check(). */
     _PyGILState_check_enabled = 0;