diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fc0f72efdae48e..2d9aefcda31edb 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -26,7 +26,7 @@ extern void _PyEval_FiniState(struct _ceval_state *ceval); PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); PyAPI_FUNC(int) _PyEval_AddPendingCall( PyInterpreterState *interp, - int (*func)(void *), + _Py_pending_call_func func, void *arg, int mainthreadonly); PyAPI_FUNC(void) _PyEval_SignalAsyncExc(PyInterpreterState *interp); diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index e56e43c6e0c6a7..2d54c6bd343076 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -13,6 +13,8 @@ extern "C" { #include "pycore_gil.h" // struct _gil_runtime_state +typedef int (*_Py_pending_call_func)(void *); + struct _pending_calls { int busy; PyThread_type_lock lock; @@ -24,7 +26,7 @@ struct _pending_calls { int async_exc; #define NPENDINGCALLS 32 struct _pending_call { - int (*func)(void *); + _Py_pending_call_func func; void *arg; } calls[NPENDINGCALLS]; int first; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1db23145a539cb..3eaf08e122daa2 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -68,6 +68,8 @@ struct _is { uint64_t next_unique_id; /* The linked list of threads, newest first. */ PyThreadState *head; + /* The thread currently executing in the __main__ module, if any. */ + PyThreadState *main; /* Used in Modules/_threadmodule.c. */ long count; /* Support for runtime thread stack size tuning. diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 5be0ff6764c693..9ae82372c7203c 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -40,6 +40,12 @@ _Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) interp == &interp->runtime->_main_interpreter); } +// Export for _xxsubinterpreters module. +PyAPI_FUNC(int) _PyInterpreterState_SetRunningMain(PyInterpreterState *); +PyAPI_FUNC(void) _PyInterpreterState_SetNotRunningMain(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_IsRunningMain(PyInterpreterState *); +PyAPI_FUNC(int) _PyInterpreterState_FailIfRunningMain(PyInterpreterState *); + static inline const PyConfig * _Py_GetMainConfig(void) @@ -148,6 +154,8 @@ extern PyStatus _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime); extern void _PySignal_AfterFork(void); #endif +PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *); + PyAPI_FUNC(int) _PyState_AddModule( PyThreadState *tstate, diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 5ed97e9715b2b0..de504b15945041 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -105,6 +105,11 @@ typedef struct pyruntimestate { unsigned long main_thread; + /* The value to use for sys.path[0] in new subinterpreters. + Normally this would be part of the PyConfig struct. However, + we cannot add it there in 3.12 since that's an ABI change. */ + wchar_t *sys_path_0; + /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as possible to facilitate out-of-process observability diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 27a143c7f5f38d..0e930e7958ab3c 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,5 +1,8 @@ import contextlib +import json import os +import os.path +import sys import threading from textwrap import dedent import unittest @@ -8,6 +11,7 @@ from test import support from test.support import import_helper from test.support import threading_helper +from test.support import os_helper _interpreters = import_helper.import_module('_xxsubinterpreters') _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters @@ -257,6 +261,16 @@ def test_subinterpreter(self): self.assertTrue(interp.is_running()) self.assertFalse(interp.is_running()) + def test_finished(self): + r, w = os.pipe() + interp = interpreters.create() + interp.run(f"""if True: + import os + os.write({w}, b'x') + """) + self.assertFalse(interp.is_running()) + self.assertEqual(os.read(r, 1), b'x') + def test_from_subinterpreter(self): interp = interpreters.create() out = _run_output(interp, dedent(f""" @@ -284,6 +298,31 @@ def test_bad_id(self): with self.assertRaises(ValueError): interp.is_running() + def test_with_only_background_threads(self): + r_interp, w_interp = os.pipe() + r_thread, w_thread = os.pipe() + + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + self.assertFalse(interp.is_running()) + + os.write(w_thread, DONE) + interp.run('t.join()') + self.assertEqual(os.read(r_interp, 1), FINISHED) + class TestInterpreterClose(TestBase): @@ -385,6 +424,37 @@ def test_still_running(self): interp.close() self.assertTrue(interp.is_running()) + def test_subthreads_still_running(self): + r_interp, w_interp = os.pipe() + r_thread, w_thread = os.pipe() + + FINISHED = b'F' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + class TestInterpreterRun(TestBase): @@ -461,6 +531,37 @@ def test_bytes_for_script(self): with self.assertRaises(TypeError): interp.run(b'print("spam")') + def test_with_background_threads_still_running(self): + r_interp, w_interp = os.pipe() + r_thread, w_thread = os.pipe() + + RAN = b'R' + DONE = b'D' + FINISHED = b'F' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + + def task(): + v = os.read({r_thread}, 1) + assert v == {DONE!r} + os.write({w_interp}, {FINISHED!r}) + t = threading.Thread(target=task) + t.start() + os.write({w_interp}, {RAN!r}) + """) + interp.run(f"""if True: + os.write({w_interp}, {RAN!r}) + """) + + os.write(w_thread, DONE) + interp.run('t.join()') + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), RAN) + self.assertEqual(os.read(r_interp, 1), FINISHED) + # test_xxsubinterpreters covers the remaining Interpreter.run() behavior. @@ -488,6 +589,154 @@ def task(): pass +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.run(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + class TestIsShareable(TestBase): def test_default_shareables(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index f63e5c6184ef0b..ce477d2df7d7d4 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -26,6 +26,11 @@ from test import lock_tests from test import support +try: + from test.support import interpreters +except ModuleNotFoundError: + interpreters = None + threading_helper.requires_working_threading(module=True) # Between fork() and exec(), only async-safe functions are allowed (issues @@ -45,6 +50,12 @@ def skip_unless_reliable_fork(test): return test +def requires_subinterpreters(meth): + """Decorator to skip a test if subinterpreters are not supported.""" + return unittest.skipIf(interpreters is None, + 'subinterpreters required')(meth) + + def restore_default_excepthook(testcase): testcase.addCleanup(setattr, threading, 'excepthook', threading.excepthook) threading.excepthook = threading.__excepthook__ @@ -1296,6 +1307,44 @@ def f(): # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") + @requires_subinterpreters + def test_threads_join_with_no_main(self): + r_interp, w_interp = self.pipe() + + INTERP = b'I' + FINI = b'F' + DONE = b'D' + + interp = interpreters.create() + interp.run(f"""if True: + import os + import threading + import time + + done = False + + def notify_fini(): + global done + done = True + os.write({w_interp}, {FINI!r}) + t.join() + threading._register_atexit(notify_fini) + + def task(): + while not done: + time.sleep(0.1) + os.write({w_interp}, {DONE!r}) + t = threading.Thread(target=task) + t.start() + + os.write({w_interp}, {INTERP!r}) + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), INTERP) + self.assertEqual(os.read(r_interp, 1), FINI) + self.assertEqual(os.read(r_interp, 1), DONE) + @cpython_only def test_daemon_threads_fatal_error(self): subinterp_code = f"""if 1: diff --git a/Lib/threading.py b/Lib/threading.py index a746dee5708124..c881f1107014c2 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -37,6 +37,7 @@ _allocate_lock = _thread.allocate_lock _set_sentinel = _thread._set_sentinel get_ident = _thread.get_ident +_is_main_interpreter = _thread._is_main_interpreter try: get_native_id = _thread.get_native_id _HAVE_THREAD_NATIVE_ID = True @@ -1566,7 +1567,7 @@ def _shutdown(): # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. - if _main_thread._is_stopped: + if _main_thread._is_stopped and _is_main_interpreter(): # _shutdown() was already called return @@ -1584,8 +1585,11 @@ def _shutdown(): # The main thread isn't finished yet, so its thread state lock can't # have been released. assert tlock is not None - assert tlock.locked() - tlock.release() + if tlock.locked(): + # It should have been released already by + # _PyInterpreterState_SetNotRunningMain(), but there may be + # embedders that aren't calling that yet. + tlock.release() _main_thread._stop() else: # bpo-1596321: _shutdown() must be called in the main thread. @@ -1619,6 +1623,7 @@ def main_thread(): In normal conditions, the main thread is the thread from which the Python interpreter was started. """ + # XXX Figure this out for subinterpreters. (See gh-75698.) return _main_thread # get thread-local implementation, either from the thread diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst new file mode 100644 index 00000000000000..b35550fa650dcc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-26-14-00-25.gh-issue-105716.SUJkW1.rst @@ -0,0 +1,3 @@ +Subinterpreters now correctly handle the case where they have threads +running in the background. Before, such threads would interfere with +cleaning up and destroying them, as well as prevent running another script. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst new file mode 100644 index 00000000000000..45de3ba8877b01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst @@ -0,0 +1 @@ +``sys.path[0]`` is now set correctly for subinterpreters. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 5edb6e9875d1ab..568fe8375d1eb4 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1604,6 +1604,18 @@ PyDoc_STRVAR(excepthook_doc, \n\ Handle uncaught Thread.run() exception."); +static PyObject * +thread__is_main_interpreter(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyBool_FromLong(_Py_IsMainInterpreter(interp)); +} + +PyDoc_STRVAR(thread__is_main_interpreter_doc, +"_is_main_interpreter()\n\ +\n\ +Return True if the current interpreter is the main Python interpreter."); + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_doc}, @@ -1633,8 +1645,10 @@ static PyMethodDef thread_methods[] = { METH_VARARGS, stack_size_doc}, {"_set_sentinel", thread__set_sentinel, METH_NOARGS, _set_sentinel_doc}, - {"_excepthook", thread_excepthook, + {"_excepthook", thread_excepthook, METH_O, excepthook_doc}, + {"_is_main_interpreter", thread__is_main_interpreter, + METH_NOARGS, thread__is_main_interpreter_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 1d7e7f1d71af3e..6bee11c1218518 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -2,8 +2,13 @@ /* interpreters module */ /* low-level access to interpreter primitives */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" #include "interpreteridobject.h" +#include "pycore_pystate.h" // _PyCrossInterpreterData_ReleaseAndRawFree() /* @@ -161,14 +166,24 @@ add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) return cls; } +#define XID_IGNORE_EXC 1 +#define XID_FREE 2 + static int -_release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) +_release_xid_data(_PyCrossInterpreterData *data, int flags) { + int ignoreexc = flags & XID_IGNORE_EXC; PyObject *exc; if (ignoreexc) { exc = PyErr_GetRaisedException(); } - int res = _PyCrossInterpreterData_Release(data); + int res; + if (flags & XID_FREE) { + res = _PyCrossInterpreterData_ReleaseAndRawFree(data); + } + else { + res = _PyCrossInterpreterData_Release(data); + } if (res < 0) { /* The owning interpreter is already destroyed. */ if (ignoreexc) { @@ -176,6 +191,9 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) PyErr_Clear(); } } + if (flags & XID_FREE) { + /* Either way, we free the data. */ + } if (ignoreexc) { PyErr_SetRaisedException(exc); } @@ -367,9 +385,8 @@ static void _channelitem_clear(_channelitem *item) { if (item->data != NULL) { - (void)_release_xid_data(item->data, 1); // It was allocated in _channel_send(). - GLOBAL_FREE(item->data); + (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); item->data = NULL; } item->next = NULL; @@ -1440,14 +1457,12 @@ _channel_recv(_channels *channels, int64_t id, PyObject **res) PyObject *obj = _PyCrossInterpreterData_NewObject(data); if (obj == NULL) { assert(PyErr_Occurred()); - (void)_release_xid_data(data, 1); - // It was allocated in _channel_send(). - GLOBAL_FREE(data); + // It was allocated in _channel_send(), so we free it. + (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE); return -1; } - int release_res = _release_xid_data(data, 0); - // It was allocated in _channel_send(). - GLOBAL_FREE(data); + // It was allocated in _channel_send(), so we free it. + int release_res = _release_xid_data(data, XID_FREE); if (release_res < 0) { // The source interpreter has been destroyed already. assert(PyErr_Occurred()); diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 4801f37d6f6c5f..bb7a76912330cb 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -2,7 +2,14 @@ /* interpreters module */ /* low-level access to interpreter primitives */ +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "pycore_initconfig.h" // _PyErr_SetFromPyStatus() +#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain() #include "interpreteridobject.h" @@ -53,24 +60,17 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) static int -_release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) +_release_xid_data(_PyCrossInterpreterData *data) { - PyObject *exc; - if (ignoreexc) { - exc = PyErr_GetRaisedException(); - } + PyObject *exc = PyErr_GetRaisedException(); int res = _PyCrossInterpreterData_Release(data); if (res < 0) { /* The owning interpreter is already destroyed. */ _PyCrossInterpreterData_Clear(NULL, data); - if (ignoreexc) { - // XXX Emit a warning? - PyErr_Clear(); - } - } - if (ignoreexc) { - PyErr_SetRaisedException(exc); + // XXX Emit a warning? + PyErr_Clear(); } + PyErr_SetRaisedException(exc); return res; } @@ -140,7 +140,7 @@ _sharednsitem_clear(struct _sharednsitem *item) PyMem_RawFree((void *)item->name); item->name = NULL; } - (void)_release_xid_data(&item->data, 1); + (void)_release_xid_data(&item->data); } static int @@ -169,16 +169,16 @@ typedef struct _sharedns { static _sharedns * _sharedns_new(Py_ssize_t len) { - _sharedns *shared = PyMem_NEW(_sharedns, 1); + _sharedns *shared = PyMem_RawCalloc(sizeof(_sharedns), 1); if (shared == NULL) { PyErr_NoMemory(); return NULL; } shared->len = len; - shared->items = PyMem_NEW(struct _sharednsitem, len); + shared->items = PyMem_RawCalloc(sizeof(struct _sharednsitem), len); if (shared->items == NULL) { PyErr_NoMemory(); - PyMem_Free(shared); + PyMem_RawFree(shared); return NULL; } return shared; @@ -190,8 +190,8 @@ _sharedns_free(_sharedns *shared) for (Py_ssize_t i=0; i < shared->len; i++) { _sharednsitem_clear(&shared->items[i]); } - PyMem_Free(shared->items); - PyMem_Free(shared); + PyMem_RawFree(shared->items); + PyMem_RawFree(shared); } static _sharedns * @@ -243,6 +243,11 @@ _sharedns_apply(_sharedns *shared, PyObject *ns) // of the exception in the calling interpreter. typedef struct _sharedexception { + PyInterpreterState *interp; +#define ERR_NOT_SET 0 +#define ERR_NO_MEMORY 1 +#define ERR_ALREADY_RUNNING 2 + int code; const char *name; const char *msg; } _sharedexception; @@ -264,14 +269,26 @@ _sharedexception_clear(_sharedexception *exc) } static const char * -_sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) +_sharedexception_bind(PyObject *exc, int code, _sharedexception *sharedexc) { + if (sharedexc->interp == NULL) { + sharedexc->interp = PyInterpreterState_Get(); + } + + if (code != ERR_NOT_SET) { + assert(exc == NULL); + assert(code > 0); + sharedexc->code = code; + return NULL; + } + assert(exc != NULL); const char *failure = NULL; PyObject *nameobj = PyUnicode_FromFormat("%S", Py_TYPE(exc)); if (nameobj == NULL) { failure = "unable to format exception type name"; + code = ERR_NO_MEMORY; goto error; } sharedexc->name = _copy_raw_string(nameobj); @@ -282,6 +299,7 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) } else { failure = "unable to encode and copy exception type name"; } + code = ERR_NO_MEMORY; goto error; } @@ -289,6 +307,7 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) PyObject *msgobj = PyUnicode_FromFormat("%S", exc); if (msgobj == NULL) { failure = "unable to format exception message"; + code = ERR_NO_MEMORY; goto error; } sharedexc->msg = _copy_raw_string(msgobj); @@ -299,6 +318,7 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) } else { failure = "unable to encode and copy exception message"; } + code = ERR_NO_MEMORY; goto error; } } @@ -309,7 +329,10 @@ _sharedexception_bind(PyObject *exc, _sharedexception *sharedexc) assert(failure != NULL); PyErr_Clear(); _sharedexception_clear(sharedexc); - *sharedexc = no_exception; + *sharedexc = (_sharedexception){ + .interp = sharedexc->interp, + .code = code, + }; return failure; } @@ -317,6 +340,7 @@ static void _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) { if (exc->name != NULL) { + assert(exc->code == ERR_NOT_SET); if (exc->msg != NULL) { PyErr_Format(wrapperclass, "%s: %s", exc->name, exc->msg); } @@ -325,9 +349,19 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) } } else if (exc->msg != NULL) { + assert(exc->code == ERR_NOT_SET); PyErr_SetString(wrapperclass, exc->msg); } + else if (exc->code == ERR_NO_MEMORY) { + PyErr_NoMemory(); + } + else if (exc->code == ERR_ALREADY_RUNNING) { + assert(exc->interp != NULL); + assert(_PyInterpreterState_IsRunningMain(exc->interp)); + _PyInterpreterState_FailIfRunningMain(exc->interp); + } else { + assert(exc->code == ERR_NOT_SET); PyErr_SetNone(wrapperclass); } } @@ -360,41 +394,21 @@ exceptions_init(PyObject *mod) } static int -_is_running(PyInterpreterState *interp) +_run_script(PyInterpreterState *interp, const char *codestr, + _sharedns *shared, _sharedexception *sharedexc) { - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - if (PyThreadState_Next(tstate) != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "interpreter has more than one thread"); - return -1; - } + int errcode = ERR_NOT_SET; - assert(!PyErr_Occurred()); - struct _PyInterpreterFrame *frame = tstate->cframe->current_frame; - if (frame == NULL) { - return 0; - } - return 1; -} - -static int -_ensure_not_running(PyInterpreterState *interp) -{ - int is_running = _is_running(interp); - if (is_running < 0) { - return -1; - } - if (is_running) { - PyErr_Format(PyExc_RuntimeError, "interpreter already running"); - return -1; + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + assert(PyErr_Occurred()); + // In the case where we didn't switch interpreters, it would + // be more efficient to leave the exception in place and return + // immediately. However, life is simpler if we don't. + PyErr_Clear(); + errcode = ERR_ALREADY_RUNNING; + goto error; } - return 0; -} -static int -_run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception *sharedexc) -{ PyObject *excval = NULL; PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { @@ -424,20 +438,23 @@ _run_script(PyInterpreterState *interp, const char *codestr, else { Py_DECREF(result); // We throw away the result. } + _PyInterpreterState_SetNotRunningMain(interp); *sharedexc = no_exception; return 0; error: excval = PyErr_GetRaisedException(); - const char *failure = _sharedexception_bind(excval, sharedexc); + const char *failure = _sharedexception_bind(excval, errcode, sharedexc); if (failure != NULL) { fprintf(stderr, "RunFailedError: script raised an uncaught exception (%s)", failure); - PyErr_Clear(); } Py_XDECREF(excval); + if (errcode != ERR_ALREADY_RUNNING) { + _PyInterpreterState_SetNotRunningMain(interp); + } assert(!PyErr_Occurred()); return -1; } @@ -446,10 +463,8 @@ static int _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, const char *codestr, PyObject *shareables) { - if (_ensure_not_running(interp) < 0) { - return -1; - } module_state *state = get_module_state(mod); + assert(state != NULL); _sharedns *shared = _get_shared_ns(shareables); if (shared == NULL && PyErr_Occurred()) { @@ -458,30 +473,29 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, // Switch to interpreter. PyThreadState *save_tstate = NULL; + PyThreadState *tstate = NULL; if (interp != PyInterpreterState_Get()) { - // XXX Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + tstate = PyThreadState_New(interp); // XXX Possible GILState issues? save_tstate = PyThreadState_Swap(tstate); } // Run the script. - _sharedexception exc = {NULL, NULL}; + _sharedexception exc = (_sharedexception){ .interp = interp }; int result = _run_script(interp, codestr, shared, &exc); // Switch back. if (save_tstate != NULL) { + PyThreadState_Clear(tstate); PyThreadState_Swap(save_tstate); + PyThreadState_Delete(tstate); } // Propagate any exception out to the caller. - if (exc.name != NULL) { - assert(state != NULL); + if (result < 0) { + assert(!PyErr_Occurred()); _sharedexception_apply(&exc, state->RunFailedError); - } - else if (result != 0) { - // We were unable to allocate a shared exception. - PyErr_NoMemory(); + assert(PyErr_Occurred()); } if (shared != NULL) { @@ -511,6 +525,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) const PyInterpreterConfig config = isolated ? (PyInterpreterConfig)_PyInterpreterConfig_INIT : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + // XXX Possible GILState issues? PyThreadState *tstate = NULL; PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); @@ -526,6 +541,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } assert(tstate != NULL); + PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate); PyObject *idobj = _PyInterpreterState_GetIDObject(interp); if (idobj == NULL) { @@ -535,6 +551,10 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) PyThreadState_Swap(save_tstate); return NULL; } + + PyThreadState_Clear(tstate); + PyThreadState_Delete(tstate); + _PyInterpreterState_RequireIDRef(interp, 1); return idobj; } @@ -576,12 +596,13 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) // Ensure the interpreter isn't running. /* XXX We *could* support destroying a running interpreter but aren't going to worry about it for now. */ - if (_ensure_not_running(interp) < 0) { + if (_PyInterpreterState_IsRunningMain(interp)) { + PyErr_Format(PyExc_RuntimeError, "interpreter running"); return NULL; } // Destroy the interpreter. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *tstate = PyThreadState_New(interp); // XXX Possible GILState issues? PyThreadState *save_tstate = PyThreadState_Swap(tstate); Py_EndInterpreter(tstate); @@ -750,11 +771,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds) if (interp == NULL) { return NULL; } - int is_running = _is_running(interp); - if (is_running < 0) { - return NULL; - } - if (is_running) { + if (_PyInterpreterState_IsRunningMain(interp)) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -765,6 +782,7 @@ PyDoc_STRVAR(is_running_doc, \n\ Return whether or not the identified interpreter is running."); + static PyMethodDef module_functions[] = { {"create", _PyCFunction_CAST(interp_create), METH_VARARGS | METH_KEYWORDS, create_doc}, diff --git a/Modules/main.c b/Modules/main.c index 7edfeb3365b4c6..07eaa5befe0d68 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -559,6 +559,11 @@ pymain_run_python(int *exitcode) goto error; } + // XXX Calculate runtime->sys_path_0 in getpath.py. + // The tricky part is that we can't check the path importers yet + // at that point. + assert(interp->runtime->sys_path_0 == NULL); + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be @@ -574,29 +579,48 @@ pymain_run_python(int *exitcode) // import readline and rlcompleter before script dir is added to sys.path pymain_import_readline(config); + PyObject *path0 = NULL; if (main_importer_path != NULL) { - if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - goto error; - } + path0 = Py_NewRef(main_importer_path); } else if (!config->safe_path) { - PyObject *path0 = NULL; int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0); if (res < 0) { goto error; } - - if (res > 0) { - if (pymain_sys_path_add_path0(interp, path0) < 0) { - Py_DECREF(path0); - goto error; - } + else if (res == 0) { + Py_CLEAR(path0); + } + } + // XXX Apply runtime->sys_path_0 in init_interp_main(). We have + // to be sure to get readline/rlcompleter imported at the correct time. + if (path0 != NULL) { + wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL); + if (wstr == NULL) { Py_DECREF(path0); + goto error; + } + PyMemAllocatorEx old_alloc; + _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + interp->runtime->sys_path_0 = _PyMem_RawWcsdup(wstr); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + PyMem_Free(wstr); + if (interp->runtime->sys_path_0 == NULL) { + Py_DECREF(path0); + goto error; + } + int res = pymain_sys_path_add_path0(interp, path0); + Py_DECREF(path0); + if (res < 0) { + goto error; } } pymain_header(config); + _PyInterpreterState_SetRunningMain(interp); + assert(!PyErr_Occurred()); + if (config->run_command) { *exitcode = pymain_run_command(config->run_command); } @@ -620,6 +644,7 @@ pymain_run_python(int *exitcode) *exitcode = pymain_exit_err_print(); done: + _PyInterpreterState_SetNotRunningMain(interp); Py_XDECREF(main_importer_path); } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index c1ab5883568e7d..18ab4ccc176284 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -757,7 +757,7 @@ _PyEval_SignalReceived(PyInterpreterState *interp) /* Push one item onto the queue while holding the lock. */ static int _push_pending_call(struct _pending_calls *pending, - int (*func)(void *), void *arg) + _Py_pending_call_func func, void *arg) { int i = pending->last; int j = (i + 1) % NPENDINGCALLS; @@ -804,7 +804,7 @@ _pop_pending_call(struct _pending_calls *pending, int _PyEval_AddPendingCall(PyInterpreterState *interp, - int (*func)(void *), void *arg, + _Py_pending_call_func func, void *arg, int mainthreadonly) { assert(!mainthreadonly || _Py_IsMainInterpreter(interp)); @@ -828,7 +828,7 @@ _PyEval_AddPendingCall(PyInterpreterState *interp, } int -Py_AddPendingCall(int (*func)(void *), void *arg) +Py_AddPendingCall(_Py_pending_call_func func, void *arg) { /* Legacy users of this API will continue to target the main thread (of the main interpreter). */ @@ -872,7 +872,7 @@ _make_pending_calls(struct _pending_calls *pending) { /* perform a bounded number of calls, in case of recursion */ for (int i=0; iruntime->sys_path_0; + if (sys_path_0 != NULL) { + PyObject *path0 = PyUnicode_FromWideChar(sys_path_0, -1); + if (path0 == NULL) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sysdict = interp->sysdict; + if (sysdict == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); + if (sys_path == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + int res = PyList_Insert(sys_path, 0, path0); + Py_DECREF(path0); + if (res) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + } + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK(); diff --git a/Python/pystate.c b/Python/pystate.c index 6f60c3dccce222..d512f2aafc8f61 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -524,6 +524,10 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) } #undef FREE_LOCK + if (runtime->sys_path_0 != NULL) { + PyMem_RawFree(runtime->sys_path_0); + runtime->sys_path_0 = NULL; + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -1045,6 +1049,60 @@ _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime) #endif +int +_PyInterpreterState_SetRunningMain(PyInterpreterState *interp) +{ + if (_PyInterpreterState_FailIfRunningMain(interp) < 0) { + return -1; + } + PyThreadState *tstate = current_fast_get(&_PyRuntime); + _Py_EnsureTstateNotNULL(tstate); + if (tstate->interp != interp) { + PyErr_SetString(PyExc_RuntimeError, + "current tstate has wrong interpreter"); + return -1; + } + interp->threads.main = tstate; + return 0; +} + +void +_PyInterpreterState_SetNotRunningMain(PyInterpreterState *interp) +{ + PyThreadState *tstate = interp->threads.main; + assert(tstate == current_fast_get(&_PyRuntime)); + + if (tstate->on_delete != NULL) { + // The threading module was imported for the first time in this + // thread, so it was set as threading._main_thread. (See gh-75698.) + // The thread has finished running the Python program so we mark + // the thread object as finished. + tstate->on_delete(tstate->on_delete_data); + tstate->on_delete = NULL; + tstate->on_delete_data = NULL; + } + + interp->threads.main = NULL; +} + +int +_PyInterpreterState_IsRunningMain(PyInterpreterState *interp) +{ + return (interp->threads.main != NULL); +} + +int +_PyInterpreterState_FailIfRunningMain(PyInterpreterState *interp) +{ + if (interp->threads.main != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter already running"); + return -1; + } + return 0; +} + + //---------- // accessors //---------- @@ -1104,8 +1162,9 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp) PyThread_release_lock(interp->id_mutex); if (refcount == 0 && interp->requires_idref) { - // XXX Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *tstate = _PyThreadState_New(interp); + _PyThreadState_Bind(tstate); + // XXX Possible GILState issues? PyThreadState *save_tstate = _PyThreadState_Swap(runtime, tstate); Py_EndInterpreter(tstate); @@ -1259,7 +1318,14 @@ free_threadstate(PyThreadState *tstate) { // The initial thread state of the interpreter is allocated // as part of the interpreter state so should not be freed. - if (tstate != &tstate->interp->_initial_thread) { + if (tstate == &tstate->interp->_initial_thread) { + // Restore to _PyThreadState_INIT. + tstate = &tstate->interp->_initial_thread; + memcpy(tstate, + &initial._main_interpreter._initial_thread, + sizeof(*tstate)); + } + else { PyMem_RawFree(tstate); } } @@ -1316,8 +1382,6 @@ add_threadstate(PyInterpreterState *interp, PyThreadState *tstate, PyThreadState *next) { assert(interp->threads.head != tstate); - assert((next != NULL && tstate->id != 1) || - (next == NULL && tstate->id == 1)); if (next != NULL) { assert(next->prev == NULL || next->prev == tstate); next->prev = tstate; @@ -1351,10 +1415,10 @@ new_threadstate(PyInterpreterState *interp) PyThreadState *old_head = interp->threads.head; if (old_head == NULL) { // It's the interpreter's initial thread state. - assert(id == 1); used_newtstate = 0; tstate = &interp->_initial_thread; } + // XXX Re-use interp->_initial_thread if not in use? else { // Every valid interpreter must have at least one thread. assert(id > 1); @@ -1501,6 +1565,12 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->context); if (tstate->on_delete != NULL) { + // For the "main" thread of each interpreter, this is meant + // to be done in _PyInterpreterState_SetNotRunningMain(). + // That leaves threads created by the threading module, + // and any threads killed by forking. + // However, we also accommodate "main" threads that still + // don't call _PyInterpreterState_SetNotRunningMain() yet. tstate->on_delete(tstate->on_delete_data); } @@ -2161,6 +2231,7 @@ PyGILState_Ensure(void) int has_gil; if (tcur == NULL) { /* Create a new Python thread state for this thread */ + // XXX Use PyInterpreterState_EnsureThreadState()? tcur = new_threadstate(runtime->gilstate.autoInterpreterState); if (tcur == NULL) { Py_FatalError("Couldn't create thread-state for new thread"); @@ -2259,10 +2330,16 @@ _xidata_init(_PyCrossInterpreterData *data) static inline void _xidata_clear(_PyCrossInterpreterData *data) { - if (data->free != NULL) { - data->free(data->data); + // _PyCrossInterpreterData only has two members that need to be + // cleaned up, if set: "data" must be freed and "obj" must be decref'ed. + // In both cases the original (owning) interpreter must be used, + // which is the caller's responsibility to ensure. + if (data->data != NULL) { + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; } - data->data = NULL; Py_CLEAR(data->obj); } @@ -2407,40 +2484,32 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) return data->new_object(data); } -typedef void (*releasefunc)(PyInterpreterState *, void *); - -static void -_call_in_interpreter(PyInterpreterState *interp, releasefunc func, void *arg) +static int +_release_xidata_pending(void *data) { - /* We would use Py_AddPendingCall() if it weren't specific to the - * main interpreter (see bpo-33608). In the meantime we take a - * naive approach. - */ - _PyRuntimeState *runtime = interp->runtime; - PyThreadState *save_tstate = NULL; - if (interp != current_fast_get(runtime)->interp) { - // XXX Using the "head" thread isn't strictly correct. - PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); - // XXX Possible GILState issues? - save_tstate = _PyThreadState_Swap(runtime, tstate); - } - - // XXX Once the GIL is per-interpreter, this should be called with the - // calling interpreter's GIL released and the target interpreter's held. - func(interp, arg); + _xidata_clear((_PyCrossInterpreterData *)data); + return 0; +} - // Switch back. - if (save_tstate != NULL) { - _PyThreadState_Swap(runtime, save_tstate); - } +static int +_xidata_release_and_rawfree_pending(void *data) +{ + _xidata_clear((_PyCrossInterpreterData *)data); + PyMem_RawFree(data); + return 0; } -int -_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +static int +_xidata_release(_PyCrossInterpreterData *data, int rawfree) { - if (data->free == NULL && data->obj == NULL) { + if ((data->data == NULL || data->free == NULL) && data->obj == NULL) { // Nothing to release! - data->data = NULL; + if (rawfree) { + PyMem_RawFree(data); + } + else { + data->data = NULL; + } return 0; } @@ -2451,15 +2520,42 @@ _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) // This function shouldn't have been called. // XXX Someone leaked some memory... assert(PyErr_Occurred()); + if (rawfree) { + PyMem_RawFree(data); + } return -1; } // "Release" the data and/or the object. - _call_in_interpreter(interp, - (releasefunc)_PyCrossInterpreterData_Clear, data); + if (interp == current_fast_get(interp->runtime)->interp) { + _xidata_clear(data); + if (rawfree) { + PyMem_RawFree(data); + } + } + else { + _Py_pending_call_func func = _release_xidata_pending; + if (rawfree) { + func = _xidata_release_and_rawfree_pending; + } + // XXX Emit a warning if this fails? + _PyEval_AddPendingCall(interp, func, data, 0); + } return 0; } +int +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 0); +} + +int +_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data) +{ + return _xidata_release(data, 1); +} + /* registry of {type -> crossinterpdatafunc} */ /* For now we use a global registry of shareable classes. An @@ -2728,6 +2824,10 @@ _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) } +/*************/ +/* Other API */ +/*************/ + _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) {