From 30814e42b2331f3594050617e2cd9e9b6fd7918c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 28 Sep 2023 11:40:15 -0400 Subject: [PATCH 01/10] gh-88402: Add new sysconfig variables on Windows This adds `LIBRARY`, `LDLIBRARY`, `LIBPL`, `SOABI`, and `Py_NOGIL` variables to sysconfig on Windows. Note that `Py_NOGIL` is only defined in `--disable-gil` builds. --- Lib/sysconfig.py | 25 ++++++++++++++++--- Lib/test/test_sysconfig.py | 23 ++++++++++++++--- ...3-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst | 2 ++ 3 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index a8b5c5f7dfba5b..75ec2d859cad36 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -545,15 +545,32 @@ def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories import _imp + import re vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - try: + + extension_suffixes = _imp.extension_suffixes() + if len(extension_suffixes) >= 1: # GH-99201: _imp.extension_suffixes may be empty when # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass + # e.g., "_d.cp313-win_amd64.pyd" + vars['EXT_SUFFIX'] = extension_suffixes[0] + + # e.g., "cp313-win_amd64" + _, soabi, _ = vars['EXT_SUFFIX'].split('.') + vars['SOABI'] = soabi + + # e.g., check for a "t" (for "threading") in SOABI + if re.match(r'cp\d+t', soabi): + vars['Py_NOGIL'] = 1 + + vars['LIBPL'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + import _winapi + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index b6dbf3d52cb4c3..b49c210b036f88 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -17,6 +17,7 @@ get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, _expand_vars, _get_preferred_schemes, _main) +import _imp import _osx_support @@ -394,6 +395,24 @@ def test_ldshared_value(self): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + if sys.platform == 'win32': + self.assertTrue(library.startswith(f'python{major}{minor}')) + self.assertTrue(library.endswith('.dll')) + self.assertEqual(library, ldlibrary) + else: + self.assertTrue(library.startswith(f'libpython{major}.{minor}')) + self.assertTrue(library.endswith('.a')) + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -472,10 +491,8 @@ def test_srcdir_independent_of_cwd(self): @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) diff --git a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst new file mode 100644 index 00000000000000..369c91ec38e11e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst @@ -0,0 +1,2 @@ +Add new :mod:`sysconfig` variables on Windows: ``LIBRARY``, ``LDLIBRARY``, +``LIBPL``, ``SOABI``, and ``Py_NOGIL``. From 3f2a744ea2fd44c24fe9b099ed1d808a27d3d84c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 2 Oct 2023 14:52:14 +0000 Subject: [PATCH 02/10] Update 2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filipe Laíns --- .../next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst index 369c91ec38e11e..a5654775ce6e4a 100644 --- a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst +++ b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst @@ -1,2 +1,2 @@ -Add new :mod:`sysconfig` variables on Windows: ``LIBRARY``, ``LDLIBRARY``, +Add new variables to :py:meth:`sysconfig.get_config_vars` on Windows: ``LIBRARY``, ``LDLIBRARY``, ``LIBPL``, ``SOABI``, and ``Py_NOGIL``. From 0ce92b29b216f86294e8483875ace6e0e747b009 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 10:20:08 -0400 Subject: [PATCH 03/10] Compute some sysconfig variables in _winapi --- Lib/sysconfig.py | 22 +++----------- ...3-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst | 4 +-- Modules/_winapi.c | 29 +++++++++++++++++++ Modules/clinic/_winapi.c.h | 20 ++++++++++++- Python/dynload_win.c | 7 +++-- Python/importdl.h | 1 + 6 files changed, 60 insertions(+), 23 deletions(-) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 75ec2d859cad36..4b88d2d1af8036 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -544,30 +544,16 @@ def _init_posix(vars): def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories - import _imp - import re + import _winapi vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - extension_suffixes = _imp.extension_suffixes() - if len(extension_suffixes) >= 1: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - # e.g., "_d.cp313-win_amd64.pyd" - vars['EXT_SUFFIX'] = extension_suffixes[0] - - # e.g., "cp313-win_amd64" - _, soabi, _ = vars['EXT_SUFFIX'].split('.') - vars['SOABI'] = soabi - - # e.g., check for a "t" (for "threading") in SOABI - if re.match(r'cp\d+t', soabi): - vars['Py_NOGIL'] = 1 + # Add EXT_SUFFIX, SOABI, and Py_NOGIL + vars.update(_winapi._sysconfig_vars()) - vars['LIBPL'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) if hasattr(sys, 'dllhandle'): - import _winapi dllhandle = _winapi.GetModuleFileName(sys.dllhandle) vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) vars['LDLIBRARY'] = vars['LIBRARY'] diff --git a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst index a5654775ce6e4a..80ec65081c0dc8 100644 --- a/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst +++ b/Misc/NEWS.d/next/Library/2023-09-28-12-32-57.gh-issue-88402.hoa3Gx.rst @@ -1,2 +1,2 @@ -Add new variables to :py:meth:`sysconfig.get_config_vars` on Windows: ``LIBRARY``, ``LDLIBRARY``, -``LIBPL``, ``SOABI``, and ``Py_NOGIL``. +Add new variables to :py:meth:`sysconfig.get_config_vars` on Windows: +``LIBRARY``, ``LDLIBRARY``, ``LIBDIR``, ``SOABI``, and ``Py_NOGIL``. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eec33499b983fe..e5746ce2f4c6c0 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -38,6 +38,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyInterpreterState_GET +#include "importdl.h" // _PyImport_DynLoadFiletab @@ -2166,6 +2167,32 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, } +/*[clinic input] +_winapi._sysconfig_vars + +Returns a dictionary containing variables intended to be exposed by sysconfig. +[clinic start generated code]*/ + +static PyObject * +_winapi__sysconfig_vars_impl(PyObject *module) +/*[clinic end generated code: output=8384e61338037cb1 input=065a8a08627f27bf]*/ +{ + PyObject *sysconfig = Py_BuildValue("{ssss}", + "EXT_SUFFIX", _PyImport_DynLoadFiletab[0], + "SOABI", _Py_SOABI); + if (sysconfig == NULL) { + return NULL; + } +#ifdef Py_NOGIL + if (PyDict_SetItem(sysconfig, "Py_NOGIL", _PyLong_GetOne()) < 0) { + Py_DECREF(sysconfig); + return NULL; + } +#endif + return sysconfig; +} + + static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF @@ -2190,6 +2217,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF + _WINAPI__SYSCONFIG_VARS_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF @@ -2376,6 +2404,7 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT("i", NULL); + return 0; } diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 7fef127bef8606..a15bde6b73ca3e 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -1478,4 +1478,22 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=6df38b5eb93f2e5a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_winapi__sysconfig_vars__doc__, +"_sysconfig_vars($module, /)\n" +"--\n" +"\n" +"Returns a dictionary containing variables intended to be exposed by sysconfig."); + +#define _WINAPI__SYSCONFIG_VARS_METHODDEF \ + {"_sysconfig_vars", (PyCFunction)_winapi__sysconfig_vars, METH_NOARGS, _winapi__sysconfig_vars__doc__}, + +static PyObject * +_winapi__sysconfig_vars_impl(PyObject *module); + +static PyObject * +_winapi__sysconfig_vars(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _winapi__sysconfig_vars_impl(module); +} +/*[clinic end generated code: output=56bf6530df63a262 input=a9049054013a1b77]*/ diff --git a/Python/dynload_win.c b/Python/dynload_win.c index fcb3cb744047ce..faeff26720ad50 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -22,13 +22,16 @@ #endif #ifdef PYD_PLATFORM_TAG -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG ".pyd" +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG #else -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX ".cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG ".pyd" +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG #endif +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" #define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" +const char *_Py_SOABI = PYD_SOABI; + const char *_PyImport_DynLoadFiletab[] = { PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, diff --git a/Python/importdl.h b/Python/importdl.h index 9171adc2770689..0f44de2bb357c8 100644 --- a/Python/importdl.h +++ b/Python/importdl.h @@ -18,6 +18,7 @@ typedef PyObject *(*PyModInitFunction)(void); #ifdef MS_WINDOWS #include typedef FARPROC dl_funcptr; +extern const char *_Py_SOABI; #else typedef void (*dl_funcptr)(void); #endif From aa396e5ef43c5dd1e5c2f8dfc357aaeafd39ec1c Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 15:23:20 -0400 Subject: [PATCH 04/10] Move Python/importdl.h to Include/internal/pycore_importdl.h --- .../importdl.h => Include/internal/pycore_importdl.h | 10 +++++++--- Modules/_winapi.c | 2 +- PCbuild/pythoncore.vcxproj | 2 +- PCbuild/pythoncore.vcxproj.filters | 6 +++--- Python/dynload_hpux.c | 2 +- Python/dynload_shlib.c | 2 +- Python/dynload_stub.c | 2 +- Python/dynload_win.c | 2 +- Python/import.c | 2 +- Python/importdl.c | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 11 files changed, 19 insertions(+), 15 deletions(-) rename Python/importdl.h => Include/internal/pycore_importdl.h (72%) diff --git a/Python/importdl.h b/Include/internal/pycore_importdl.h similarity index 72% rename from Python/importdl.h rename to Include/internal/pycore_importdl.h index 0f44de2bb357c8..cfbb0419749733 100644 --- a/Python/importdl.h +++ b/Include/internal/pycore_importdl.h @@ -1,10 +1,14 @@ -#ifndef Py_IMPORTDL_H -#define Py_IMPORTDL_H +#ifndef Py_INTERNAL_IMPORTDL_H +#define Py_INTERNAL_IMPORTDL_H #ifdef __cplusplus extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + extern const char *_PyImport_DynLoadFiletab[]; @@ -27,4 +31,4 @@ typedef void (*dl_funcptr)(void); #ifdef __cplusplus } #endif -#endif /* !Py_IMPORTDL_H */ +#endif /* !Py_INTERNAL_IMPORTDL_H */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index e5746ce2f4c6c0..9a85845e6d2fee 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -38,7 +38,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyInterpreterState_GET -#include "importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1ec106777db56d..3f558bb7e77faf 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -240,6 +240,7 @@ + @@ -367,7 +368,6 @@ - diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index f381120c9b035a..496f9ce5a6c502 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -300,9 +300,6 @@ PC - - Python - Python @@ -633,6 +630,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/dynload_hpux.c b/Python/dynload_hpux.c index a53373038ed859..1c44722ff9a2d0 100644 --- a/Python/dynload_hpux.c +++ b/Python/dynload_hpux.c @@ -5,7 +5,7 @@ #include #include "Python.h" -#include "importdl.h" +#include "pycore_importdl.h" #if defined(__hp9000s300) #define FUNCNAME_PATTERN "_%.20s_%.200s" diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 6761bba457983b..5a37a83805ba78 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -4,7 +4,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState.dlopenflags #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "importdl.h" +#include "pycore_importdl.h" #include #include diff --git a/Python/dynload_stub.c b/Python/dynload_stub.c index 59160483caa448..11f7e5f643f79e 100644 --- a/Python/dynload_stub.c +++ b/Python/dynload_stub.c @@ -3,7 +3,7 @@ not present. */ #include "Python.h" -#include "importdl.h" +#include "pycore_importdl.h" const char *_PyImport_DynLoadFiletab[] = {NULL}; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index faeff26720ad50..daec75ae7a4de3 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -5,7 +5,7 @@ #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "importdl.h" // dl_funcptr +#include "pycore_importdl.h" // dl_funcptr #include "patchlevel.h" // PY_MAJOR_VERSION #include diff --git a/Python/import.c b/Python/import.c index 5a06cb367e828b..b12e22ea910122 100644 --- a/Python/import.c +++ b/Python/import.c @@ -17,7 +17,7 @@ #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() -#include "importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab #include "pydtrace.h" // PyDTrace_IMPORT_FIND_LOAD_START_ENABLED() #include // bool diff --git a/Python/importdl.c b/Python/importdl.c index 9ab0a5ad33aaac..7dfd301d77efb4 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -15,7 +15,7 @@ */ #ifdef HAVE_DYNAMIC_LOADING -#include "importdl.h" +#include "pycore_importdl.h" #ifdef MS_WINDOWS extern dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 1f398701a7a5b5..cb9a23b4293b5a 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -603,6 +603,7 @@ Modules/_xxtestfuzz/fuzzer.c LLVMFuzzerTestOneInput AST_LITERAL_EVAL_INITIALIZED # XXX Fix the analyzer. ## forward/extern references +Include/internal/pycore_importdl.h - _PyImport_DynLoadFiletab - Include/py_curses.h - PyCurses_API - Include/pydecimal.h - _decimal_api - Modules/_blake2/blake2module.c - blake2b_type_spec - @@ -664,7 +665,6 @@ Objects/object.c - _PyLineIterator - Objects/object.c - _PyPositionsIterator - Python/perf_trampoline.c - _Py_trampoline_func_start - Python/perf_trampoline.c - _Py_trampoline_func_end - -Python/importdl.h - _PyImport_DynLoadFiletab - Modules/expat/xmlrole.c - prolog0 - Modules/expat/xmlrole.c - prolog1 - Modules/expat/xmlrole.c - prolog2 - From 89f891c7b6b8c90f998c09f57b5f5eb8bcae1a0b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 15:18:18 -0400 Subject: [PATCH 05/10] Add _sysconfig module --- Lib/sysconfig.py | 3 +- Lib/test/test_sysconfig.py | 3 + Modules/Setup | 1 + Modules/Setup.bootstrap.in | 1 + Modules/_sysconfig.c | 91 ++++++++++++++++++++++++++++++ Modules/_winapi.c | 29 ---------- Modules/clinic/_sysconfig.c.h | 22 ++++++++ Modules/clinic/_winapi.c.h | 20 +------ PC/config.c | 2 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/stdlib_module_names.h | 1 + 12 files changed, 128 insertions(+), 49 deletions(-) create mode 100644 Modules/_sysconfig.c create mode 100644 Modules/clinic/_sysconfig.c.h diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 4b88d2d1af8036..edfe451a86bfd0 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -545,12 +545,13 @@ def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories import _winapi + import _sysconfig vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') # Add EXT_SUFFIX, SOABI, and Py_NOGIL - vars.update(_winapi._sysconfig_vars()) + vars.update(_sysconfig.config_vars()) vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) if hasattr(sys, 'dllhandle'): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index b49c210b036f88..dea046716764c6 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -19,6 +19,7 @@ _expand_vars, _get_preferred_schemes, _main) import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -493,8 +494,10 @@ def test_srcdir_independent_of_cwd(self): 'EXT_SUFFIX required for this test') @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): + expected = _sysconfig.config_vars() vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) + self.assertEqual(vars['EXT_SUFFIX'], expected['EXT_SUFFIX']) @unittest.skipUnless(sys.platform == 'linux' and hasattr(sys.implementation, '_multiarch'), diff --git a/Modules/Setup b/Modules/Setup index 8676f9ddce4841..1367f0ef4fa54a 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -155,6 +155,7 @@ PYTHONPATH=$(COREPYTHONPATH) #math mathmodule.c #mmap mmapmodule.c #select selectmodule.c +#_sysconfig _sysconfig.c # XML #_elementtree _elementtree.c diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 8ef0f203a82a8e..cd12c1bd0df8f9 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -19,6 +19,7 @@ errno errnomodule.c _io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c itertools itertoolsmodule.c _sre _sre/sre.c +_sysconfig _sysconfig.c _thread _threadmodule.c time timemodule.c _typing _typingmodule.c diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c new file mode 100644 index 00000000000000..e5140d91b19111 --- /dev/null +++ b/Modules/_sysconfig.c @@ -0,0 +1,91 @@ +// _sysconfig provides data for the Python sysconfig module + +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 +#endif + +#include "Python.h" + +#include "pycore_importdl.h" // _PyImport_DynLoadFiletab + + +/*[clinic input] +module _sysconfig +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=0a7c02d3e212ac97]*/ + +#include "clinic/_sysconfig.c.h" + +static int +add_string_value(PyObject *dict, const char *key, const char *str_value) +{ + PyObject *value = PyUnicode_FromString(str_value); + if (value == NULL) { + return -1; + } + int err = PyDict_SetItemString(dict, key, value); + Py_DECREF(value); + return err; +} + +/*[clinic input] +_sysconfig.config_vars + +Returns a dictionary containing variables intended to be exposed by sysconfig. +[clinic start generated code]*/ + +static PyObject * +_sysconfig_config_vars_impl(PyObject *module) +/*[clinic end generated code: output=9c41cdee63ea9487 input=1458713a8d30b474]*/ +{ + PyObject *config = PyDict_New(); + if (config == NULL) { + return NULL; + } + if (_PyImport_DynLoadFiletab[0]) { + if (add_string_value(config, "EXT_SUFFIX", _PyImport_DynLoadFiletab[0]) < 0) { + Py_DECREF(config); + return NULL; + } + } +#ifdef MS_WINDOWS + if (add_string_value(config, "SOABI", _Py_SOABI) < 0) { + Py_DECREF(config); + return NULL; + } +#endif +#ifdef Py_NOGIL + if (PyDict_SetItem(config, "Py_NOGIL", _PyLong_GetOne()) < 0) { + Py_DECREF(config); + return NULL; + } +#endif + return config; +} + +PyDoc_STRVAR(sysconfig__doc__, +"A helper for the sysconfig module."); + +static struct PyMethodDef sysconfig_methods[] = { + _SYSCONFIG_CONFIG_VARS_METHODDEF + {NULL, NULL} +}; + +static PyModuleDef_Slot sysconfig_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef sysconfig_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_sysconfig", + .m_doc = sysconfig__doc__, + .m_methods = sysconfig_methods, + .m_slots = sysconfig_slots, +}; + +PyMODINIT_FUNC +PyInit__sysconfig(void) +{ + return PyModuleDef_Init(&sysconfig_module); +} diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 9a85845e6d2fee..eec33499b983fe 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -38,7 +38,6 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() #include "pycore_pystate.h" // _PyInterpreterState_GET -#include "pycore_importdl.h" // _PyImport_DynLoadFiletab @@ -2167,32 +2166,6 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, } -/*[clinic input] -_winapi._sysconfig_vars - -Returns a dictionary containing variables intended to be exposed by sysconfig. -[clinic start generated code]*/ - -static PyObject * -_winapi__sysconfig_vars_impl(PyObject *module) -/*[clinic end generated code: output=8384e61338037cb1 input=065a8a08627f27bf]*/ -{ - PyObject *sysconfig = Py_BuildValue("{ssss}", - "EXT_SUFFIX", _PyImport_DynLoadFiletab[0], - "SOABI", _Py_SOABI); - if (sysconfig == NULL) { - return NULL; - } -#ifdef Py_NOGIL - if (PyDict_SetItem(sysconfig, "Py_NOGIL", _PyLong_GetOne()) < 0) { - Py_DECREF(sysconfig); - return NULL; - } -#endif - return sysconfig; -} - - static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF @@ -2217,7 +2190,6 @@ static PyMethodDef winapi_functions[] = { _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF - _WINAPI__SYSCONFIG_VARS_METHODDEF _WINAPI_TERMINATEPROCESS_METHODDEF _WINAPI_UNMAPVIEWOFFILE_METHODDEF _WINAPI_VIRTUALQUERYSIZE_METHODDEF @@ -2404,7 +2376,6 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT("i", NULL); - return 0; } diff --git a/Modules/clinic/_sysconfig.c.h b/Modules/clinic/_sysconfig.c.h new file mode 100644 index 00000000000000..3ca7d79d0d2dea --- /dev/null +++ b/Modules/clinic/_sysconfig.c.h @@ -0,0 +1,22 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +PyDoc_STRVAR(_sysconfig_config_vars__doc__, +"config_vars($module, /)\n" +"--\n" +"\n" +"Returns a dictionary containing variables intended to be exposed by sysconfig."); + +#define _SYSCONFIG_CONFIG_VARS_METHODDEF \ + {"config_vars", (PyCFunction)_sysconfig_config_vars, METH_NOARGS, _sysconfig_config_vars__doc__}, + +static PyObject * +_sysconfig_config_vars_impl(PyObject *module); + +static PyObject * +_sysconfig_config_vars(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _sysconfig_config_vars_impl(module); +} +/*[clinic end generated code: output=fc65aa12b5547bf8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index a15bde6b73ca3e..7fef127bef8606 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -1478,22 +1478,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } - -PyDoc_STRVAR(_winapi__sysconfig_vars__doc__, -"_sysconfig_vars($module, /)\n" -"--\n" -"\n" -"Returns a dictionary containing variables intended to be exposed by sysconfig."); - -#define _WINAPI__SYSCONFIG_VARS_METHODDEF \ - {"_sysconfig_vars", (PyCFunction)_winapi__sysconfig_vars, METH_NOARGS, _winapi__sysconfig_vars__doc__}, - -static PyObject * -_winapi__sysconfig_vars_impl(PyObject *module); - -static PyObject * -_winapi__sysconfig_vars(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return _winapi__sysconfig_vars_impl(module); -} -/*[clinic end generated code: output=56bf6530df63a262 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6df38b5eb93f2e5a input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index 88f69758aac764..da2bde640961e0 100644 --- a/PC/config.c +++ b/PC/config.c @@ -22,6 +22,7 @@ extern PyObject* PyInit__sha1(void); extern PyObject* PyInit__sha2(void); extern PyObject* PyInit__sha3(void); extern PyObject* PyInit__statistics(void); +extern PyObject* PyInit__sysconfig(void); extern PyObject* PyInit__typing(void); extern PyObject* PyInit__blake2(void); extern PyObject* PyInit_time(void); @@ -102,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = { {"_sha2", PyInit__sha2}, {"_sha3", PyInit__sha3}, {"_blake2", PyInit__blake2}, + {"_sysconfig", PyInit__sysconfig}, {"time", PyInit_time}, {"_thread", PyInit__thread}, {"_tokenize", PyInit__tokenize}, diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 3f558bb7e77faf..43a79fd5938486 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -438,6 +438,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 496f9ce5a6c502..59159ed609968b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -959,6 +959,9 @@ Modules + + Modules + Modules diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 13b1764f0886d1..701bfc35cc8182 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -77,6 +77,7 @@ static const char* _Py_stdlib_module_names[] = { "_strptime", "_struct", "_symtable", +"_sysconfig", "_thread", "_threading_local", "_tkinter", From 14d201c48114020bcdc53e87c3530fa416501a6f Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 17:39:46 -0400 Subject: [PATCH 06/10] Update Modules/_sysconfig.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Filipe Laíns --- Modules/_sysconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index e5140d91b19111..4c87d48ab78f57 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -31,7 +31,7 @@ add_string_value(PyObject *dict, const char *key, const char *str_value) /*[clinic input] _sysconfig.config_vars -Returns a dictionary containing variables intended to be exposed by sysconfig. +Returns a dictionary containing build variables intended to be exposed by sysconfig. [clinic start generated code]*/ static PyObject * From 34fe412468a1b5be2adbf14f78856c117880a46b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 17:45:19 -0400 Subject: [PATCH 07/10] Fix "Py_NOGIL" sysconfig variable. It should be zero or one and always defined. --- Modules/_sysconfig.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index 4c87d48ab78f57..a9259335a12e3e 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -7,6 +7,7 @@ #include "Python.h" #include "pycore_importdl.h" // _PyImport_DynLoadFiletab +#include "pycore_long.h" // _PyLong_GetZero, _PyLong_GetOne /*[clinic input] @@ -54,12 +55,17 @@ _sysconfig_config_vars_impl(PyObject *module) return NULL; } #endif + #ifdef Py_NOGIL - if (PyDict_SetItem(config, "Py_NOGIL", _PyLong_GetOne()) < 0) { + PyObject *py_nogil = _PyLong_GetOne(); +#else + PyObject *py_nogil = _PyLong_GetZero(); +#endif + if (PyDict_SetItemString(config, "Py_NOGIL", py_nogil) < 0) { Py_DECREF(config); return NULL; } -#endif + return config; } From db30ede9de8c6cb682c94cbfc3b0131498019341 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 17:51:11 -0400 Subject: [PATCH 08/10] Compute PYD_TAGGED_SUFFIX and PYD_SOABI in pycore_importdl.h --- Include/internal/pycore_importdl.h | 25 ++++++++++++++++++++++++- Modules/_sysconfig.c | 15 ++++++++------- Python/dynload_win.c | 23 ----------------------- 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index cfbb0419749733..dee64241c763f3 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -1,6 +1,8 @@ #ifndef Py_INTERNAL_IMPORTDL_H #define Py_INTERNAL_IMPORTDL_H +#include "patchlevel.h" // PY_MAJOR_VERSION + #ifdef __cplusplus extern "C" { #endif @@ -22,7 +24,28 @@ typedef PyObject *(*PyModInitFunction)(void); #ifdef MS_WINDOWS #include typedef FARPROC dl_funcptr; -extern const char *_Py_SOABI; + +#ifdef _DEBUG +# define PYD_DEBUG_SUFFIX "_d" +#else +# define PYD_DEBUG_SUFFIX "" +#endif + +#ifdef Py_NOGIL +# define PYD_THREADING_TAG "t" +#else +# define PYD_THREADING_TAG "" +#endif + +#ifdef PYD_PLATFORM_TAG +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG +#else +# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG +#endif + +#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" +#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" + #else typedef void (*dl_funcptr)(void); #endif diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index a9259335a12e3e..ae3dc9444e077d 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -17,6 +17,7 @@ module _sysconfig #include "clinic/_sysconfig.c.h" +#ifdef MS_WINDOWS static int add_string_value(PyObject *dict, const char *key, const char *str_value) { @@ -28,6 +29,7 @@ add_string_value(PyObject *dict, const char *key, const char *str_value) Py_DECREF(value); return err; } +#endif /*[clinic input] _sysconfig.config_vars @@ -43,14 +45,13 @@ _sysconfig_config_vars_impl(PyObject *module) if (config == NULL) { return NULL; } - if (_PyImport_DynLoadFiletab[0]) { - if (add_string_value(config, "EXT_SUFFIX", _PyImport_DynLoadFiletab[0]) < 0) { - Py_DECREF(config); - return NULL; - } - } + #ifdef MS_WINDOWS - if (add_string_value(config, "SOABI", _Py_SOABI) < 0) { + if (add_string_value(config, "EXT_SUFFIX", PYD_TAGGED_SUFFIX) < 0) { + Py_DECREF(config); + return NULL; + } + if (add_string_value(config, "SOABI", PYD_SOABI) < 0) { Py_DECREF(config); return NULL; } diff --git a/Python/dynload_win.c b/Python/dynload_win.c index daec75ae7a4de3..a0ac31c80a5f6e 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -9,29 +9,6 @@ #include "patchlevel.h" // PY_MAJOR_VERSION #include -#ifdef _DEBUG -#define PYD_DEBUG_SUFFIX "_d" -#else -#define PYD_DEBUG_SUFFIX "" -#endif - -#ifdef Py_NOGIL -# define PYD_THREADING_TAG "t" -#else -# define PYD_THREADING_TAG "" -#endif - -#ifdef PYD_PLATFORM_TAG -# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG "-" PYD_PLATFORM_TAG -#else -# define PYD_SOABI "cp" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) PYD_THREADING_TAG -#endif - -#define PYD_TAGGED_SUFFIX PYD_DEBUG_SUFFIX "." PYD_SOABI ".pyd" -#define PYD_UNTAGGED_SUFFIX PYD_DEBUG_SUFFIX ".pyd" - -const char *_Py_SOABI = PYD_SOABI; - const char *_PyImport_DynLoadFiletab[] = { PYD_TAGGED_SUFFIX, PYD_UNTAGGED_SUFFIX, From f0fb2354784d247347570a5d8eecbd743156f9c4 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 18:05:53 -0400 Subject: [PATCH 09/10] regen clinic --- Modules/_sysconfig.c | 2 +- Modules/clinic/_sysconfig.c.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index ae3dc9444e077d..6f1cc16b58467d 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -39,7 +39,7 @@ Returns a dictionary containing build variables intended to be exposed by syscon static PyObject * _sysconfig_config_vars_impl(PyObject *module) -/*[clinic end generated code: output=9c41cdee63ea9487 input=1458713a8d30b474]*/ +/*[clinic end generated code: output=9c41cdee63ea9487 input=391ff42f3af57d01]*/ { PyObject *config = PyDict_New(); if (config == NULL) { diff --git a/Modules/clinic/_sysconfig.c.h b/Modules/clinic/_sysconfig.c.h index 3ca7d79d0d2dea..eb3d396298bb21 100644 --- a/Modules/clinic/_sysconfig.c.h +++ b/Modules/clinic/_sysconfig.c.h @@ -6,7 +6,7 @@ PyDoc_STRVAR(_sysconfig_config_vars__doc__, "config_vars($module, /)\n" "--\n" "\n" -"Returns a dictionary containing variables intended to be exposed by sysconfig."); +"Returns a dictionary containing build variables intended to be exposed by sysconfig."); #define _SYSCONFIG_CONFIG_VARS_METHODDEF \ {"config_vars", (PyCFunction)_sysconfig_config_vars, METH_NOARGS, _sysconfig_config_vars__doc__}, @@ -19,4 +19,4 @@ _sysconfig_config_vars(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _sysconfig_config_vars_impl(module); } -/*[clinic end generated code: output=fc65aa12b5547bf8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=25d395cf02eced1f input=a9049054013a1b77]*/ From 0a84d766292c767560e82033c08dc404f3bb864b Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 4 Oct 2023 18:21:14 -0400 Subject: [PATCH 10/10] Fix sysconfig test on POSIX systems. The C _sysconfig module no longer defines EXT_SUFFIX for POSIX systems --- Lib/test/test_sysconfig.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index dea046716764c6..a077ac5349fdc6 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -494,10 +494,8 @@ def test_srcdir_independent_of_cwd(self): 'EXT_SUFFIX required for this test') @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - expected = _sysconfig.config_vars() vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - self.assertEqual(vars['EXT_SUFFIX'], expected['EXT_SUFFIX']) @unittest.skipUnless(sys.platform == 'linux' and hasattr(sys.implementation, '_multiarch'),