From b9b572126f0785fa039deec113d69601d9628ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 29 Apr 2021 13:52:59 +0100 Subject: [PATCH 1/9] bpo-41282: add vendor config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Doc/library/site.rst | 8 +- Doc/library/sysconfig.rst | 19 +- Doc/using/configure.rst | 26 ++ Lib/_sysconfig.py | 285 ++++++++++++++++++ Lib/_vendor/__init__.py | 0 Lib/distutils/command/install.py | 3 +- Lib/distutils/sysconfig.py | 12 +- Lib/site.py | 20 ++ Lib/sysconfig.py | 282 ++--------------- Lib/test/support/__init__.py | 24 ++ Lib/test/test_site.py | 40 +++ Lib/test/test_sysconfig.py | 39 ++- Lib/test/vendor_config.py | 26 ++ Makefile.pre.in | 7 +- .../2021-04-28-21-57-32.bpo-41282.xgQ6Cn.rst | 2 + configure.ac | 21 ++ 16 files changed, 531 insertions(+), 283 deletions(-) create mode 100644 Lib/_sysconfig.py create mode 100644 Lib/_vendor/__init__.py create mode 100644 Lib/test/vendor_config.py create mode 100644 Misc/NEWS.d/next/Build/2021-04-28-21-57-32.bpo-41282.xgQ6Cn.rst diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e2ad3c48f9754e..6962fdd0b3e364 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -32,7 +32,9 @@ It starts by constructing up to four directories from a head and a tail part. For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and macOS). For each +:file:`lib/python{X.Y}/site-packages` (on Unix and macOS), and finally +the ``purelib`` and ``platlib`` paths for each scheme specified in the +``EXTRA_SITE_INSTALL_SCHEMES`` list variable of the vendor config. For each of the distinct head-tail combinations, it sees if it refers to an existing directory, and if so, adds it to ``sys.path`` and also inspects the newly added path for configuration files. @@ -40,6 +42,10 @@ added path for configuration files. .. versionchanged:: 3.5 Support for the "site-python" directory has been removed. +.. versionchanged:: 3.11 + Extra site install schemes specified in the vendor config + (``--with-vendor-config`` configure option) will also be loaded. + If a file named "pyvenv.cfg" exists one directory above sys.executable, sys.prefix and sys.exec_prefix are set to that directory and it is also checked for site-packages (sys.base_prefix and diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 6327318eb108da..67b09e22395828 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -85,6 +85,10 @@ Python currently supports seven schemes: - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. +Additionally to these, Python also supports vendor schemes specified in the +``EXTRA_INSTALL_SCHEMES`` dictionary variable of the vendor config +(``--with-vendor-config`` configure option). + Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: @@ -129,21 +133,6 @@ identifier. Python currently uses eight paths: .. versionadded:: 3.10 -.. function:: _get_preferred_schemes() - - Return a dict containing preferred scheme names on the current platform. - Python implementers and redistributors may add their preferred schemes to - the ``_INSTALL_SCHEMES`` module-level global value, and modify this function - to return those scheme names, to e.g. provide different schemes for system - and language package managers to use, so packages installed by either do not - mix with those by the other. - - End users should not use this function, but :func:`get_default_scheme` and - :func:`get_preferred_scheme()` instead. - - .. versionadded:: 3.10 - - .. function:: get_path_names() Return a tuple containing all path names currently supported in diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 75f572c61877fe..aef768a93e31a1 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -116,6 +116,32 @@ General Options .. versionadded:: 3.10 +.. cmdoption:: --with-vendor-config=config.py + + Path to the vendor config (none by default). + + The vendor config is a Python file that allows configuring some aspects of + the Python distribution. + + A ``EXTRA_INSTALL_SCHEMES`` dictionary variable can be specified in the + config to add extra install schemes. These schemes will be picked up by the + :mod:`sysconfig` module. + + A ``EXTRA_SITE_INSTALL_SCHEMES`` list variable can be specified in the config + to add extra schemes to the :mod:`site` module initialization. This options + allow Python distributors to define custom locations to use for their Python + packages. + + A ``get_preferred_schemes()`` function can be specified in the config. This + function should return a dict containing preferred scheme names on the + current platform. Python implementers and redistributors may add their + preferred schemes to the ``EXTRA_INSTALL_SCHEMES`` vendor config variable, + and modify this function to return those scheme names, to e.g. provide + different schemes for system and language package managers to use, so + packages installed by either do not mix with those by the other. + + .. versionadded:: 3.11 + Install Options --------------- diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py new file mode 100644 index 00000000000000..0c0e5c6f138839 --- /dev/null +++ b/Lib/_sysconfig.py @@ -0,0 +1,285 @@ +import os +import sys + +_INSTALL_SCHEMES = { + 'posix_prefix': { + 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'include': + '{installed_base}/include/python{py_version_short}{abiflags}', + 'platinclude': + '{installed_platbase}/include/python{py_version_short}{abiflags}', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'posix_home': { + 'stdlib': '{installed_base}/lib/python', + 'platstdlib': '{base}/lib/python', + 'purelib': '{base}/lib/python', + 'platlib': '{base}/lib/python', + 'include': '{installed_base}/include/python', + 'platinclude': '{installed_base}/include/python', + 'scripts': '{base}/bin', + 'data': '{base}', + }, + 'nt': { + 'stdlib': '{installed_base}/Lib', + 'platstdlib': '{base}/Lib', + 'purelib': '{base}/Lib/site-packages', + 'platlib': '{base}/Lib/site-packages', + 'include': '{installed_base}/Include', + 'platinclude': '{installed_base}/Include', + 'scripts': '{base}/Scripts', + 'data': '{base}', + }, + } + +_get_preferred_schemes = None + + +def _load_vendor_schemes(): + # add vendor defined schemes + try: + import _vendor.config + + extra_schemes = _vendor.config.EXTRA_INSTALL_SCHEMES + _INSTALL_SCHEMES.update({ + name: scheme + for name, scheme in extra_schemes.items() + if name not in _INSTALL_SCHEMES + }) + except (ModuleNotFoundError, AttributeError): + pass + + +_load_vendor_schemes() + + +# NOTE: site.py has copy of this function. +# Sync it when modify this function. +def _getuserbase(): + env_base = os.environ.get("PYTHONUSERBASE", None) + if env_base: + return env_base + + # VxWorks has no home directories + if sys.platform == "vxworks": + return None + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + return joinuser(base, "Python") + + if sys.platform == "darwin" and sys._framework: + return joinuser("~", "Library", sys._framework, + f"{sys.version_info[0]}.{sys.version_info[1]}") + + return joinuser("~", ".local") + + +_HAS_USER_BASE = (_getuserbase() is not None) + +if _HAS_USER_BASE: + _INSTALL_SCHEMES |= { + # NOTE: When modifying "purelib" scheme, update site._get_path() too. + 'nt_user': { + 'stdlib': '{userbase}/Python{py_version_nodot_plat}', + 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', + 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/Python{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'data': '{userbase}', + }, + 'posix_user': { + 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', + 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, + 'osx_framework_user': { + 'stdlib': '{userbase}/lib/python', + 'platstdlib': '{userbase}/lib/python', + 'purelib': '{userbase}/lib/python/site-packages', + 'platlib': '{userbase}/lib/python/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, + } + +_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', + 'scripts', 'data') + +_PY_VERSION = sys.version.split()[0] +_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' +_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' +_PREFIX = os.path.normpath(sys.prefix) +_BASE_PREFIX = os.path.normpath(sys.base_prefix) +_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) +_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +_USER_BASE = None + + +def _safe_realpath(path): + try: + return os.path.realpath(path) + except OSError: + return path + + +if sys.executable: + _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) +else: + # sys.executable can be empty if argv[0] has been changed and Python is + # unable to retrieve the real program name + _PROJECT_BASE = _safe_realpath(os.getcwd()) + +if (os.name == 'nt' and + _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): + _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, os.path.pardir, os.path.pardir)) + +# set for cross builds +if "_PYTHON_PROJECT_BASE" in os.environ: + _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) + + +def _is_python_source_dir(d): + for fn in ("Setup", "Setup.local"): + if os.path.isfile(os.path.join(d, "Modules", fn)): + return True + return False + + +_SYS_HOME = getattr(sys, '_home', None) + + +if os.name == 'nt': + def _fix_pcbuild(d): + if d and os.path.normcase(d).startswith( + os.path.normcase(os.path.join(_PREFIX, "PCbuild"))): + return _PREFIX + return d + _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE) + _SYS_HOME = _fix_pcbuild(_SYS_HOME) + + +def is_python_build(check_home=False): + if check_home and _SYS_HOME: + return _is_python_source_dir(_SYS_HOME) + return _is_python_source_dir(getattr(sys.modules[__name__], '_PROJECT_BASE')) + + +_PYTHON_BUILD = is_python_build(True) + +if _PYTHON_BUILD: + for scheme in ('posix_prefix', 'posix_home'): + # On POSIX-y platforms, Python will: + # - Build from .h files in 'headers' (which is only added to the + # scheme when building CPython) + # - Install .h files to 'include' + scheme = _INSTALL_SCHEMES[scheme] + scheme['headers'] = scheme['include'] + scheme['include'] = '{srcdir}/Include' + scheme['platinclude'] = '{projectbase}/.' + + +_SCHEME_CONFIG_VARS = { + 'prefix': _PREFIX, + 'exec_prefix': _EXEC_PREFIX, + 'py_version': _PY_VERSION, + 'py_version_short': _PY_VERSION_SHORT, + 'py_version_nodot': _PY_VERSION_SHORT_NO_DOT, + 'installed_base': _BASE_PREFIX, + 'base': _PREFIX, + 'installed_platbase': _BASE_EXEC_PREFIX, + 'platbase': _EXEC_PREFIX, + 'projectbase': _PROJECT_BASE, + 'platlibdir': sys.platlibdir, + 'abiflags': getattr(sys, 'abiflags', ''), + 'py_version_nodot_plat': getattr(sys, 'winver', '').replace('.', '') +} + + +def _subst_vars(s, local_vars): + try: + return s.format(**local_vars) + except KeyError as var: + try: + return s.format(**os.environ) + except KeyError: + raise AttributeError(f'{var}') from None + + +def _expand_vars(scheme, vars): + if vars is None: + vars = _SCHEME_CONFIG_VARS + res = {} + for key, value in _INSTALL_SCHEMES[scheme].items(): + if os.name in ('posix', 'nt'): + value = os.path.expanduser(value) + res[key] = os.path.normpath(_subst_vars(value, vars)) + return res + + +def _get_preferred_schemes_default(): + if os.name == 'nt': + return { + 'prefix': 'nt', + 'home': 'posix_home', + 'user': 'nt_user', + } + if sys.platform == 'darwin' and sys._framework: + return { + 'prefix': 'posix_prefix', + 'home': 'posix_home', + 'user': 'osx_framework_user', + } + return { + 'prefix': 'posix_prefix', + 'home': 'posix_home', + 'user': 'posix_user', + } + + +def get_preferred_scheme(key): + global _get_preferred_schemes + if not _get_preferred_schemes: + try: + import _vendor.config + + _get_preferred_schemes = _vendor.config.get_preferred_schemes + except (ModuleNotFoundError, AttributeError): + _get_preferred_schemes = _get_preferred_schemes_default + + scheme = _get_preferred_schemes()[key] + if scheme not in _INSTALL_SCHEMES: + raise ValueError( + f"{key!r} returned {scheme!r}, which is not a valid scheme " + f"on this platform" + ) + return scheme + + +def get_default_scheme(): + return get_preferred_scheme('prefix') + + +def _get_paths(scheme=get_default_scheme(), vars=None, expand=True): + """Return a mapping containing an install scheme. + + ``scheme`` is the install scheme name. If not provided, it will + return the default scheme for the current platform. + """ + if expand: + return _expand_vars(scheme, vars) + else: + return _INSTALL_SCHEMES[scheme] diff --git a/Lib/_vendor/__init__.py b/Lib/_vendor/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 01d5331a63069b..b31304b4c91633 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -3,6 +3,7 @@ Implements the Distutils 'install' command.""" import sys +import _sysconfig import sysconfig import os import re @@ -41,7 +42,7 @@ ("unix_prefix", "posix_prefix"), ("unix_home", "posix_home"), ("nt", "nt")): sys_key = key - sys_scheme = sysconfig._INSTALL_SCHEMES[sys_scheme_name] + sys_scheme = _sysconfig._INSTALL_SCHEMES[sys_scheme_name] if key == "headers" and key not in sys_scheme: # On POSIX-y platforms, Python will: # - Build from .h files in 'headers' (only there when diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 3414a761e76b99..c67fca5b8aee0e 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -19,19 +19,23 @@ from .errors import DistutilsPlatformError -from sysconfig import ( +from _sysconfig import ( _PREFIX as PREFIX, _BASE_PREFIX as BASE_PREFIX, _EXEC_PREFIX as EXEC_PREFIX, _BASE_EXEC_PREFIX as BASE_EXEC_PREFIX, _PROJECT_BASE as project_base, _PYTHON_BUILD as python_build, + _SYS_HOME as _sys_home, + + _is_python_source_dir, +) + +from sysconfig import ( _init_posix as sysconfig_init_posix, parse_config_h as sysconfig_parse_config_h, _init_non_posix, - _is_python_source_dir, - _sys_home, _variable_rx, _findvar1_rx, @@ -53,7 +57,7 @@ _config_vars = get_config_vars() if os.name == "nt": - from sysconfig import _fix_pcbuild + from _sysconfig import _fix_pcbuild warnings.warn( 'The distutils.sysconfig module is deprecated, use sysconfig instead', diff --git a/Lib/site.py b/Lib/site.py index e129f3b4851f3d..f94d8ca59dd265 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -87,6 +87,8 @@ USER_SITE = None USER_BASE = None +_VENDOR_SCHEMES = None + def _trace(message): if sys.flags.verbose: @@ -350,6 +352,7 @@ def getsitepackages(prefixes=None): this function will find its `site-packages` subdirectory depending on the system environment, and will return a list of full paths. """ + global _VENDOR_SCHEMES sitepackages = [] seen = set() @@ -377,6 +380,23 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, "site-packages") sitepackages.append(path) + + if _VENDOR_SCHEMES is None: # delayed execution + try: + import _vendor.config + + _VENDOR_SCHEMES = _vendor.config.EXTRA_SITE_INSTALL_SCHEMES + except (ModuleNotFoundError, AttributeError): + _VENDOR_SCHEMES = [] + + # vendor site schemes + if _VENDOR_SCHEMES: + import _sysconfig + + for scheme in _VENDOR_SCHEMES: + paths = _sysconfig._get_paths(scheme) + sitepackages += list({paths['purelib'], paths['platlib']}) + return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index daf9f000060a35..df5420f0c1e35a 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -2,7 +2,13 @@ import os import sys -from os.path import pardir, realpath +from _sysconfig import ( + _get_paths, _getuserbase, _safe_realpath, + get_default_scheme, get_preferred_scheme, is_python_build, + _HAS_USER_BASE, _INSTALL_SCHEMES, _PROJECT_BASE, + _PYTHON_BUILD, _PY_VERSION_SHORT, _PY_VERSION_SHORT_NO_DOT, + _SCHEME_CONFIG_VARS, _SCHEME_KEYS, _SYS_HOME, +) __all__ = [ 'get_config_h_filename', @@ -13,8 +19,10 @@ 'get_path_names', 'get_paths', 'get_platform', + 'get_preferred_scheme', 'get_python_version', 'get_scheme_names', + 'is_python_build', 'parse_config_h', ] @@ -23,112 +31,7 @@ 'MACOSX_DEPLOYMENT_TARGET', } -_INSTALL_SCHEMES = { - 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', - 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', - 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', - 'scripts': '{base}/bin', - 'data': '{base}', - }, - 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', - 'scripts': '{base}/bin', - 'data': '{base}', - }, - 'nt': { - 'stdlib': '{installed_base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{installed_base}/Include', - 'platinclude': '{installed_base}/Include', - 'scripts': '{base}/Scripts', - 'data': '{base}', - }, - } - - -# NOTE: site.py has copy of this function. -# Sync it when modify this function. -def _getuserbase(): - env_base = os.environ.get("PYTHONUSERBASE", None) - if env_base: - return env_base - - # VxWorks has no home directories - if sys.platform == "vxworks": - return None - - def joinuser(*args): - return os.path.expanduser(os.path.join(*args)) - - if os.name == "nt": - base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") - - if sys.platform == "darwin" and sys._framework: - return joinuser("~", "Library", sys._framework, - f"{sys.version_info[0]}.{sys.version_info[1]}") - - return joinuser("~", ".local") - -_HAS_USER_BASE = (_getuserbase() is not None) - -if _HAS_USER_BASE: - _INSTALL_SCHEMES |= { - # NOTE: When modifying "purelib" scheme, update site._get_path() too. - 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', - 'data': '{userbase}', - }, - 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - } - -_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', - 'scripts', 'data') - -_PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' -_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) -_BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) _CONFIG_VARS = None -_USER_BASE = None # Regexes needed for parsing Makefile (and similar syntaxes, # like old-style Setup files). @@ -137,127 +40,6 @@ def joinuser(*args): _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" -def _safe_realpath(path): - try: - return realpath(path) - except OSError: - return path - -if sys.executable: - _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) -else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - _PROJECT_BASE = _safe_realpath(os.getcwd()) - -if (os.name == 'nt' and - _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir)) - -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) - -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False - -_sys_home = getattr(sys, '_home', None) - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(_PREFIX, "PCbuild"))): - return _PREFIX - return d - _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE) - _sys_home = _fix_pcbuild(_sys_home) - -def is_python_build(check_home=False): - if check_home and _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(_PROJECT_BASE) - -_PYTHON_BUILD = is_python_build(True) - -if _PYTHON_BUILD: - for scheme in ('posix_prefix', 'posix_home'): - # On POSIX-y platforms, Python will: - # - Build from .h files in 'headers' (which is only added to the - # scheme when building CPython) - # - Install .h files to 'include' - scheme = _INSTALL_SCHEMES[scheme] - scheme['headers'] = scheme['include'] - scheme['include'] = '{srcdir}/Include' - scheme['platinclude'] = '{projectbase}/.' - - -def _subst_vars(s, local_vars): - try: - return s.format(**local_vars) - except KeyError as var: - try: - return s.format(**os.environ) - except KeyError: - raise AttributeError(f'{var}') from None - -def _extend_dict(target_dict, other_dict): - target_keys = target_dict.keys() - for key, value in other_dict.items(): - if key in target_keys: - continue - target_dict[key] = value - - -def _expand_vars(scheme, vars): - res = {} - if vars is None: - vars = {} - _extend_dict(vars, get_config_vars()) - - for key, value in _INSTALL_SCHEMES[scheme].items(): - if os.name in ('posix', 'nt'): - value = os.path.expanduser(value) - res[key] = os.path.normpath(_subst_vars(value, vars)) - return res - - -def _get_preferred_schemes(): - if os.name == 'nt': - return { - 'prefix': 'nt', - 'home': 'posix_home', - 'user': 'nt_user', - } - if sys.platform == 'darwin' and sys._framework: - return { - 'prefix': 'posix_prefix', - 'home': 'posix_home', - 'user': 'osx_framework_user', - } - return { - 'prefix': 'posix_prefix', - 'home': 'posix_home', - 'user': 'posix_user', - } - - -def get_preferred_scheme(key): - scheme = _get_preferred_schemes()[key] - if scheme not in _INSTALL_SCHEMES: - raise ValueError( - f"{key!r} returned {scheme!r}, which is not a valid scheme " - f"on this platform" - ) - return scheme - - -def get_default_scheme(): - return get_preferred_scheme('prefix') - - def _parse_makefile(filename, vars=None, keep_unresolved=True): """Parse a Makefile-style file. @@ -389,7 +171,7 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: - return os.path.join(_sys_home or _PROJECT_BASE, "Makefile") + return os.path.join(_SYS_HOME or _PROJECT_BASE, "Makefile") if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: @@ -470,6 +252,7 @@ def _generate_posix_vars(): with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) + def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() @@ -478,6 +261,7 @@ def _init_posix(vars): build_time_vars = _temp.build_time_vars vars.update(build_time_vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories @@ -491,6 +275,14 @@ def _init_non_posix(vars): vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) vars['TZPATH'] = '' + +def _extend_dict(target_dict, other_dict): + target_keys = target_dict.keys() + for key, value in other_dict.items(): + if key in target_keys: + continue + target_dict[key] = value + # # public APIs # @@ -534,9 +326,9 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_sys_home or _PROJECT_BASE, "PC") + inc_dir = os.path.join(_SYS_HOME or _PROJECT_BASE, "PC") else: - inc_dir = _sys_home or _PROJECT_BASE + inc_dir = _SYS_HOME or _PROJECT_BASE else: inc_dir = get_path('platinclude') return os.path.join(inc_dir, 'pyconfig.h') @@ -558,10 +350,10 @@ def get_paths(scheme=get_default_scheme(), vars=None, expand=True): ``scheme`` is the install scheme name. If not provided, it will return the default scheme for the current platform. """ - if expand: - return _expand_vars(scheme, vars) - else: - return _INSTALL_SCHEMES[scheme] + if vars is None: + vars = {} + _extend_dict(vars, get_config_vars()) + return _get_paths(scheme, vars, expand) def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): @@ -584,30 +376,10 @@ def get_config_vars(*args): """ global _CONFIG_VARS if _CONFIG_VARS is None: - _CONFIG_VARS = {} + _CONFIG_VARS = _SCHEME_CONFIG_VARS # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX - _CONFIG_VARS['py_version'] = _PY_VERSION - _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX - _CONFIG_VARS['projectbase'] = _PROJECT_BASE - _CONFIG_VARS['platlibdir'] = sys.platlibdir - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' - try: - _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') - except AttributeError: - _CONFIG_VARS['py_version_nodot_plat'] = '' if os.name == 'nt': _init_non_posix(_CONFIG_VARS) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 85fd74126b5f47..e299f630c347be 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -15,6 +15,7 @@ import unittest import warnings +import test.vendor_config from .testresult import get_test_runner @@ -55,6 +56,7 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", + "with_test_vendor_config", ] @@ -2091,3 +2093,25 @@ def clear_ignored_deprecations(*tokens: object) -> None: if warnings.filters != new_filters: warnings.filters[:] = new_filters warnings._filters_mutated() + + +@contextlib.contextmanager +def with_test_vendor_config(): + # this is needed because we are mocking package module + try: + import _vendor.config + old_config = _vendor.config + except ModuleNotFoundError: + old_config = None + + with unittest.mock.patch.dict(sys.modules, {'_vendor.config': test.vendor_config}): + import _vendor + + _vendor.config = test.vendor_config + + yield + + if old_config: + _vendor.config = old_config + else: + delattr(_vendor, 'config') diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 5f06a0d4b03725..1ca111a08266c3 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -10,6 +10,7 @@ from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr +from test.support import with_test_vendor_config from test.support.os_helper import TESTFN, EnvironmentVarGuard, change_cwd import builtins import encodings @@ -20,6 +21,7 @@ import shutil import subprocess import sys +import _sysconfig import sysconfig import tempfile import urllib.error @@ -69,8 +71,11 @@ def setUp(self): self.old_base = site.USER_BASE self.old_site = site.USER_SITE self.old_prefixes = site.PREFIXES + self.old_vendor_schemes = site._VENDOR_SCHEMES self.original_vars = sysconfig._CONFIG_VARS self.old_vars = copy(sysconfig._CONFIG_VARS) + self.original_schemes = sysconfig._INSTALL_SCHEMES + self.old_schemes = copy(sysconfig._INSTALL_SCHEMES) def tearDown(self): """Restore sys.path""" @@ -78,11 +83,15 @@ def tearDown(self): site.USER_BASE = self.old_base site.USER_SITE = self.old_site site.PREFIXES = self.old_prefixes + site._VENDOR_SCHEMES = self.old_vendor_schemes sysconfig._CONFIG_VARS = self.original_vars # _CONFIG_VARS is None before get_config_vars() is called if sysconfig._CONFIG_VARS is not None: sysconfig._CONFIG_VARS.clear() sysconfig._CONFIG_VARS.update(self.old_vars) + sysconfig._INSTALL_SCHEMES = self.original_schemes + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(self.old_schemes) def test_makepath(self): # Test makepath() have an absolute path for its first return value @@ -302,6 +311,37 @@ def test_getsitepackages(self): wanted = os.path.join('xoxo', 'lib', 'site-packages') self.assertEqual(dirs[1], wanted) + @with_test_vendor_config() + def test_getsitepackages_vendor(self): + # force re-load of vendor schemes with the patched sys.modules + site._VENDOR_SCHEMES = None + _sysconfig._load_vendor_schemes() + + site.PREFIXES = ['xoxo'] + dirs = site.getsitepackages() + if os.sep == '/': + # OS X, Linux, FreeBSD, etc + if sys.platlibdir != "lib": + self.assertEqual(len(dirs), 2) + wanted = os.path.join('xoxo', sys.platlibdir, + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + self.assertEqual(dirs[0], wanted) + else: + self.assertEqual(len(dirs), 3) + wanted = os.path.join('xoxo', 'lib', + 'python%d.%d' % sys.version_info[:2], + 'site-packages') + self.assertEqual(dirs[-3], wanted) + self.assertEqual(sorted(dirs[-2:]), ['vendor-plat-packages', 'vendor-pure-packages']) + else: + # other platforms + self.assertEqual(len(dirs), 4) + self.assertEqual(dirs[0], 'xoxo') + wanted = os.path.join('xoxo', 'lib', 'site-packages') + self.assertEqual(dirs[1], wanted) + self.assertEqual(sorted(dirs[2:]), ['vendor-plat-packages', 'vendor-pure-packages']) + @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_no_home_directory(self): # bpo-10496: getuserbase() and getusersitepackages() must not fail if diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9408657c918863..bbc3c13976380d 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,21 +1,24 @@ import unittest +import unittest.mock import sys import os import subprocess import shutil from copy import copy -from test.support import (captured_stdout, PythonSymlink) +from test.support import (captured_stdout, with_test_vendor_config, PythonSymlink) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) from test.support.warnings_helper import check_warnings +import _sysconfig import sysconfig +from _sysconfig import _get_preferred_schemes_default, _expand_vars from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + get_preferred_scheme, _main) import _osx_support @@ -43,7 +46,9 @@ def setUp(self): self.join = os.path.join self.isabs = os.path.isabs self.splitdrive = os.path.splitdrive + self._get_preferred_schemes = _sysconfig._get_preferred_schemes self._config_vars = sysconfig._CONFIG_VARS, copy(sysconfig._CONFIG_VARS) + self._schemes = sysconfig._INSTALL_SCHEMES, copy(sysconfig._INSTALL_SCHEMES) self._added_envvars = [] self._changed_envvars = [] for var in ('MACOSX_DEPLOYMENT_TARGET', 'PATH'): @@ -66,9 +71,13 @@ def tearDown(self): os.path.join = self.join os.path.isabs = self.isabs os.path.splitdrive = self.splitdrive + _sysconfig._get_preferred_schemes = self._get_preferred_schemes sysconfig._CONFIG_VARS = self._config_vars[0] sysconfig._CONFIG_VARS.clear() sysconfig._CONFIG_VARS.update(self._config_vars[1]) + sysconfig._INSTALL_SCHEMES = self._schemes[0] + sysconfig._INSTALL_SCHEMES.clear() + sysconfig._INSTALL_SCHEMES.update(self._schemes[1]) for var, value in self._changed_envvars: os.environ[var] = value for var in self._added_envvars: @@ -113,18 +122,26 @@ def test_get_path(self): def test_get_default_scheme(self): self.assertIn(get_default_scheme(), _INSTALL_SCHEMES) - def test_get_preferred_schemes(self): + @with_test_vendor_config() + def test_get_preferred_schemes_vendor(self): + # force re-load of vendor schemes with the patched sys.modules + _sysconfig._get_preferred_schemes = None + _sysconfig._load_vendor_schemes() + + self.assertEqual(get_preferred_scheme('prefix'), 'some_vendor') + + def test_get_preferred_schemes_default(self): expected_schemes = {'prefix', 'home', 'user'} # Windows. os.name = 'nt' - schemes = _get_preferred_schemes() + schemes = _get_preferred_schemes_default() self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) # Mac and Linux, shared library build. os.name = 'posix' - schemes = _get_preferred_schemes() + schemes = _get_preferred_schemes_default() self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) @@ -268,6 +285,16 @@ def test_get_scheme_names(self): wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) self.assertEqual(get_scheme_names(), tuple(sorted(wanted))) + @with_test_vendor_config() + def test_get_scheme_names_vendor(self): + # force re-load of vendor schemes with the patched sys.modules + _sysconfig._load_vendor_schemes() + + wanted = ['nt', 'posix_home', 'posix_prefix', 'some_vendor'] + if HAS_USER_BASE: + wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) + self.assertEqual(sysconfig.get_scheme_names(), tuple(sorted(wanted))) + @skip_unless_symlink def test_symlink(self): # Issue 7880 with PythonSymlink() as py: @@ -376,7 +403,7 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - self.assertTrue(sysconfig._is_python_source_dir(srcdir)) + self.assertTrue(_sysconfig._is_python_source_dir(srcdir)) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already diff --git a/Lib/test/vendor_config.py b/Lib/test/vendor_config.py new file mode 100644 index 00000000000000..0c9f484bd89c06 --- /dev/null +++ b/Lib/test/vendor_config.py @@ -0,0 +1,26 @@ +EXTRA_INSTALL_SCHEMES = { + 'some_vendor': { + 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', + 'include': + '{installed_base}/include/python{py_version_short}{abiflags}', + 'platinclude': + '{installed_platbase}/include/python{py_version_short}{abiflags}', + 'purelib': 'vendor-pure-packages', + 'platlib': 'vendor-plat-packages', + 'scripts': 'vendor-scripts', + 'data': 'vendor-data', + }, +} + +EXTRA_SITE_INSTALL_SCHEMES = [ + 'some_vendor', +] + + +def get_preferred_schemes(): + return { + 'prefix': 'some_vendor', + 'home': 'some_vendor', + 'user': 'some_vendor', + } diff --git a/Makefile.pre.in b/Makefile.pre.in index f03f535f6faa60..4a17e0138dae45 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1517,7 +1517,8 @@ maninstall: altmaninstall # Install the library XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax -LIBSUBDIRS= asyncio \ +LIBSUBDIRS= _vendor \ + asyncio \ collections \ concurrent concurrent/futures \ csv \ @@ -1629,6 +1630,7 @@ TESTSUBDIRS= ctypes/test \ unittest/test unittest/test/testmock TEST_MODULES=@TEST_MODULES@ +VENDOR_CONFIG=@VENDOR_CONFIG@ libinstall: build_all $(srcdir)/Modules/xxmodule.c @for i in $(SCRIPTDIR) $(LIBDEST); \ do \ @@ -1664,6 +1666,9 @@ libinstall: build_all $(srcdir)/Modules/xxmodule.c echo $(INSTALL_DATA) $$i $(LIBDEST); \ fi; \ done + @if test ! -z "$(VENDOR_CONFIG)"; then \ + $(INSTALL_SCRIPT) $(VENDOR_CONFIG) $(DESTDIR)$(LIBDEST)/_vendor.config.py; \ + fi @if test "$(TEST_MODULES)" = yes; then \ subdirs="$(LIBSUBDIRS) $(TESTSUBDIRS)"; \ else \ diff --git a/Misc/NEWS.d/next/Build/2021-04-28-21-57-32.bpo-41282.xgQ6Cn.rst b/Misc/NEWS.d/next/Build/2021-04-28-21-57-32.bpo-41282.xgQ6Cn.rst new file mode 100644 index 00000000000000..3543b0cec115eb --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-04-28-21-57-32.bpo-41282.xgQ6Cn.rst @@ -0,0 +1,2 @@ +Introduced support for Python distributors to specify a vendor config, via +--with-vendor-config, which allows them to add custom install schemes. diff --git a/configure.ac b/configure.ac index ab3fc2839d4f8b..ab9692328db0e0 100644 --- a/configure.ac +++ b/configure.ac @@ -6008,6 +6008,27 @@ else fi AC_SUBST(TEST_MODULES) +# --with-vendor-config +VENDOR_CONFIG='' +AC_MSG_CHECKING(for --with-vendor-config) +AC_ARG_WITH(vendor-config, + AS_HELP_STRING([--with-vendor-config=] + [use a vendor config to customize the certain details of the Python installation]), +[ + AC_CHECK_FILE("$withval", + [ + if ( echo "$withval" | grep '.*\.py$' > /dev/null); then + VENDOR_CONFIG="$withval" + else + AC_MSG_ERROR([--with-vendor-config requires a Python file]) + fi + ], + [ + AC_MSG_ERROR([--with-vendor-config requires a valid file]) + ]) +], +[]) +AC_SUBST(VENDOR_CONFIG) # generate output files AC_CONFIG_FILES(Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh) From 40424fbea3ec3f75bbe24b7ba32376c8cced50bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 18 Oct 2021 21:03:15 +0100 Subject: [PATCH 2/9] update generated files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Python/stdlib_module_names.h | 2 ++ configure | 48 ++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 1743292593f363..601cde7d60c6c4 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -77,6 +77,7 @@ static const char* _Py_stdlib_module_names[] = { "_strptime", "_struct", "_symtable", +"_sysconfig", "_thread", "_threading_local", "_tkinter", @@ -84,6 +85,7 @@ static const char* _Py_stdlib_module_names[] = { "_tracemalloc", "_typing", "_uuid", +"_vendor", "_warnings", "_weakref", "_weakrefset", diff --git a/configure b/configure index 81ee4282d9412b..58ba1c64040f13 100755 --- a/configure +++ b/configure @@ -623,6 +623,7 @@ ac_includes_default="\ #endif" ac_subst_vars='LTLIBOBJS +VENDOR_CONFIG TEST_MODULES LIBRARY_DEPS STATIC_LIBPYTHON @@ -864,6 +865,7 @@ with_builtin_hashlib_hashes with_experimental_isolated_subinterpreters with_static_libpython enable_test_modules +with_vendor_config ' ac_precious_vars='build_alias host_alias @@ -1630,6 +1632,9 @@ Optional Packages: --without-static-libpython do not build libpythonMAJOR.MINOR.a and do not install python.o (default is yes) + --with-vendor-config= + use a vendor config to customize the certain details of the Python installation + Some influential environment variables: MACHDEP name for machine-dependent library files @@ -18106,6 +18111,49 @@ $as_echo "no" >&6; } fi +# --with-vendor-config +VENDOR_CONFIG='' +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-vendor-config" >&5 +$as_echo_n "checking for --with-vendor-config... " >&6; } + +# Check whether --with-vendor-config was given. +if test "${with_vendor_config+set}" = set; then : + withval=$with_vendor_config; + as_ac_File=`$as_echo "ac_cv_file_"$withval"" | $as_tr_sh` +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for \"$withval\"" >&5 +$as_echo_n "checking for \"$withval\"... " >&6; } +if eval \${$as_ac_File+:} false; then : + $as_echo_n "(cached) " >&6 +else + test "$cross_compiling" = yes && + as_fn_error $? "cannot check for file existence when cross compiling" "$LINENO" 5 +if test -r ""$withval""; then + eval "$as_ac_File=yes" +else + eval "$as_ac_File=no" +fi +fi +eval ac_res=\$$as_ac_File + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +if eval test \"x\$"$as_ac_File"\" = x"yes"; then : + + if ( echo "$withval" | grep '.*\.py$' > /dev/null); then + VENDOR_CONFIG="$withval" + else + as_fn_error $? "--with-vendor-config requires a Python file" "$LINENO" 5 + fi + +else + + as_fn_error $? "--with-vendor-config requires a valid file" "$LINENO" 5 + +fi + + +fi + + # generate output files ac_config_files="$ac_config_files Makefile.pre Misc/python.pc Misc/python-embed.pc Misc/python-config.sh" From 4fa5bf0b61b6cc098f444e9638d98cd446c14b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Oct 2021 21:19:35 +0100 Subject: [PATCH 3/9] implement lazy loading of attributes in _sysconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/_sysconfig.py | 338 ++++++++++++++++++------------- Lib/distutils/command/install.py | 3 +- Lib/sysconfig.py | 24 ++- 3 files changed, 215 insertions(+), 150 deletions(-) diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py index 0c0e5c6f138839..35a1432cb35590 100644 --- a/Lib/_sysconfig.py +++ b/Lib/_sysconfig.py @@ -1,7 +1,35 @@ import os import sys -_INSTALL_SCHEMES = { + +# helpers + + +def _safe_realpath(path): + try: + return os.path.realpath(path) + except OSError: + return path + + +def _is_python_source_dir(d): + for fn in ("Setup", "Setup.local"): + if os.path.isfile(os.path.join(d, "Modules", fn)): + return True + return False + + +if os.name == 'nt': + def _fix_pcbuild(d): + if d and os.path.normcase(d).startswith( + os.path.normcase(os.path.join(_PREFIX, "PCbuild"))): + return _PREFIX + return d + + +# constants + +_ALL_INSTALL_SCHEMES = { 'posix_prefix': { 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', @@ -34,27 +62,74 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # userbase schemes + # NOTE: When modifying "purelib" scheme, update site._get_path() too. + 'nt_user': { + 'stdlib': '{userbase}/Python{py_version_nodot_plat}', + 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', + 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/Python{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'data': '{userbase}', + }, + 'posix_user': { + 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', + 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', + 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, + 'osx_framework_user': { + 'stdlib': '{userbase}/lib/python', + 'platstdlib': '{userbase}/lib/python', + 'purelib': '{userbase}/lib/python/site-packages', + 'platlib': '{userbase}/lib/python/site-packages', + 'include': '{userbase}/include/python{py_version_short}', + 'scripts': '{userbase}/bin', + 'data': '{userbase}', + }, } -_get_preferred_schemes = None +_USER_BASE_SCHEME_NAMES = ('nt_user', 'posix_user', 'osx_framework_user') +_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', + 'scripts', 'data') -def _load_vendor_schemes(): - # add vendor defined schemes - try: - import _vendor.config +_PY_VERSION = sys.version.split()[0] +_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' +_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' +_PREFIX = os.path.normpath(sys.prefix) +_BASE_PREFIX = os.path.normpath(sys.base_prefix) +_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) +_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - extra_schemes = _vendor.config.EXTRA_INSTALL_SCHEMES - _INSTALL_SCHEMES.update({ - name: scheme - for name, scheme in extra_schemes.items() - if name not in _INSTALL_SCHEMES - }) - except (ModuleNotFoundError, AttributeError): - pass +_CHEAP_SCHEME_CONFIG_VARS = { + 'prefix': _PREFIX, + 'exec_prefix': _EXEC_PREFIX, + 'py_version': _PY_VERSION, + 'py_version_short': _PY_VERSION_SHORT, + 'py_version_nodot': _PY_VERSION_SHORT_NO_DOT, + 'installed_base': _BASE_PREFIX, + 'base': _PREFIX, + 'installed_platbase': _BASE_EXEC_PREFIX, + 'platbase': _EXEC_PREFIX, + 'platlibdir': sys.platlibdir, + 'abiflags': getattr(sys, 'abiflags', ''), + 'py_version_nodot_plat': getattr(sys, 'winver', '').replace('.', '') +} + +_MODULE = sys.modules[__name__] + +# uninitialized + +_get_preferred_schemes = None -_load_vendor_schemes() + +# lazy constants # NOTE: site.py has copy of this function. @@ -82,131 +157,81 @@ def joinuser(*args): return joinuser("~", ".local") -_HAS_USER_BASE = (_getuserbase() is not None) - -if _HAS_USER_BASE: - _INSTALL_SCHEMES |= { - # NOTE: When modifying "purelib" scheme, update site._get_path() too. - 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', - 'data': '{userbase}', - }, - 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', - 'scripts': '{userbase}/bin', - 'data': '{userbase}', - }, - } - -_SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', - 'scripts', 'data') - -_PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' -_PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) -_BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -_BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) -_USER_BASE = None - - -def _safe_realpath(path): - try: - return os.path.realpath(path) - except OSError: - return path - - -if sys.executable: - _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) -else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - _PROJECT_BASE = _safe_realpath(os.getcwd()) - -if (os.name == 'nt' and - _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, os.path.pardir, os.path.pardir)) - -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) - - -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False - - -_SYS_HOME = getattr(sys, '_home', None) - - -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(_PREFIX, "PCbuild"))): - return _PREFIX - return d - _PROJECT_BASE = _fix_pcbuild(_PROJECT_BASE) - _SYS_HOME = _fix_pcbuild(_SYS_HOME) - - def is_python_build(check_home=False): - if check_home and _SYS_HOME: + if check_home and _MODULE._SYS_HOME: return _is_python_source_dir(_SYS_HOME) - return _is_python_source_dir(getattr(sys.modules[__name__], '_PROJECT_BASE')) - - -_PYTHON_BUILD = is_python_build(True) - -if _PYTHON_BUILD: - for scheme in ('posix_prefix', 'posix_home'): - # On POSIX-y platforms, Python will: - # - Build from .h files in 'headers' (which is only added to the - # scheme when building CPython) - # - Install .h files to 'include' - scheme = _INSTALL_SCHEMES[scheme] - scheme['headers'] = scheme['include'] - scheme['include'] = '{srcdir}/Include' - scheme['platinclude'] = '{projectbase}/.' - + return _is_python_source_dir(_MODULE._PROJECT_BASE) + + +def __getattr__(name): + match name: + case '_HAS_USER_BASE': + value = (_getuserbase() is not None) + case '_PROJECT_BASE': + # set for cross builds + if "_PYTHON_PROJECT_BASE" in os.environ: + value = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"]) + else: + if sys.executable: + value = os.path.dirname(_safe_realpath(sys.executable)) + else: + # sys.executable can be empty if argv[0] has been changed and Python is + # unable to retrieve the real program name + value = _safe_realpath(os.getcwd()) + + if (os.name == 'nt' and + value.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): + value = _safe_realpath(os.path.join(_PROJECT_BASE, os.path.pardir, os.path.pardir)) + + if os.name == 'nt': + value = _fix_pcbuild(value) + case '_SYS_HOME': + value = getattr(sys, '_home', None) + if os.name == 'nt': + value = _fix_pcbuild(value) + case '_PYTHON_BUILD': + value = is_python_build(check_home=True) + case '_SCHEME_CONFIG_VARS': + value = _CHEAP_SCHEME_CONFIG_VARS.copy() + value['projectbase'] = _MODULE._PROJECT_BASE + case _: + raise AttributeError(f"module '{__name__}' has no attribute '{name}'") + setattr(_MODULE, name, value) + return value + + +# methods + + +def _get_raw_scheme_paths(scheme): + # lazy loading of install schemes -- only run the code paths we need to + + # check our schemes + if scheme in _ALL_INSTALL_SCHEMES: + if scheme in _USER_BASE_SCHEME_NAMES and not _MODULE._HAS_USER_BASE: + raise KeyError(repr(scheme)) + + if scheme in ('posix_prefix', 'posix_home') and _MODULE._PYTHON_BUILD: + # On POSIX-y platforms, Python will: + # - Build from .h files in 'headers' (which is only added to the + # scheme when building CPython) + # - Install .h files to 'include' + scheme = _ALL_INSTALL_SCHEMES[scheme] + scheme['headers'] = scheme['include'] + scheme['include'] = '{srcdir}/Include' + scheme['platinclude'] = '{projectbase}/.' + return scheme + + return _ALL_INSTALL_SCHEMES[scheme] + + # check vendor schemes + try: + import _vendor.config -_SCHEME_CONFIG_VARS = { - 'prefix': _PREFIX, - 'exec_prefix': _EXEC_PREFIX, - 'py_version': _PY_VERSION, - 'py_version_short': _PY_VERSION_SHORT, - 'py_version_nodot': _PY_VERSION_SHORT_NO_DOT, - 'installed_base': _BASE_PREFIX, - 'base': _PREFIX, - 'installed_platbase': _BASE_EXEC_PREFIX, - 'platbase': _EXEC_PREFIX, - 'projectbase': _PROJECT_BASE, - 'platlibdir': sys.platlibdir, - 'abiflags': getattr(sys, 'abiflags', ''), - 'py_version_nodot_plat': getattr(sys, 'winver', '').replace('.', '') -} + vendor_schemes = _vendor.config.EXTRA_INSTALL_SCHEMES + except (ModuleNotFoundError, AttributeError): + pass + return vendor_schemes[scheme] def _subst_vars(s, local_vars): @@ -221,11 +246,18 @@ def _subst_vars(s, local_vars): def _expand_vars(scheme, vars): if vars is None: - vars = _SCHEME_CONFIG_VARS + vars = {} + vars.update(_CHEAP_SCHEME_CONFIG_VARS) + res = {} - for key, value in _INSTALL_SCHEMES[scheme].items(): + for key, value in _get_raw_scheme_paths(scheme).items(): if os.name in ('posix', 'nt'): value = os.path.expanduser(value) + + # this is an expensive and uncommon config var, let's only load it if we need to + if '{projectbase}' in value: + vars['projectbase'] = _MODULE._PROJECT_BASE + res[key] = os.path.normpath(_subst_vars(value, vars)) return res @@ -261,12 +293,26 @@ def get_preferred_scheme(key): _get_preferred_schemes = _get_preferred_schemes_default scheme = _get_preferred_schemes()[key] - if scheme not in _INSTALL_SCHEMES: - raise ValueError( - f"{key!r} returned {scheme!r}, which is not a valid scheme " - f"on this platform" - ) - return scheme + + # check out schemes + if scheme in _ALL_INSTALL_SCHEMES: + return scheme + + # check vendor schemes + try: + import _vendor.config + + vendor_schemes = _vendor.config.EXTRA_INSTALL_SCHEMES + except (ModuleNotFoundError, AttributeError): + pass + else: + if scheme in vendor_schemes: + return scheme + + raise ValueError( + f"{key!r} returned {scheme!r}, which is not a valid scheme " + f"on this platform" + ) def get_default_scheme(): @@ -282,4 +328,4 @@ def _get_paths(scheme=get_default_scheme(), vars=None, expand=True): if expand: return _expand_vars(scheme, vars) else: - return _INSTALL_SCHEMES[scheme] + return _get_raw_scheme_paths(scheme) diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index b31304b4c91633..01d5331a63069b 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -3,7 +3,6 @@ Implements the Distutils 'install' command.""" import sys -import _sysconfig import sysconfig import os import re @@ -42,7 +41,7 @@ ("unix_prefix", "posix_prefix"), ("unix_home", "posix_home"), ("nt", "nt")): sys_key = key - sys_scheme = _sysconfig._INSTALL_SCHEMES[sys_scheme_name] + sys_scheme = sysconfig._INSTALL_SCHEMES[sys_scheme_name] if key == "headers" and key not in sys_scheme: # On POSIX-y platforms, Python will: # - Build from .h files in 'headers' (only there when diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index df5420f0c1e35a..51ef39ff7128c1 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -5,8 +5,8 @@ from _sysconfig import ( _get_paths, _getuserbase, _safe_realpath, get_default_scheme, get_preferred_scheme, is_python_build, - _HAS_USER_BASE, _INSTALL_SCHEMES, _PROJECT_BASE, - _PYTHON_BUILD, _PY_VERSION_SHORT, _PY_VERSION_SHORT_NO_DOT, + _HAS_USER_BASE, _ALL_INSTALL_SCHEMES, _USER_BASE_SCHEME_NAMES, + _PROJECT_BASE, _PYTHON_BUILD, _PY_VERSION_SHORT, _PY_VERSION_SHORT_NO_DOT, _SCHEME_CONFIG_VARS, _SCHEME_KEYS, _SYS_HOME, ) @@ -31,6 +31,26 @@ 'MACOSX_DEPLOYMENT_TARGET', } +_INSTALL_SCHEMES = { + key: value + for key, value in _ALL_INSTALL_SCHEMES.items() + if key not in _USER_BASE_SCHEME_NAMES +} + + +def _load_vendor_schemes(): + global _INSTALL_SCHEMES + try: + import _vendor.config + + _INSTALL_SCHEMES = _INSTALL_SCHEMES | _vendor.config.EXTRA_INSTALL_SCHEMES + except (ModuleNotFoundError, AttributeError): + pass + + +_load_vendor_schemes() + + _CONFIG_VARS = None # Regexes needed for parsing Makefile (and similar syntaxes, From a4c21140789cd90659b34ccb34e1f09947f6e220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Oct 2021 23:38:12 +0100 Subject: [PATCH 4/9] note _sysconfig/sysconfig split in the module docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/_sysconfig.py | 2 ++ Lib/sysconfig.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py index 35a1432cb35590..e163989fee8ba0 100644 --- a/Lib/_sysconfig.py +++ b/Lib/_sysconfig.py @@ -1,3 +1,5 @@ +"""Bits of sysconfig that are required at startup (by the site module)""" + import os import sys diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 51ef39ff7128c1..9f42d90453c939 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -1,4 +1,8 @@ -"""Access to Python's configuration information.""" +"""Access to Python's configuration information. + +This module is split into _sysconfig, with the bits that are required at startup +(by the site module), and sysconfig, with the rest of the module functionality. +""" import os import sys From 31049fd658e4f6b3fe727dc11e6cc663ec924fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Oct 2021 23:49:54 +0100 Subject: [PATCH 5/9] fix _PROJECT_BASE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/_sysconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py index e163989fee8ba0..b25db36bc64404 100644 --- a/Lib/_sysconfig.py +++ b/Lib/_sysconfig.py @@ -183,7 +183,7 @@ def __getattr__(name): if (os.name == 'nt' and value.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - value = _safe_realpath(os.path.join(_PROJECT_BASE, os.path.pardir, os.path.pardir)) + value = _safe_realpath(os.path.join(value, os.path.pardir, os.path.pardir)) if os.name == 'nt': value = _fix_pcbuild(value) From dd1c61a3c53899c85389ea5a9c8bba20d2391f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 20 Oct 2021 01:07:53 +0100 Subject: [PATCH 6/9] forgot to add this to the main commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I fixed the main commit in another branch, but forgot I am no longer rebasing here .-. Signed-off-by: Filipe Laíns --- Lib/_sysconfig.py | 77 ++++++++++++++++++++++++++++-------- Lib/sysconfig.py | 51 ++++++++---------------- Lib/test/support/__init__.py | 2 + Lib/test/test_site.py | 3 +- Lib/test/test_sysconfig.py | 21 ++++------ 5 files changed, 89 insertions(+), 65 deletions(-) diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py index b25db36bc64404..0caf5940505859 100644 --- a/Lib/_sysconfig.py +++ b/Lib/_sysconfig.py @@ -31,7 +31,7 @@ def _fix_pcbuild(d): # constants -_ALL_INSTALL_SCHEMES = { +_BASE_INSTALL_SCHEMES = { 'posix_prefix': { 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', @@ -64,7 +64,8 @@ def _fix_pcbuild(d): 'scripts': '{base}/Scripts', 'data': '{base}', }, - # userbase schemes + } +_USER_INSTALL_SCHEMES = { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { 'stdlib': '{userbase}/Python{py_version_nodot_plat}', @@ -95,8 +96,6 @@ def _fix_pcbuild(d): }, } -_USER_BASE_SCHEME_NAMES = ('nt_user', 'posix_user', 'osx_framework_user') - _SCHEME_KEYS = ('stdlib', 'platstdlib', 'purelib', 'platlib', 'include', 'scripts', 'data') @@ -108,7 +107,6 @@ def _fix_pcbuild(d): _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - _CHEAP_SCHEME_CONFIG_VARS = { 'prefix': _PREFIX, 'exec_prefix': _EXEC_PREFIX, @@ -167,6 +165,9 @@ def is_python_build(check_home=False): def __getattr__(name): match name: + case '_BUILD_TIME_VARS': + _sysconfigdata = __import__(_get_sysconfigdata_name(), globals(), locals(), ['build_time_vars'], 0) + value = _sysconfigdata.build_time_vars case '_HAS_USER_BASE': value = (_getuserbase() is not None) case '_PROJECT_BASE': @@ -183,10 +184,31 @@ def __getattr__(name): if (os.name == 'nt' and value.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - value = _safe_realpath(os.path.join(value, os.path.pardir, os.path.pardir)) + value = _safe_realpath(os.path.join(_PROJECT_BASE, os.path.pardir, os.path.pardir)) if os.name == 'nt': value = _fix_pcbuild(value) + case '_SRCDIR': + if os.name == 'posix' and not _MODULE._PYTHON_BUILD: + # srcdir is not meaningful since the installation is + # spread about the filesystem. We choose the + # directory containing the Makefile since we know it + # exists. + value = os.path.dirname(get_makefile_filename()) + else: + if 'srcdir' in _MODULE._BUILD_TIME_VARS: + value = _MODULE._BUILD_TIME_VARS['srcdir'] + else: + value = _MODULE._PROJECT_BASE + + if os.name == 'posix' and _MODULE._PYTHON_BUILD: + # If srcdir is a relative path (typically '.' or '..') + # then it should be interpreted relative to the directory + # containing Makefile. + base = os.path.dirname(get_makefile_filename()) + value = os.path.join(base, value) + + value = _safe_realpath(value) case '_SYS_HOME': value = getattr(sys, '_home', None) if os.name == 'nt': @@ -208,23 +230,22 @@ def __getattr__(name): def _get_raw_scheme_paths(scheme): # lazy loading of install schemes -- only run the code paths we need to - # check our schemes - if scheme in _ALL_INSTALL_SCHEMES: - if scheme in _USER_BASE_SCHEME_NAMES and not _MODULE._HAS_USER_BASE: - raise KeyError(repr(scheme)) - + # check base schemes + if scheme in _BASE_INSTALL_SCHEMES: if scheme in ('posix_prefix', 'posix_home') and _MODULE._PYTHON_BUILD: # On POSIX-y platforms, Python will: # - Build from .h files in 'headers' (which is only added to the # scheme when building CPython) # - Install .h files to 'include' - scheme = _ALL_INSTALL_SCHEMES[scheme] + scheme = _BASE_INSTALL_SCHEMES[scheme] scheme['headers'] = scheme['include'] scheme['include'] = '{srcdir}/Include' scheme['platinclude'] = '{projectbase}/.' return scheme + return _BASE_INSTALL_SCHEMES[scheme] - return _ALL_INSTALL_SCHEMES[scheme] + if scheme in _USER_INSTALL_SCHEMES and _MODULE._HAS_USER_BASE: + return _USER_INSTALL_SCHEMES[scheme] # check vendor schemes try: @@ -236,6 +257,28 @@ def _get_raw_scheme_paths(scheme): return vendor_schemes[scheme] +def _get_sysconfigdata_name(): + multiarch = getattr(sys.implementation, '_multiarch', '') + return os.environ.get( + '_PYTHON_SYSCONFIGDATA_NAME', + f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', + ) + + +def get_makefile_filename(): + """Return the path of the Makefile.""" + if _MODULE._PYTHON_BUILD: + return os.path.join(_MODULE._SYS_HOME or _MODULE._PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): + config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' + else: + config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): + config_dir_name += f'-{sys.implementation._multiarch}' + stdlib = _get_paths(get_default_scheme())['stdlib'] + return os.path.join(stdlib, config_dir_name, 'Makefile') + + def _subst_vars(s, local_vars): try: return s.format(**local_vars) @@ -256,9 +299,11 @@ def _expand_vars(scheme, vars): if os.name in ('posix', 'nt'): value = os.path.expanduser(value) - # this is an expensive and uncommon config var, let's only load it if we need to + # these are an expensive and uncommon config vars, let's only load them if we need to if '{projectbase}' in value: vars['projectbase'] = _MODULE._PROJECT_BASE + if '{srcdir}' in value: + vars['srcdir'] = _MODULE._SRCDIR res[key] = os.path.normpath(_subst_vars(value, vars)) return res @@ -296,8 +341,8 @@ def get_preferred_scheme(key): scheme = _get_preferred_schemes()[key] - # check out schemes - if scheme in _ALL_INSTALL_SCHEMES: + # check our schemes + if scheme in _BASE_INSTALL_SCHEMES or scheme in _USER_INSTALL_SCHEMES: return scheme # check vendor schemes diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 9f42d90453c939..ec228547b2af41 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -7,11 +7,11 @@ import os import sys from _sysconfig import ( - _get_paths, _getuserbase, _safe_realpath, - get_default_scheme, get_preferred_scheme, is_python_build, - _HAS_USER_BASE, _ALL_INSTALL_SCHEMES, _USER_BASE_SCHEME_NAMES, - _PROJECT_BASE, _PYTHON_BUILD, _PY_VERSION_SHORT, _PY_VERSION_SHORT_NO_DOT, - _SCHEME_CONFIG_VARS, _SCHEME_KEYS, _SYS_HOME, + _get_paths, _getuserbase, _get_sysconfigdata_name, _safe_realpath, + get_default_scheme, get_preferred_scheme, get_makefile_filename, + is_python_build, _HAS_USER_BASE, _BASE_INSTALL_SCHEMES, + _USER_INSTALL_SCHEMES, _PROJECT_BASE, _PYTHON_BUILD, _PY_VERSION_SHORT, + _PY_VERSION_SHORT_NO_DOT, _SCHEME_CONFIG_VARS, _SCHEME_KEYS, _SYS_HOME, ) __all__ = [ @@ -35,24 +35,28 @@ 'MACOSX_DEPLOYMENT_TARGET', } -_INSTALL_SCHEMES = { - key: value - for key, value in _ALL_INSTALL_SCHEMES.items() - if key not in _USER_BASE_SCHEME_NAMES -} +_INSTALL_SCHEMES = None -def _load_vendor_schemes(): +def _reload_schemes(): global _INSTALL_SCHEMES + + # our schemes + _INSTALL_SCHEMES = _BASE_INSTALL_SCHEMES.copy() + if _HAS_USER_BASE: + _INSTALL_SCHEMES |= _USER_INSTALL_SCHEMES + + # vendor schemes try: import _vendor.config - _INSTALL_SCHEMES = _INSTALL_SCHEMES | _vendor.config.EXTRA_INSTALL_SCHEMES + # make sure we do not let the vendor install schemes override ours + _INSTALL_SCHEMES = _vendor.config.EXTRA_INSTALL_SCHEMES | _INSTALL_SCHEMES except (ModuleNotFoundError, AttributeError): pass -_load_vendor_schemes() +_reload_schemes() _CONFIG_VARS = None @@ -192,27 +196,6 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): return vars -def get_makefile_filename(): - """Return the path of the Makefile.""" - if _PYTHON_BUILD: - return os.path.join(_SYS_HOME or _PROJECT_BASE, "Makefile") - if hasattr(sys, 'abiflags'): - config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' - else: - config_dir_name = 'config' - if hasattr(sys.implementation, '_multiarch'): - config_dir_name += f'-{sys.implementation._multiarch}' - return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') - - -def _get_sysconfigdata_name(): - multiarch = getattr(sys.implementation, '_multiarch', '') - return os.environ.get( - '_PYTHON_SYSCONFIGDATA_NAME', - f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', - ) - - def _generate_posix_vars(): """Generate the Python module containing build-time variables.""" import pprint diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e299f630c347be..1ce6ecda67f61f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2108,6 +2108,7 @@ def with_test_vendor_config(): import _vendor _vendor.config = test.vendor_config + sysconfig._reload_schemes() yield @@ -2115,3 +2116,4 @@ def with_test_vendor_config(): _vendor.config = old_config else: delattr(_vendor, 'config') + sysconfig._reload_schemes() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 1ca111a08266c3..f2dfa451272ab5 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -313,9 +313,8 @@ def test_getsitepackages(self): @with_test_vendor_config() def test_getsitepackages_vendor(self): - # force re-load of vendor schemes with the patched sys.modules + # force re-load of vendor schemes site._VENDOR_SCHEMES = None - _sysconfig._load_vendor_schemes() site.PREFIXES = ['xoxo'] dirs = site.getsitepackages() diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index bbc3c13976380d..55038540da3593 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -16,7 +16,7 @@ import sysconfig from _sysconfig import _get_preferred_schemes_default, _expand_vars from sysconfig import (get_paths, get_platform, get_config_vars, - get_path, get_path_names, _INSTALL_SCHEMES, + get_path, get_path_names, _INSTALL_SCHEMES, _SCHEME_KEYS, get_default_scheme, get_scheme_names, get_config_var, get_preferred_scheme, _main) import _osx_support @@ -112,7 +112,7 @@ def test_get_paths(self): def test_get_path(self): config_vars = get_config_vars() for scheme in _INSTALL_SCHEMES: - for name in _INSTALL_SCHEMES[scheme]: + for name in _SCHEME_KEYS: expected = _INSTALL_SCHEMES[scheme][name].format(**config_vars) self.assertEqual( os.path.normpath(get_path(name, scheme)), @@ -124,9 +124,7 @@ def test_get_default_scheme(self): @with_test_vendor_config() def test_get_preferred_schemes_vendor(self): - # force re-load of vendor schemes with the patched sys.modules _sysconfig._get_preferred_schemes = None - _sysconfig._load_vendor_schemes() self.assertEqual(get_preferred_scheme('prefix'), 'some_vendor') @@ -280,20 +278,17 @@ def test_get_config_h_filename(self): self.assertTrue(os.path.isfile(config_h), config_h) def test_get_scheme_names(self): - wanted = ['nt', 'posix_home', 'posix_prefix'] + wanted = {'nt', 'posix_home', 'posix_prefix'} if HAS_USER_BASE: - wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) - self.assertEqual(get_scheme_names(), tuple(sorted(wanted))) + wanted |= {'nt_user', 'osx_framework_user', 'posix_user'} + self.assertEqual(set(get_scheme_names()), wanted) @with_test_vendor_config() def test_get_scheme_names_vendor(self): - # force re-load of vendor schemes with the patched sys.modules - _sysconfig._load_vendor_schemes() - - wanted = ['nt', 'posix_home', 'posix_prefix', 'some_vendor'] + wanted = {'nt', 'posix_home', 'posix_prefix', 'some_vendor'} if HAS_USER_BASE: - wanted.extend(['nt_user', 'osx_framework_user', 'posix_user']) - self.assertEqual(sysconfig.get_scheme_names(), tuple(sorted(wanted))) + wanted |= {'nt_user', 'osx_framework_user', 'posix_user'} + self.assertEqual(set(sysconfig.get_scheme_names()), wanted) @skip_unless_symlink def test_symlink(self): # Issue 7880 From 12f97ae8b4a2c149c0d7301b5157a85b208ee966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 20 Oct 2021 01:37:01 +0100 Subject: [PATCH 7/9] fix _PROJECT_BASE again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/_sysconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py index 0caf5940505859..f7124124aab7bc 100644 --- a/Lib/_sysconfig.py +++ b/Lib/_sysconfig.py @@ -184,7 +184,7 @@ def __getattr__(name): if (os.name == 'nt' and value.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))): - value = _safe_realpath(os.path.join(_PROJECT_BASE, os.path.pardir, os.path.pardir)) + value = _safe_realpath(os.path.join(value, os.path.pardir, os.path.pardir)) if os.name == 'nt': value = _fix_pcbuild(value) From 57c91c2c39d4eb181738ad68df52606552147683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Wed, 20 Oct 2021 02:32:42 +0100 Subject: [PATCH 8/9] fix _SRCDIR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/_sysconfig.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Lib/_sysconfig.py b/Lib/_sysconfig.py index f7124124aab7bc..f066849e155216 100644 --- a/Lib/_sysconfig.py +++ b/Lib/_sysconfig.py @@ -189,25 +189,25 @@ def __getattr__(name): if os.name == 'nt': value = _fix_pcbuild(value) case '_SRCDIR': - if os.name == 'posix' and not _MODULE._PYTHON_BUILD: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - value = os.path.dirname(get_makefile_filename()) - else: - if 'srcdir' in _MODULE._BUILD_TIME_VARS: - value = _MODULE._BUILD_TIME_VARS['srcdir'] - else: - value = _MODULE._PROJECT_BASE - - if os.name == 'posix' and _MODULE._PYTHON_BUILD: + if os.name == 'posix': + if _MODULE._PYTHON_BUILD: # If srcdir is a relative path (typically '.' or '..') # then it should be interpreted relative to the directory # containing Makefile. + if 'srcdir' in _MODULE._BUILD_TIME_VARS: + value = _MODULE._BUILD_TIME_VARS['srcdir'] + else: + value = _MODULE._PROJECT_BASE base = os.path.dirname(get_makefile_filename()) value = os.path.join(base, value) - + else: + # srcdir is not meaningful since the installation is + # spread about the filesystem. We choose the + # directory containing the Makefile since we know it + # exists. + value = os.path.dirname(get_makefile_filename()) + else: + value = _MODULE._PROJECT_BASE value = _safe_realpath(value) case '_SYS_HOME': value = getattr(sys, '_home', None) From 3e25a85fc5e931376b965c44a3621f37c9d196aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Oct 2021 23:47:15 +0100 Subject: [PATCH 9/9] freeze _sysconfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Makefile.pre.in | 5 +++++ PCbuild/_freeze_module.vcxproj | 5 +++++ PCbuild/_freeze_module.vcxproj.filters | 3 +++ Python/frozen.c | 2 ++ Tools/scripts/freeze_modules.py | 1 + 5 files changed, 16 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 4a17e0138dae45..d7e888b6622720 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -746,6 +746,7 @@ FROZEN_FILES_IN = \ Lib/io.py \ Lib/_collections_abc.py \ Lib/_sitebuiltins.py \ + Lib/_sysconfig.py \ Lib/genericpath.py \ Lib/ntpath.py \ Lib/posixpath.py \ @@ -768,6 +769,7 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/io.h \ Python/frozen_modules/_collections_abc.h \ Python/frozen_modules/_sitebuiltins.h \ + Python/frozen_modules/_sysconfig.h \ Python/frozen_modules/genericpath.h \ Python/frozen_modules/ntpath.h \ Python/frozen_modules/posixpath.h \ @@ -813,6 +815,9 @@ Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collectio Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h +Python/frozen_modules/_sysconfig.h: Programs/_freeze_module Lib/_sysconfig.py + Programs/_freeze_module _sysconfig $(srcdir)/Lib/_sysconfig.py $(srcdir)/Python/frozen_modules/_sysconfig.h + Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 12bdde2af84d9a..ed4ee78b61681f 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -270,6 +270,11 @@ $(IntDir)_sitebuiltins.g.h $(PySourcePath)Python\frozen_modules\_sitebuiltins.h + + _sysconfig + $(IntDir)_sysconfig.g.h + $(PySourcePath)Python\frozen_modules\_sysconfig.h + genericpath $(IntDir)genericpath.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5894909e0fbe1e..4328c6a9f73a38 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -40,6 +40,9 @@ Python Files + + Python Files + Python Files diff --git a/Python/frozen.c b/Python/frozen.c index 499b3b99570573..92a8ea375201db 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -47,6 +47,7 @@ #include "frozen_modules/io.h" #include "frozen_modules/_collections_abc.h" #include "frozen_modules/_sitebuiltins.h" +#include "frozen_modules/_sysconfig.h" #include "frozen_modules/genericpath.h" #include "frozen_modules/ntpath.h" #include "frozen_modules/posixpath.h" @@ -80,6 +81,7 @@ static const struct _frozen _PyImport_FrozenModules[] = { {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc)}, {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)}, + {"_sysconfig", _Py_M___sysconfig, (int)sizeof(_Py_M___sysconfig)}, {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)}, {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)}, {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)}, diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 5c7eee42952896..0dff681fdf1a0d 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -84,6 +84,7 @@ def find_tool(): ('stdlib - startup, with site', [ '_collections_abc', '_sitebuiltins', + '_sysconfig', 'genericpath', 'ntpath', 'posixpath',