From 1380972482b36a8ec2ae681998ce063ff508f95e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Mon, 18 Nov 2024 23:43:18 +0000 Subject: [PATCH 01/13] GH-126985: move pyconf.cfg detection from site to getpath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_getpath.py | 32 ++++++++--------- ...-11-18-23-42-06.gh-issue-126985.7XplY9.rst | 3 ++ Modules/getpath.py | 36 +++++++++++-------- 3 files changed, 41 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index d5dcdad9614ecc..7e5c4a3d14ddc5 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -92,8 +92,8 @@ def test_venv_win32(self): ]) expected = dict( executable=r"C:\venv\Scripts\python.exe", - prefix=r"C:\Python", - exec_prefix=r"C:\Python", + prefix=r"C:\venv", + exec_prefix=r"C:\venv", base_executable=r"C:\Python\python.exe", base_prefix=r"C:\Python", base_exec_prefix=r"C:\Python", @@ -339,8 +339,8 @@ def test_venv_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python", base_prefix="/usr", base_exec_prefix="/usr", @@ -371,8 +371,8 @@ def test_venv_changed_name_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python3", base_prefix="/usr", base_exec_prefix="/usr", @@ -404,8 +404,8 @@ def test_venv_non_installed_zip_path_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/path/to/non-installed", - exec_prefix="/path/to/non-installed", + prefix="/venv", + exec_prefix="/venv", base_executable="/path/to/non-installed/bin/python", base_prefix="/path/to/non-installed", base_exec_prefix="/path/to/non-installed", @@ -435,8 +435,8 @@ def test_venv_changed_name_copy_posix(self): ]) expected = dict( executable="/venv/bin/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/venv", + exec_prefix="/venv", base_executable="/usr/bin/python9", base_prefix="/usr", base_exec_prefix="/usr", @@ -652,8 +652,8 @@ def test_venv_framework_macos(self): ]) expected = dict( executable=f"{venv_path}/bin/python", - prefix="/Library/Frameworks/Python.framework/Versions/9.8", - exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + prefix=venv_path, + exec_prefix=venv_path, base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", base_prefix="/Library/Frameworks/Python.framework/Versions/9.8", base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", @@ -697,8 +697,8 @@ def test_venv_alt_framework_macos(self): ]) expected = dict( executable=f"{venv_path}/bin/python", - prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", - exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + prefix=venv_path, + exec_prefix=venv_path, base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", @@ -734,8 +734,8 @@ def test_venv_macos(self): ]) expected = dict( executable="/framework/Python9.8/python", - prefix="/usr", - exec_prefix="/usr", + prefix="/framework/Python9.8", + exec_prefix="/framework/Python9.8", base_executable="/usr/bin/python", base_prefix="/usr", base_exec_prefix="/usr", diff --git a/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst b/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst new file mode 100644 index 00000000000000..c875c7b547bba9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-23-42-06.gh-issue-126985.7XplY9.rst @@ -0,0 +1,3 @@ +When running under a virtual environment with the :mod:`site` disabled (see +:option:`-S`), :data:`sys.prefix` and :data:`sys.base_prefix` will now point +to the virtual environment, instead of the base installation. diff --git a/Modules/getpath.py b/Modules/getpath.py index 1f1bfcb4f64dd4..1ad467c79102b5 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -640,11 +640,18 @@ def search_up(prefix, *landmarks, test=isfile): # For a venv, update the main prefix/exec_prefix but leave the base ones unchanged -# XXX: We currently do not update prefix here, but it happens in site.py -#if venv_prefix: -# base_prefix = prefix -# base_exec_prefix = exec_prefix -# prefix = exec_prefix = venv_prefix +if venv_prefix: + base_prefix = prefix + base_exec_prefix = exec_prefix + prefix = exec_prefix = venv_prefix + + +# After calculating prefix and exec_prefix, use their values for base_prefix and +# base_exec_prefix if they haven't been set. +if not base_prefix: + base_prefix = prefix +if not base_exec_prefix: + base_exec_prefix = exec_prefix # ****************************************************************************** @@ -679,7 +686,7 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: POSIX uses the default prefix when in the build directory pythonpath.append(joinpath(PREFIX, ZIP_LANDMARK)) else: - pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) + pythonpath.append(joinpath(base_prefix, ZIP_LANDMARK)) if os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored @@ -714,13 +721,13 @@ def search_up(prefix, *landmarks, test=isfile): # Then add any entries compiled into the PYTHONPATH macro. if PYTHONPATH: for p in PYTHONPATH.split(DELIM): - pythonpath.append(joinpath(prefix, p)) + pythonpath.append(joinpath(base_prefix, p)) # Then add stdlib_dir and platstdlib_dir - if not stdlib_dir and prefix: - stdlib_dir = joinpath(prefix, STDLIB_SUBDIR) - if not platstdlib_dir and exec_prefix: - platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) + if not stdlib_dir and base_prefix: + stdlib_dir = joinpath(base_prefix, STDLIB_SUBDIR) + if not platstdlib_dir and base_exec_prefix: + platstdlib_dir = joinpath(base_exec_prefix, PLATSTDLIB_LANDMARK) if os_name == 'nt': # QUIRK: Windows generates paths differently @@ -750,7 +757,8 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. -if os_name != 'nt' and build_prefix: +# Virtual environments using the build directory Python still keep their prefix. +if not venv_prefix and os_name != 'nt' and build_prefix: prefix = config.get('prefix') or PREFIX exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix @@ -788,8 +796,8 @@ def search_up(prefix, *landmarks, test=isfile): config['base_executable'] = base_executable config['prefix'] = prefix config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['base_prefix'] = base_prefix +config['base_exec_prefix'] = base_exec_prefix config['platlibdir'] = platlibdir # test_embed expects empty strings, not None From 9cc5c44ce2f4ad864361f615302b8658499f1609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 00:02:25 +0000 Subject: [PATCH 02/13] Revert "GH-126789: fix some sysconfig data on late site initializations" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit acbd5c9c6c62dac34d2ed1a789d36fe61841c16d. Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 18 ++--- Lib/test/support/venv.py | 70 ------------------- Lib/test/test_sysconfig.py | 67 +----------------- ...-11-13-22-25-57.gh-issue-126789.lKzlc7.rst | 4 -- 4 files changed, 5 insertions(+), 154 deletions(-) delete mode 100644 Lib/test/support/venv.py delete mode 100644 Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 67a071963d8c7d..ee52700b51fd07 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -173,7 +173,9 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' +_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) +_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Mutex guarding initialization of _CONFIG_VARS. _CONFIG_VARS_LOCK = threading.RLock() @@ -465,10 +467,8 @@ def _init_config_vars(): # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _PREFIX = os.path.normpath(sys.prefix) - _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) - _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['prefix'] = _PREFIX + _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT @@ -541,7 +541,6 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ - global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. if not _CONFIG_VARS_INITIALIZED: @@ -552,15 +551,6 @@ def get_config_vars(*args): # don't re-enter init_config_vars(). if _CONFIG_VARS is None: _init_config_vars() - else: - # If the site module initialization happened after _CONFIG_VARS was - # initialized, a virtual environment might have been activated, resulting in - # variables like sys.prefix changing their value, so we need to re-init the - # config vars (see GH-126789). - if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): - with _CONFIG_VARS_LOCK: - _CONFIG_VARS_INITIALIZED = False - _init_config_vars() if args: vals = [] diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py deleted file mode 100644 index 78e6a51ec1815e..00000000000000 --- a/Lib/test/support/venv.py +++ /dev/null @@ -1,70 +0,0 @@ -import contextlib -import logging -import os -import subprocess -import shlex -import sys -import sysconfig -import tempfile -import venv - - -class VirtualEnvironment: - def __init__(self, prefix, **venv_create_args): - self._logger = logging.getLogger(self.__class__.__name__) - venv.create(prefix, **venv_create_args) - self._prefix = prefix - self._paths = sysconfig.get_paths( - scheme='venv', - vars={'base': self.prefix}, - expand=True, - ) - - @classmethod - @contextlib.contextmanager - def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args): - delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV')) - with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir: - yield cls(tmpdir, **venv_create_args) - - @property - def prefix(self): - return self._prefix - - @property - def paths(self): - return self._paths - - @property - def interpreter(self): - return os.path.join(self.paths['scripts'], os.path.basename(sys.executable)) - - def _format_output(self, name, data, indent='\t'): - if not data: - return indent + f'{name}: (none)' - if len(data.splitlines()) == 1: - return indent + f'{name}: {data}' - else: - prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines()) - return indent + f'{name}:\n' + prefixed_lines - - def run(self, *args, **subprocess_args): - if subprocess_args.get('shell'): - raise ValueError('Running the subprocess in shell mode is not supported.') - default_args = { - 'capture_output': True, - 'check': True, - } - try: - result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args) - except subprocess.CalledProcessError as e: - if e.returncode != 0: - self._logger.error( - f'Interpreter returned non-zero exit status {e.returncode}.\n' - + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n' - + self._format_output('STDOUT', e.stdout.decode()) + '\n' - + self._format_output('STDERR', e.stderr.decode()) + '\n' - ) - raise - else: - return result diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 9bbf8d0c6cf2da..a705dd0cd89e4d 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -110,6 +110,7 @@ def venv(self, **venv_create_args): **venv_create_args, ) + def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -591,71 +592,6 @@ def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) - @requires_subprocess() - def test_config_vars_depend_on_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig - - config_vars = sysconfig.get_config_vars() - - import json - print(json.dumps(config_vars, indent=2)) - """) - - with self.venv() as venv: - site_config_vars = json.loads(venv.run('-c', script).stdout) - no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(site_config_vars, no_site_config_vars) - # With the site initialization, the virtual environment should be enabled. - self.assertEqual(site_config_vars['base'], venv.prefix) - self.assertEqual(site_config_vars['platbase'], venv.prefix) - #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix - # Without the site initialization, the virtual environment should be disabled. - self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) - self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) - - @requires_subprocess() - def test_config_vars_recalculation_after_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig - - before = sysconfig.get_config_vars() - - import site - site.main() - - after = sysconfig.get_config_vars() - - import json - print(json.dumps({'before': before, 'after': after}, indent=2)) - """) - - with self.venv() as venv: - config_vars = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(config_vars['before'], config_vars['after']) - self.assertEqual(config_vars['after']['base'], venv.prefix) - #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix - #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix - - @requires_subprocess() - def test_paths_depend_on_site_initialization(self): - script = textwrap.dedent(""" - import sysconfig - - paths = sysconfig.get_paths() - - import json - print(json.dumps(paths, indent=2)) - """) - - with self.venv() as venv: - site_paths = json.loads(venv.run('-c', script).stdout) - no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) - - self.assertNotEqual(site_paths, no_site_paths) - @requires_subprocess() def test_makefile_overwrites_config_vars(self): script = textwrap.dedent(""" @@ -689,7 +625,6 @@ def test_makefile_overwrites_config_vars(self): self.assertNotEqual(data['prefix'], data['base_prefix']) self.assertNotEqual(data['exec_prefix'], data['base_exec_prefix']) - class MakefileTests(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), diff --git a/Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst b/Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst deleted file mode 100644 index 09d4d2e5ab9037..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-13-22-25-57.gh-issue-126789.lKzlc7.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fixed the values of :py:func:`sysconfig.get_config_vars`, -:py:func:`sysconfig.get_paths`, and their siblings when the :py:mod:`site` -initialization happens after :py:mod:`sysconfig` has built a cache for -:py:func:`sysconfig.get_config_vars`. From c6b638bae08e675245ffb2c45fe43863bbc8792c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 00:56:38 +0000 Subject: [PATCH 03/13] Fix test_embed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Modules/getpath.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 1ad467c79102b5..7f3960326c2ec3 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -641,9 +641,12 @@ def search_up(prefix, *landmarks, test=isfile): # For a venv, update the main prefix/exec_prefix but leave the base ones unchanged if venv_prefix: - base_prefix = prefix - base_exec_prefix = exec_prefix - prefix = exec_prefix = venv_prefix + if not prefix: + base_prefix = prefix + prefix = venv_prefix + if not exec_prefix: + base_exec_prefix = exec_prefix + exec_prefix = venv_prefix # After calculating prefix and exec_prefix, use their values for base_prefix and @@ -758,9 +761,12 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. # Virtual environments using the build directory Python still keep their prefix. -if not venv_prefix and os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX - exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix +if os_name != 'nt' and build_prefix: + base_prefix = config.get('base_prefix') or PREFIX + base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or prefix + if not venv_prefix: + prefix = config.get('prefix') or base_prefix + exec_prefix = config.get('exec_prefix') or base_exec_prefix # ****************************************************************************** From fa3adba8732978e557f35a21d4426de388c53919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 00:58:15 +0000 Subject: [PATCH 04/13] fixup! Revert "GH-126789: fix some sysconfig data on late site initializations" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/support/venv.py | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 Lib/test/support/venv.py diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py new file mode 100644 index 00000000000000..78e6a51ec1815e --- /dev/null +++ b/Lib/test/support/venv.py @@ -0,0 +1,70 @@ +import contextlib +import logging +import os +import subprocess +import shlex +import sys +import sysconfig +import tempfile +import venv + + +class VirtualEnvironment: + def __init__(self, prefix, **venv_create_args): + self._logger = logging.getLogger(self.__class__.__name__) + venv.create(prefix, **venv_create_args) + self._prefix = prefix + self._paths = sysconfig.get_paths( + scheme='venv', + vars={'base': self.prefix}, + expand=True, + ) + + @classmethod + @contextlib.contextmanager + def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args): + delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV')) + with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir: + yield cls(tmpdir, **venv_create_args) + + @property + def prefix(self): + return self._prefix + + @property + def paths(self): + return self._paths + + @property + def interpreter(self): + return os.path.join(self.paths['scripts'], os.path.basename(sys.executable)) + + def _format_output(self, name, data, indent='\t'): + if not data: + return indent + f'{name}: (none)' + if len(data.splitlines()) == 1: + return indent + f'{name}: {data}' + else: + prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines()) + return indent + f'{name}:\n' + prefixed_lines + + def run(self, *args, **subprocess_args): + if subprocess_args.get('shell'): + raise ValueError('Running the subprocess in shell mode is not supported.') + default_args = { + 'capture_output': True, + 'check': True, + } + try: + result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args) + except subprocess.CalledProcessError as e: + if e.returncode != 0: + self._logger.error( + f'Interpreter returned non-zero exit status {e.returncode}.\n' + + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n' + + self._format_output('STDOUT', e.stdout.decode()) + '\n' + + self._format_output('STDERR', e.stderr.decode()) + '\n' + ) + raise + else: + return result From c5b7d6389c171aeb81fa0d954f4fe21f1500d08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 01:26:08 +0000 Subject: [PATCH 05/13] fixup! Fix test_embed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/test/test_embed.py | 3 ++- Modules/getpath.py | 19 ++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index bf861ef06ee2d3..f0557259cc8499 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1649,7 +1649,8 @@ def test_init_pyvenv_cfg(self): config = { 'base_prefix': sysconfig.get_config_var("prefix"), 'base_exec_prefix': exec_prefix, - 'exec_prefix': exec_prefix, + 'exec_prefix': tmpdir, + 'prefix': tmpdir, 'base_executable': base_executable, 'executable': executable, 'module_search_paths': paths, diff --git a/Modules/getpath.py b/Modules/getpath.py index 7f3960326c2ec3..4ce00d02382fbf 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -641,12 +641,9 @@ def search_up(prefix, *landmarks, test=isfile): # For a venv, update the main prefix/exec_prefix but leave the base ones unchanged if venv_prefix: - if not prefix: - base_prefix = prefix - prefix = venv_prefix - if not exec_prefix: - base_exec_prefix = exec_prefix - exec_prefix = venv_prefix + base_prefix = prefix + base_exec_prefix = exec_prefix + prefix = exec_prefix = venv_prefix # After calculating prefix and exec_prefix, use their values for base_prefix and @@ -761,12 +758,12 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. # Virtual environments using the build directory Python still keep their prefix. -if os_name != 'nt' and build_prefix: - base_prefix = config.get('base_prefix') or PREFIX - base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or prefix +if not venv_prefix and os_name != 'nt' and build_prefix: if not venv_prefix: - prefix = config.get('prefix') or base_prefix - exec_prefix = config.get('exec_prefix') or base_exec_prefix + prefix = config.get('prefix') or PREFIX + exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix + base_prefix = config.get('base_prefix') or prefix + base_exec_prefix = config.get('base_exec_prefix') or base_prefix # ****************************************************************************** From 77aeff44d1ad6675453d6d227289cbe0fa53aa91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 01:29:27 +0000 Subject: [PATCH 06/13] Keep base_prefix and base_exec_prefix from PyConfig if set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Modules/getpath.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 4ce00d02382fbf..eb980f3834040b 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -641,8 +641,10 @@ def search_up(prefix, *landmarks, test=isfile): # For a venv, update the main prefix/exec_prefix but leave the base ones unchanged if venv_prefix: - base_prefix = prefix - base_exec_prefix = exec_prefix + if not base_prefix: + base_prefix = prefix + if not base_exec_prefix: + base_exec_prefix = exec_prefix prefix = exec_prefix = venv_prefix From 892876a09622ac4060f9fdc2f61f7c698f9d3e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 01:43:51 +0000 Subject: [PATCH 07/13] fixup! fixup! Fix test_embed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Modules/getpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index eb980f3834040b..9a39500c3571c7 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -760,7 +760,7 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. # Virtual environments using the build directory Python still keep their prefix. -if not venv_prefix and os_name != 'nt' and build_prefix: +if os_name != 'nt' and build_prefix: if not venv_prefix: prefix = config.get('prefix') or PREFIX exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix From bd5061800a76c71c20556cc8302c777c3c0a8987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 01:44:52 +0000 Subject: [PATCH 08/13] Keep non-windows prefix overwrite quirk on a venv's base_prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Modules/getpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/getpath.py b/Modules/getpath.py index 9a39500c3571c7..d5d16608df1a24 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -764,8 +764,8 @@ def search_up(prefix, *landmarks, test=isfile): if not venv_prefix: prefix = config.get('prefix') or PREFIX exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix - base_prefix = config.get('base_prefix') or prefix - base_exec_prefix = config.get('base_exec_prefix') or base_prefix + base_prefix = config.get('base_prefix') or PREFIX + base_exec_prefix = config.get('base_exec_prefix') or EXEC_PREFIX or base_prefix # ****************************************************************************** From 3c1fb295032314808a90b375b21639bd52bb6c66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 01:47:09 +0000 Subject: [PATCH 09/13] Remove sys.prefix overwriting from the site module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/site.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index 54f07ab5b4e6d5..3282a344f370ad 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -94,6 +94,12 @@ def _trace(message): print(message, file=sys.stderr) +def _warn(*args, **kwargs): + import warnings + + warnings.warn(*args, **kwargs) + + def makepath(*paths): dir = os.path.join(*paths) try: @@ -602,7 +608,10 @@ def venv(known_paths): elif key == 'home': sys._home = value - sys.prefix = sys.exec_prefix = site_prefix + if sys.prefix != site_prefix: + _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + if sys.exec_prefix != site_prefix: + _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) # Doing this here ensures venv takes precedence over user-site addsitepackages(known_paths, [sys.prefix]) From 3c24f4428dfc919669a81c85e5247df6353ff2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Tue, 19 Nov 2024 03:16:51 +0000 Subject: [PATCH 10/13] Fix test_embed on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tmpdir` is a venv, so `prefix` will be set to that value, but `base_prefix` should stay `pyvenv_home`. Signed-off-by: Filipe Laíns --- Lib/test/test_embed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index f0557259cc8499..5c38b28322deb4 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1657,7 +1657,6 @@ def test_init_pyvenv_cfg(self): } if MS_WINDOWS: config['base_prefix'] = pyvenv_home - config['prefix'] = pyvenv_home config['stdlib_dir'] = os.path.join(pyvenv_home, 'Lib') config['use_frozen_modules'] = bool(not support.Py_DEBUG) else: From f98ec671c8b740327a1d28e6117b81805d47f898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Nov 2024 14:15:33 +0000 Subject: [PATCH 11/13] Update documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Doc/c-api/init_config.rst | 13 +++++++ Doc/library/site.rst | 24 ++++++++----- Doc/library/sys.rst | 64 ++++++++++++++++++++++------------- Doc/library/sys_path_init.rst | 49 +++++++++++++++++++++------ Doc/library/venv.rst | 3 ++ 5 files changed, 112 insertions(+), 41 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 6194d7446c73e4..41dfd4a2b283bf 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1588,9 +1588,22 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.site_import` to ``0``. * Set :c:member:`~PyConfig.safe_path` to ``1``. +If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in +the same directory as :c:member:`~PyConfig.executable`, or its parent, +:c:member:`~PyConfig.prefix` and :c:member:`~PyConfig.exec_prefix` are set that +location. When this happens, :c:member:`~PyConfig.base_prefix` and +:c:member:`~PyConfig.base_exec_prefix` still keep their value, pointing to the +base installation. See :ref:`sys-path-init-virtual-environments` for more +information. + The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable`. +.. versionchanged:: 3.14 + + :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix` are now + set to the ``pyvenv.cfg`` directory. This was previously done by :mod:`site`, + therefore affected by :option:`-S`. PyInitConfig C API ================== diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 4508091f679dc7..27231429553c11 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -49,14 +49,22 @@ added path for configuration files. identified by the "t" suffix in the version-specific directory name, such as :file:`lib/python3.13t/`. -If a file named "pyvenv.cfg" exists one directory above sys.executable, -sys.prefix and sys.exec_prefix are set to that directory and -it is also checked for site-packages (sys.base_prefix and -sys.base_exec_prefix will always be the "real" prefixes of the Python -installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to anything other than "true" -(case-insensitive), the system-level prefixes will not be -searched for site-packages; otherwise they will. +.. versionchanged:: 3.14 + + :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is + now done during the :ref:`path initialization `. As a result, + under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and + :data:`sys.exec_prefix` no longer depend on the :ref:`site` initialization, + and are therefore unaffected by :option:`-S`. + +.. _site-virtual-environments-configuration: + +When running under a :ref:`virtual environment `, +the ``pyvenv.cfg`` file in :data:`sys.prefix` is checked for site-specific +configurations. If the ``include-system-site-packages`` key exists and is set to +``true`` (case-insensitive), the system-level prefixes will be searched for +site-packages, otherwise they won't. .. index:: single: # (hash); comment diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index d83816ec1502ca..bc974908416dce 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -130,27 +130,26 @@ always available. .. data:: base_exec_prefix - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`exec_prefix`. If not running in a - :ref:`virtual environment `, the values will stay the same; if - ``site.py`` finds that a virtual environment is in use, the values of - :data:`prefix` and :data:`exec_prefix` will be changed to point to the - virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). + Equivalent to :data:`exec_prefix`, but refering to the base Python installation. + + When running under :ref:`sys-path-init-virtual-environments`, + :data:`exec_prefix` gets overwritten to the virtual environment prefix. + :data:`base_exec_prefix`, conversely, does not change, and always points to + the base Python installation. + Refer to :ref:`sys-path-init-virtual-environments` for more information. .. versionadded:: 3.3 .. data:: base_prefix - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`prefix`. If not running in a :ref:`virtual environment `, the values - will stay the same; if ``site.py`` finds that a virtual environment is in - use, the values of :data:`prefix` and :data:`exec_prefix` will be changed to - point to the virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). + Equivalent to :data:`prefix`, but refering to the base Python installation. + + When running under :ref:`virtual environment `, + :data:`prefix` gets overwritten to the virtual environment prefix. + :data:`base_prefix`, conversely, does not change, and always points to + the base Python installation. + Refer to :ref:`sys-path-init-virtual-environments` for more information. .. versionadded:: 3.3 @@ -483,11 +482,19 @@ always available. .. note:: - If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual environment. - The value for the Python installation will still be available, via - :data:`base_exec_prefix`. + If a :ref:`virtual environment ` is in effect, this :data:`exec_prefix` + will point to the virtual environment. The value for the Python installation + will still be available, via :data:`base_exec_prefix`. + Refer to :ref:`sys-path-init-virtual-environments` for more information. + .. versionchanged:: 3.14 + + When running under a :ref:`virtual environment `, + :data:`prefix` and :data:`exec_prefix` are now set to the virtual + environment prefix by the :ref:`path initialization `, + instead of :mod:`site`. This means that :data:`prefix` and + :data:`exec_prefix` always point to the virtual environment, even when + :mod:`site` is disabled (:option:`-S`). .. data:: executable @@ -1483,10 +1490,21 @@ always available. argument to the :program:`configure` script. See :ref:`installation_paths` for derived paths. - .. note:: If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual - environment. The value for the Python installation will still be - available, via :data:`base_prefix`. + .. note:: + + If a :ref:`virtual environment ` is in effect, this :data:`prefix` + will point to the virtual environment. The value for the Python installation + will still be available, via :data:`base_prefix`. + Refer to :ref:`sys-path-init-virtual-environments` for more information. + + .. versionchanged:: 3.14 + + When running under a :ref:`virtual environment `, + :data:`prefix` and :data:`exec_prefix` are now set to the virtual + environment prefix by the :ref:`path initialization `, + instead of :mod:`site`. This means that :data:`prefix` and + :data:`exec_prefix` always point to the virtual environment, even when + :mod:`site` is disabled (:option:`-S`). .. data:: ps1 diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index a87a41cf829fa8..18fe32d9c7f10a 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -47,8 +47,15 @@ however on other platforms :file:`lib/python{majorversion}.{minorversion}/lib-dy ``exec_prefix``. On some platforms :file:`lib` may be :file:`lib64` or another value, see :data:`sys.platlibdir` and :envvar:`PYTHONPLATLIBDIR`. -Once found, ``prefix`` and ``exec_prefix`` are available at :data:`sys.prefix` and -:data:`sys.exec_prefix` respectively. +Once found, ``prefix`` and ``exec_prefix`` are available at +:data:`sys.base_prefix` and :data:`sys.base_exec_prefix` respectively. + +If :envvar:`PYTHONHOME` is not set, and a ``pyvenv.cfg`` file is found alongside +the main executable, or in its parent directory, :data:`sys.prefix` and +:data:`sys.exec_prefix` get set to the directory containing ``pyvenv.cfg``, +otherwise they are set to the same value as :data:`sys.base_prefix` and +:data:`sys.base_exec_prefix`, respectively. +This is used by :ref:`sys-path-init-virtual-environments`. Finally, the :mod:`site` module is processed and :file:`site-packages` directories are added to the module search path. A common way to customize the search path is @@ -60,18 +67,40 @@ the :mod:`site` module documentation. Certain command line options may further affect path calculations. See :option:`-E`, :option:`-I`, :option:`-s` and :option:`-S` for further details. -Virtual environments +.. versionchanged:: 3.14 + + :data:`sys.prefix` and :data:`sys.exec_prefix` are now set to the + ``pyvenv.cfg`` directory during the path initialization. This was previously + done by :mod:`site`, therefore affected by :option:`-S`. + +.. _sys-path-init-virtual-environments: + +Virtual Environments -------------------- -If Python is run in a virtual environment (as described at :ref:`tut-venv`) -then ``prefix`` and ``exec_prefix`` are specific to the virtual environment. +Virtual environments place a ``pyvenv.cfg`` file in their prefix, which causes +:data:`sys.prefix` and :data:`sys.exec_prefix` to point to them, instead of the +base installation. + +The ``prefix`` and ``exec_prefix`` values of the base installation are available +at :data:`sys.base_prefix` and :data:`sys.base_exec_prefix`. -If a ``pyvenv.cfg`` file is found alongside the main executable, or in the -directory one level above the executable, the following variations apply: +As well as being used as a marker to identify virtual environments, +``pyvenv.cfg`` may also be used to configure the :mod:`site` initialization. +Please refer to :mod:`site`'s +:ref:`virtual environments documentation `. + +.. note:: + + :envvar:`PYTHONHOME` overrides the ``pyvenv.cfg`` detection. + +.. note:: -* If ``home`` is an absolute path and :envvar:`PYTHONHOME` is not set, this - path is used instead of the path to the main executable when deducing ``prefix`` - and ``exec_prefix``. + There are other ways how "virtual environments" could be implemented, this + documentation referes implementations based on the ``pyvenv.cfg`` mechanism, + such as :mod:`venv`. Most virtual environment implementations follow the + model set by :mod:`venv`, but there may be exotic implementations that + diverge from it. _pth files ---------- diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 5205c6c211d9bf..bed799aedfdfb1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -25,6 +25,9 @@ A virtual environment is created on top of an existing Python installation, known as the virtual environment's "base" Python, and may optionally be isolated from the packages in the base environment, so only those explicitly installed in the virtual environment are available. +See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s +:ref:`virtual environments documentation ` +for more information. When used from within a virtual environment, common installation tools such as :pypi:`pip` will install Python packages into a virtual environment From 3bfb073f851fcca4b97a9e5a648647a5bb387917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Nov 2024 14:47:22 +0000 Subject: [PATCH 12/13] Fix documentation typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Doc/library/site.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 27231429553c11..5f2a0f610e1aa5 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -55,7 +55,7 @@ added path for configuration files. :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is now done during the :ref:`path initialization `. As a result, under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and - :data:`sys.exec_prefix` no longer depend on the :ref:`site` initialization, + :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, and are therefore unaffected by :option:`-S`. .. _site-virtual-environments-configuration: From 7e776b7d52ed331f7f86f69c28f6e632418c0b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Nov 2024 19:01:28 +0000 Subject: [PATCH 13/13] Minor documentation grammar fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Doc/c-api/init_config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 41dfd4a2b283bf..ca993bc93b3ee6 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1601,7 +1601,7 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set .. versionchanged:: 3.14 - :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix` are now + :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix`, are now set to the ``pyvenv.cfg`` directory. This was previously done by :mod:`site`, therefore affected by :option:`-S`.