Skip to content

Enhance sitecustomize.py script installed by Python easyblock to fix support for virtualenv #2483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
boegel merged 3 commits into from
Jun 30, 2021
Merged

Enhance sitecustomize.py script installed by Python easyblock to fix support for virtualenv #2483

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4d53333
Add support for older virtualenv versions
Flamefire Jun 21, 2021
750c046
update comment
Flamefire Jun 21, 2021
b1d1508
Fix EBPYTHONPREFIXES for virtualenvs and add sanity check for base case
Flamefire Jun 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 43 additions & 5 deletions easybuild/easyblocks/p/python.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,24 @@

EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'

# We want the following import order:
# 1. Packages installed into VirtualEnv
# 2. Packages installed into $EBPYTHONPREFIXES (e.g. our modules)
# 3. Packages installed in the Python module
# Note that this script is run after all sys.path manipulation by Python and Virtualenv are done.
# Hence prepending $EBPYTHONPREFIXES would shadow VirtualEnv packages and
# appending would NOT shadow the Python-module packages which makes updating packages via ECs impossible.
# Hence we move all paths which are prefixed with the Python-module path to the back but need to make sure
# not to move the VirtualEnv paths.
SITECUSTOMIZE = """
# sitecustomize.py script installed by EasyBuild,
# to support picking up Python packages which were installed
# for multiple Python versions in the same directory
# to pick up Python packages installed with `--prefix` into folders listed in $%(EBPYTHONPREFIXES)s

import os
import site
import sys

# print debug messages when $EBPYTHONPREFIXES_DEBUG is defined
# print debug messages when $%(EBPYTHONPREFIXES)s_DEBUG is defined
debug = os.getenv('%(EBPYTHONPREFIXES)s_DEBUG')

# use prefixes from $EBPYTHONPREFIXES, so they have lower priority than
Expand All @@ -82,8 +90,9 @@
if debug:
print("[%(EBPYTHONPREFIXES)s] postfix subdirectory to consider in installation directories: %%s" %% postfix)

site_packages = site.getsitepackages()
base_paths = [p for p in sys.path if p in site_packages]
potential_sys_prefixes = (getattr(sys, attr, None) for attr in ("real_prefix", "base_prefix", "prefix"))
sys_prefix = next(p for p in potential_sys_prefixes if p)
base_paths = [p for p in sys.path if p.startswith(sys_prefix)]

for prefix in ebpythonprefixes.split(os.pathsep):
if debug:
Expand Down Expand Up @@ -461,6 +470,32 @@ def install_step(self):
symlink(target_lib_dynload, lib_dynload)
change_dir(cwd)

def _sanity_check_ebpythonprefixes(self):
"""Check that EBPYTHONPREFIXES works"""
temp_prefix = tempfile.mkdtemp(suffix='-tmp-prefix')
site_packages_path = os.path.join('lib', 'python' + self.pyshortver, 'site-packages')
temp_site_packages_path = os.path.join(temp_prefix, site_packages_path)
mkdir(temp_site_packages_path, parents=True) # Must exist
(out, _) = run_cmd("%s=%s python -c 'import sys; print(sys.path)'" % (EBPYTHONPREFIXES, temp_prefix))
out = out.strip()
# Output should be a list which we can evaluate directly
if not out.startswith('[') or not out.endswith(']'):
raise EasyBuildError("Unexpected output for sys.path: %s", out)
paths = eval(out)
base_site_packages_path = os.path.join(self.installdir, site_packages_path)
try:
base_prefix_idx = paths.index(base_site_packages_path)
except ValueError:
raise EasyBuildError("The Python install path was not added to sys.path (%s)", paths)
try:
eb_prefix_idx = paths.index(temp_site_packages_path)
except ValueError:
raise EasyBuildError("EasyBuilds sitecustomize.py did not add %s to sys.path (%s)",
temp_site_packages_path, paths)
if eb_prefix_idx > base_prefix_idx:
raise EasyBuildError("EasyBuilds sitecustomize.py did not add %s before %s to sys.path (%s)",
temp_site_packages_path, base_site_packages_path, paths)

def sanity_check_step(self):
"""Custom sanity check for Python."""

Expand Down Expand Up @@ -492,6 +527,9 @@ def sanity_check_step(self):
else:
self.log.info("No errors found in output of %s: %s", cmd, out)

if self.cfg['ebpythonprefixes']:
self._sanity_check_ebpythonprefixes()

pyver = 'python' + self.pyshortver
custom_paths = {
'files': [os.path.join('bin', pyver), os.path.join('lib', 'lib' + pyver + abiflags + '.' + shlib_ext)],
Expand Down