Skip to content

Use library search paths of compiler for RPATH when building binutils with system compiler + enhance sanity check by running --version for binutils commands #2323

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 31, 2021
Merged

Use library search paths of compiler for RPATH when building binutils with system compiler + enhance sanity check by running --version for binutils commands #2323

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
680adc4
Add sanity check commands to binutils
Flamefire Jan 27, 2021
20e281e
Use compiler library paths to determine rpaths
Flamefire Jan 28, 2021
fa0d202
style and comment fixes + use relative commands in sanity check in bi…
boegel Jan 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 46 additions & 31 deletions easybuild/easyblocks/b/binutils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,15 @@
import re
from distutils.version import LooseVersion

import easybuild.tools.environment as env
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
from easybuild.tools.filetools import apply_regex_substitutions, copy_file
from easybuild.tools.modules import get_software_libdir, get_software_root
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
from easybuild.tools.utilities import nub


class EB_binutils(ConfigureMake):
Expand All @@ -55,41 +56,54 @@ def extra_options(extra_vars=None):
})
return extra_vars

def configure_step(self):
"""Custom configuration procedure for binutils: statically link to zlib, configure options."""
def determine_used_library_paths(self):
"""Check which paths are used to search for libraries"""

# take sysroot configuration option into account;
# make sure we don't use None going forward since the value is used in os.path.join expressions
sysroot = build_option('sysroot') or os.path.sep
# determine C compiler command: use $CC, fall back to 'gcc' (when using system toolchain)
compiler_cmd = os.environ.get('CC', 'gcc')

libs = ''
# determine library search paths for GCC
stdout, ec = run_cmd('LC_ALL=C "%s" -print-search-dirs' % compiler_cmd, simple=False, log_all=True)
if ec:
raise EasyBuildError("Failed to determine library search dirs from compiler %s", compiler_cmd)

if self.toolchain.is_system_toolchain():
# determine list of 'lib' directories to use rpath for;
# this should 'harden' the resulting binutils to bootstrap GCC
# (no trouble when other libstdc++ is build etc)
m = re.search('^libraries: *=(.*)$', stdout, re.M)
paths = nub(os.path.abspath(p) for p in m.group(1).split(os.pathsep))
self.log.debug('Unique library search paths from compiler %s: %s', compiler_cmd, paths)

# The installed lib dir must come first though to avoid taking system libs over installed ones, see:
# https://github.com/easybuilders/easybuild-easyconfigs/issues/10056
# Escaping: Double $$ for Make, \$ for shell to get literal $ORIGIN in the file
libdirs = [r'\$\$ORIGIN/../lib']
for libdir in ['lib', 'lib64', os.path.join('lib', 'x86_64-linux-gnu')]:
# Filter out all paths that do not exist
paths = [p for p in paths if os.path.exists(p)]
self.log.debug("Existing library search paths: %s", ', '.join(paths))

libdir = os.path.join(sysroot, 'usr', libdir)
result = []
for path in paths:
if any(os.path.samefile(path, p) for p in result):
self.log.debug("Skipping symlink to existing path: %s", path)
elif not glob.glob(os.path.join(path, '*.so*')):
self.log.debug("Skipping path with no shared libraries: %s", path)
else:
result.append(path)

# also consider /lib, /lib64 (without /usr/)
alt_libdir = os.path.join(sysroot, libdir)
self.log.info("Determined library search paths: %s", ', '.join(result))
return result

if os.path.exists(libdir):
libdirs.append(libdir)
if os.path.exists(alt_libdir) and not os.path.samefile(libdir, alt_libdir):
libdirs.append(alt_libdir)
def configure_step(self):
"""Custom configuration procedure for binutils: statically link to zlib, configure options."""

elif os.path.exists(alt_libdir):
libdirs.append(alt_libdir)
if self.toolchain.is_system_toolchain():
# determine list of 'lib' directories to use rpath for;
# this should 'harden' the resulting binutils to bootstrap GCC
# (no trouble when other libstdc++ is build etc)
lib_paths = self.determine_used_library_paths()

# The installed lib dir must come first though to avoid taking system libs over installed ones, see:
# https://github.com/easybuilders/easybuild-easyconfigs/issues/10056;
# double $$ to get literal $ORIGIN in the file when command is run
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment is actually wrong. The $$ is (still) for make as in the comment above.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment (which was there originally) didn't make sense to me, so I tweaked it.

Which make is being referred to here?

Please open an issue or PR to follow up, keeping track of things in closed/merged PRs is close to impossible to keep up with...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#2327

lib_paths = [r'$$ORIGIN/../lib'] + lib_paths
# Mind the single quotes
libs += ' '.join("-Wl,-rpath='%s'" % libdir for libdir in libdirs)
libs = ["-Wl,-rpath='%s'" % x for x in lib_paths]
else:
libs = []

# configure using `--with-system-zlib` if zlib is a (build) dependency
zlibroot = get_software_root('zlib')
Expand All @@ -113,11 +127,9 @@ def configure_step(self):

# for older versions, injecting the path to the static libz library into $LIBS works
else:
libs += ' ' + libz_path
libs.append(libz_path)

# Using double quotes for LIBS to allow single quotes in libs
self.cfg.update('preconfigopts', 'LIBS="%s"' % libs)
self.cfg.update('prebuildopts', 'LIBS="%s"' % libs)
env.setvar('LIBS', ' '.join(libs))

# explicitly configure binutils to use / as sysroot
# this is required to ensure the binutils installation works correctly with a (system)
Expand Down Expand Up @@ -206,6 +218,9 @@ def sanity_check_step(self):
os.path.join('include', 'libiberty.h'),
])

# All binaries support --version, check that they can be run
custom_commands = ['%s --version' % b for b in binaries]

# if zlib is listed as a build dependency, it should have been linked in statically
build_deps = self.cfg.dependencies(build_only=True)
if any(dep['name'] == 'zlib' for dep in build_deps):
Expand All @@ -221,4 +236,4 @@ def sanity_check_step(self):
if re.search(r'libz\.%s' % shlib_ext, out):
raise EasyBuildError("zlib is not statically linked in %s: %s", bin_path, out)

super(EB_binutils, self).sanity_check_step(custom_paths=custom_paths)
super(EB_binutils, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)