Add CIBW_DELOCATE_COMMAND env variable

nsoranzo · nsoranzo · commit aacfab4cdd30 · 2019-11-12T23:56:45.000Z
to allow different options to auditwheel/delocate, alternative commands and a future Windows-equivalent. Fix pypa#191 .
diff --git a/README.md b/README.md
@@ -95,8 +95,9 @@ Options
 |   | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  | Choose the Python versions to build |
 | **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment)  | Set environment variables needed during the build |
 |   | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build)  | Execute a shell command preparing each wheel's build |
+|   | [`CIBW_DELOCATE_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#delocate-command)  | Execute a shell command to delocate each (non-pure Python) built wheel |
 |   | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  | Specify alternative manylinux docker images |
-| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command)  | Execute a shell command to test all built wheels |
+| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command)  | Execute a shell command to test each built wheel |
 |   | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires)  | Install Python dependencies before running the tests |
 |   | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras)  | Install your wheel for testing using extras_require |
 | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras)  | Increase/decrease the output of pip wheel |
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
@@ -89,6 +89,13 @@ def main():
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='')
     build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
+    if platform == 'linux':
+        delocate_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
+    elif platform == 'macos':
+        delocate_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'
+    else:
+        delocate_command_default = ''
+    delocate_command = get_option_from_environment('CIBW_DELOCATE_COMMAND', platform=platform, default=delocate_command_default)
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
 
     if test_extras:
@@ -130,6 +137,7 @@ def main():
         before_build=before_build,
         build_verbosity=build_verbosity,
         build_selector=build_selector,
+        delocate_command=delocate_command,
         environment=environment,
     )
 
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
@@ -30,7 +30,7 @@ def get_python_configurations(build_selector):
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux_images):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, delocate_command, environment, manylinux_images):
     try:
         subprocess.check_call(['docker', '--version'])
     except:
@@ -77,11 +77,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
                 # Delocate the wheel
                 # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
                 # the first element
-                if [[ "$built_wheel" == *none-any.whl ]]; then
-                    # pure python wheel - just copy
+                if [[ "$built_wheel" == *none-any.whl ]] || [ -z {delocate_command} ]; then
+                    # pure Python wheel or empty delocate command
                     mv "$built_wheel" /tmp/delocated_wheels
                 else
-                    auditwheel repair "$built_wheel" -w /tmp/delocated_wheels
+                    built_wheel=$built_wheel sh -c {delocate_command}
                 fi
                 delocated_wheels=(/tmp/delocated_wheels/*.whl)
 
@@ -135,12 +135,15 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
             test_requires=' '.join(test_requires),
             test_extras=test_extras,
             test_command=shlex_quote(
-                prepare_command(test_command, project='/project') if test_command else ''
+                prepare_command(test_command, {'project': '/project'}) if test_command else ''
             ),
             before_build=shlex_quote(
-                prepare_command(before_build, project='/project') if before_build else ''
+                prepare_command(before_build, {'project': '/project'}) if before_build else ''
             ),
             build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
+            delocate_command=shlex_quote(
+                prepare_command(delocate_command, {'wheel': '"$built_wheel"', 'dest_dir': '/tmp/delocated_wheels'}) if delocate_command else ''
+            ),
             environment_exports='\n'.join(environment.as_shell_commands()),
             uid=os.getuid(),
             gid=os.getgid(),
diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py
@@ -25,7 +25,12 @@ def get_python_configurations(build_selector):
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, delocate_command, environment):
+    abs_project_dir = os.path.abspath(project_dir)
+    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
+    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    delocated_wheel_dir = os.path.join(temp_dir, 'delocated_wheel')
+
     python_configurations = get_python_configurations(build_selector)
     get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
     get_pip_script = '/tmp/get-pip.py'
@@ -44,8 +49,6 @@ def call(args, env=None, cwd=None, shell=False):
 
         return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
 
-    abs_project_dir = os.path.abspath(project_dir)
-
     # get latest pip once and for all
     call(['curl', '-L', '-o', get_pip_script, get_pip_url])
 
@@ -94,32 +97,29 @@ def call(args, env=None, cwd=None, shell=False):
         call(['pip', 'install', 'wheel'], env=env)
         call(['pip', 'install', 'delocate'], env=env)
 
-        # setup dirs
-        if os.path.exists('/tmp/built_wheel'):
-            shutil.rmtree('/tmp/built_wheel')
-        os.makedirs('/tmp/built_wheel')
-        if os.path.exists('/tmp/delocated_wheel'):
-            shutil.rmtree('/tmp/delocated_wheel')
-        os.makedirs('/tmp/delocated_wheel')
-
         # run the before_build command
         if before_build:
-            before_build_prepared = prepare_command(before_build, project=abs_project_dir)
+            before_build_prepared = prepare_command(before_build, {'project': abs_project_dir})
             call(before_build_prepared, env=env, shell=True)
 
         # build the wheel
-        call(['pip', 'wheel', abs_project_dir, '-w', '/tmp/built_wheel', '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
-        built_wheel = glob('/tmp/built_wheel/*.whl')[0]
-
-        if built_wheel.endswith('none-any.whl'):
-            # pure python wheel - just move
-            shutil.move(built_wheel, '/tmp/delocated_wheel')
+        if os.path.exists(built_wheel_dir):
+            shutil.rmtree(built_wheel_dir)
+        os.makedirs(built_wheel_dir)
+        call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
+        built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
+
+        # delocate the wheel
+        if os.path.exists(delocated_wheel_dir):
+            shutil.rmtree(delocated_wheel_dir)
+        os.makedirs(delocated_wheel_dir)
+        if built_wheel.endswith('none-any.whl') or not delocate_command:
+            # pure Python wheel or empty delocate command
+            shutil.move(built_wheel, delocated_wheel_dir)
         else:
-            # list the dependencies
-            call(['delocate-listdeps', built_wheel], env=env)
-            # rebuild the wheel with shared libraries included and place in output dir
-            call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env)
-        delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0]
+            delocate_command_prepared = prepare_command(delocate_command, {'wheel': built_wheel, 'dest_dir': delocated_wheel_dir})
+            call(delocate_command_prepared, env=env, shell=True)
+        delocated_wheel = glob(os.path.join(delocated_wheel_dir, '*.whl'))[0]
 
         if test_command:
             # set up a virtual environment to install and test from, to make sure
@@ -150,7 +150,7 @@ def call(args, env=None, cwd=None, shell=False):
             # run the tests from $HOME, with an absolute path in the command
             # (this ensures that Python runs the tests against the installed wheel
             # and not the repo code)
-            test_command_prepared = prepare_command(test_command, project=abs_project_dir)
+            test_command_prepared = prepare_command(test_command, {'project': abs_project_dir})
             call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
 
             # clean up
diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
@@ -2,14 +2,14 @@
 import warnings
 
 
-def prepare_command(command, project):
+def prepare_command(command, replace_dict):
     '''
-    Preprocesses a command by expanding variables like {project}.
+    Preprocesses a command by expanding variables like {python}.
 
-    For example, used in the test_command option, to specify the path to the
-    tests directory.
+    For example, used in the test_command option to specify the path to the
+    project's root.
     '''
-    return command.format(python='python', pip='pip', project=project)
+    return command.format(python='python', pip='pip', **replace_dict)
 
 
 def get_build_verbosity_extra_flags(level):
diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py
@@ -57,8 +57,7 @@ def get_python_configurations(build_selector):
     return python_configurations
 
 
-
-def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, delocate_command, environment):
     def simple_shell(args, env=None, cwd=None):
         print('+ ' + ' '.join(args))
         args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
@@ -90,6 +89,7 @@ def shell(args, env=None, cwd=None):
     abs_project_dir = os.path.abspath(project_dir)
     temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
     built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    delocated_wheel_dir = os.path.join(temp_dir, 'delocated_wheel')
 
     # install nuget as best way to provide python
     nuget = 'C:\\cibw\\nuget.exe'
@@ -136,12 +136,20 @@ def shell(args, env=None, cwd=None):
 
         # run the before_build command
         if before_build:
-            before_build_prepared = prepare_command(before_build, project=abs_project_dir)
+            before_build_prepared = prepare_command(before_build, {'project': abs_project_dir})
             shell([before_build_prepared], env=env)
 
         # build the wheel
         shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
-        built_wheel = glob(built_wheel_dir+'/*.whl')[0]
+        built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
+
+        if built_wheel.endswith('none-any.whl') or not delocate_command:
+            # pure Python wheel or empty delocate command
+            shutil.move(built_wheel, delocated_wheel_dir)
+        else:
+            delocate_command_prepared = prepare_command(delocate_command, {'wheel': built_wheel, 'dest_dir': delocated_wheel_dir})
+            shell([delocate_command_prepared], env=env)
+        delocated_wheel = glob(os.path.join(delocated_wheel_dir, '*.whl'))[0]
 
         if test_command:
             # set up a virtual environment to install and test from, to make sure
@@ -160,7 +168,7 @@ def shell(args, env=None, cwd=None):
             shell(['which', 'python'], env=virtualenv_env)
 
             # install the wheel
-            shell(['pip', 'install', built_wheel + test_extras], env=virtualenv_env)
+            shell(['pip', 'install', delocated_wheel + test_extras], env=virtualenv_env)
 
             # test the wheel
             if test_requires:
@@ -169,14 +177,14 @@ def shell(args, env=None, cwd=None):
             # run the tests from c:\, with an absolute path in the command
             # (this ensures that Python runs the tests against the installed wheel
             # and not the repo code)
-            test_command_prepared = prepare_command(test_command, project=abs_project_dir)
+            test_command_prepared = prepare_command(test_command, {'project': abs_project_dir})
             shell([test_command_prepared], cwd='c:\\', env=virtualenv_env)
 
             # clean up
             shutil.rmtree(venv_dir)
 
         # we're all done here; move it to output (remove if already exists)
-        dst = os.path.join(output_dir, os.path.basename(built_wheel))
+        dst = os.path.join(output_dir, os.path.basename(delocated_wheel))
         if os.path.isfile(dst):
             os.remove(dst)
-        shutil.move(built_wheel, dst)
+        shutil.move(delocated_wheel, dst)
diff --git a/docs/options.md b/docs/options.md
