Add CIBW_REPAIR_COMMAND env variable

nsoranzo · nsoranzo · commit 3d864e78b013 · 2019-11-13T16:47:18.000Z
to allow different options for auditwheel/delocate, alternative commands and a future Windows equivalent. Fix pypa#191 .
diff --git a/README.md b/README.md
@@ -95,8 +95,9 @@ Options
 |   | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)  | Choose the Python versions to build |
 | **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment)  | Set environment variables needed during the build |
 |   | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build)  | Execute a shell command preparing each wheel's build |
+|   | [`CIBW_REPAIR_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-command)  | Execute a shell command to repair each (non-pure Python) built wheel |
 |   | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)  | Specify alternative manylinux docker images |
-| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command)  | Execute a shell command to test all built wheels |
+| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command)  | Execute a shell command to test each built wheel |
 |   | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires)  | Install Python dependencies before running the tests |
 |   | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras)  | Install your wheel for testing using extras_require |
 | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras)  | Increase/decrease the output of pip wheel |
@@ -122,7 +123,7 @@ Here are some repos that use cibuildwheel.
 Legal note
 ----------
 
-Since `cibuildwheel` runs the wheel through delocate or auditwheel, it might automatically bundle dynamically linked libraries from the build machine.
+Since `cibuildwheel` repairs the wheel with `delocate` or `auditwheel`, it might automatically bundle dynamically linked libraries from the build machine.
 
 It helps ensure that the library can run without any dependencies outside of the pip toolchain.
 
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
@@ -89,6 +89,13 @@ def main():
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='')
     build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
+    if platform == 'linux':
+        repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
+    elif platform == 'macos':
+        repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'
+    else:
+        repair_command_default = ''
+    repair_command = get_option_from_environment('CIBW_REPAIR_COMMAND', platform=platform, default=repair_command_default)
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
 
     if test_extras:
@@ -130,6 +137,7 @@ def main():
         before_build=before_build,
         build_verbosity=build_verbosity,
         build_selector=build_selector,
+        repair_command=repair_command,
         environment=environment,
     )
 
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
@@ -30,7 +30,7 @@ def get_python_configurations(build_selector):
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux_images):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images):
     try:
         subprocess.check_call(['docker', '--version'])
     except:
@@ -60,30 +60,28 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
             {environment_exports}
 
             for PYBIN in {pybin_paths}; do
-                # Setup
-                rm -rf /tmp/built_wheel
-                rm -rf /tmp/delocated_wheels
-                mkdir /tmp/built_wheel
-                mkdir /tmp/delocated_wheels
-
                 if [ ! -z {before_build} ]; then
                     PATH="$PYBIN:$PATH" sh -c {before_build}
                 fi
 
-                # Build that wheel
+                # Build the wheel
+                rm -rf /tmp/built_wheel
+                mkdir /tmp/built_wheel
                 PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
                 built_wheel=(/tmp/built_wheel/*.whl)
 
-                # Delocate the wheel
+                # repair the wheel
+                rm -rf /tmp/repaired_wheels
+                mkdir /tmp/repaired_wheels
                 # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
                 # the first element
-                if [[ "$built_wheel" == *none-any.whl ]]; then
-                    # pure python wheel - just copy
-                    mv "$built_wheel" /tmp/delocated_wheels
+                if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
+                    # pure Python wheel or empty repair command
+                    mv "$built_wheel" /tmp/repaired_wheels
                 else
-                    auditwheel repair "$built_wheel" -w /tmp/delocated_wheels
+                    built_wheel=$built_wheel sh -c {repair_command}
                 fi
-                delocated_wheels=(/tmp/delocated_wheels/*.whl)
+                repaired_wheels=(/tmp/repaired_wheels/*.whl)
 
                 if [ ! -z {test_command} ]; then
                     # Set up a virtual environment to install and test from, to make sure
@@ -105,7 +103,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
                         # functionally the same, differing only in name, wheel metadata, and possibly include
                         # different external shared libraries. so it doesn't matter which one we run the tests on.
                         # Let's just pick the first one.
-                        pip install "${{delocated_wheels[0]}}"{test_extras}
+                        pip install "${{repaired_wheels[0]}}"{test_extras}
 
                         # Install any requirements to run the tests
                         if [ ! -z "{test_requires}" ]; then
@@ -127,8 +125,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
                 fi
 
                 # we're all done here; move it to output
-                mv "${{delocated_wheels[@]}}" /output
-                for delocated_wheel in "${{delocated_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$delocated_wheel")"; done
+                mv "${{repaired_wheels[@]}}" /output
+                for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done
             done
         '''.format(
             pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
@@ -141,6 +139,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
                 prepare_command(before_build, project='/project') if before_build else ''
             ),
             build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
+            repair_command=shlex_quote(
+                prepare_command(repair_command, wheel='"$built_wheel"', dest_dir='/tmp/repaired_wheels') if repair_command else ''
+            ),
             environment_exports='\n'.join(environment.as_shell_commands()),
             uid=os.getuid(),
             gid=os.getgid(),
diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py
@@ -25,7 +25,12 @@ def get_python_configurations(build_selector):
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
+    abs_project_dir = os.path.abspath(project_dir)
+    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
+    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
+
     python_configurations = get_python_configurations(build_selector)
     get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
     get_pip_script = '/tmp/get-pip.py'
@@ -44,8 +49,6 @@ def call(args, env=None, cwd=None, shell=False):
 
         return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
 
-    abs_project_dir = os.path.abspath(project_dir)
-
     # get latest pip once and for all
     call(['curl', '-L', '-o', get_pip_script, get_pip_url])
 
@@ -94,32 +97,29 @@ def call(args, env=None, cwd=None, shell=False):
         call(['pip', 'install', 'wheel'], env=env)
         call(['pip', 'install', 'delocate'], env=env)
 
-        # setup dirs
-        if os.path.exists('/tmp/built_wheel'):
-            shutil.rmtree('/tmp/built_wheel')
-        os.makedirs('/tmp/built_wheel')
-        if os.path.exists('/tmp/delocated_wheel'):
-            shutil.rmtree('/tmp/delocated_wheel')
-        os.makedirs('/tmp/delocated_wheel')
-
         # run the before_build command
         if before_build:
             before_build_prepared = prepare_command(before_build, project=abs_project_dir)
             call(before_build_prepared, env=env, shell=True)
 
         # build the wheel
-        call(['pip', 'wheel', abs_project_dir, '-w', '/tmp/built_wheel', '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
-        built_wheel = glob('/tmp/built_wheel/*.whl')[0]
-
-        if built_wheel.endswith('none-any.whl'):
-            # pure python wheel - just move
-            shutil.move(built_wheel, '/tmp/delocated_wheel')
+        if os.path.exists(built_wheel_dir):
+            shutil.rmtree(built_wheel_dir)
+        os.makedirs(built_wheel_dir)
+        call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
+        built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
+
+        # repair the wheel
+        if os.path.exists(repaired_wheel_dir):
+            shutil.rmtree(repaired_wheel_dir)
+        os.makedirs(repaired_wheel_dir)
+        if built_wheel.endswith('none-any.whl') or not repair_command:
+            # pure Python wheel or empty repair command
+            shutil.move(built_wheel, repaired_wheel_dir)
         else:
-            # list the dependencies
-            call(['delocate-listdeps', built_wheel], env=env)
-            # rebuild the wheel with shared libraries included and place in output dir
-            call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env)
-        delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0]
+            repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
+            call(repair_command_prepared, env=env, shell=True)
+        repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
 
         if test_command:
             # set up a virtual environment to install and test from, to make sure
@@ -141,7 +141,7 @@ def call(args, env=None, cwd=None, shell=False):
             call(['which', 'python'], env=virtualenv_env)
 
             # install the wheel
-            call(['pip', 'install', delocated_wheel + test_extras], env=virtualenv_env)
+            call(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
 
             # test the wheel
             if test_requires:
@@ -157,5 +157,5 @@ def call(args, env=None, cwd=None, shell=False):
             shutil.rmtree(venv_dir)
 
         # we're all done here; move it to output (overwrite existing)
-        dst = os.path.join(output_dir, os.path.basename(delocated_wheel))
-        shutil.move(delocated_wheel, dst)
+        dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
+        shutil.move(repaired_wheel, dst)
diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
@@ -2,14 +2,14 @@
 import warnings
 
 
-def prepare_command(command, project):
+def prepare_command(command, **kwargs):
     '''
-    Preprocesses a command by expanding variables like {project}.
+    Preprocesses a command by expanding variables like {python}.
 
-    For example, used in the test_command option, to specify the path to the
-    tests directory.
+    For example, used in the test_command option to specify the path to the
+    project's root.
     '''
-    return command.format(python='python', pip='pip', project=project)
+    return command.format(python='python', pip='pip', **kwargs)
 
 
 def get_build_verbosity_extra_flags(level):
diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py
@@ -57,8 +57,7 @@ def get_python_configurations(build_selector):
     return python_configurations
 
 
-
-def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
+def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
     def simple_shell(args, env=None, cwd=None):
         print('+ ' + ' '.join(args))
         args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
@@ -90,6 +89,7 @@ def shell(args, env=None, cwd=None):
     abs_project_dir = os.path.abspath(project_dir)
     temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
     built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
 
     # install nuget as best way to provide python
     nuget = 'C:\\cibw\\nuget.exe'
@@ -109,11 +109,6 @@ def shell(args, env=None, cwd=None):
         assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
         assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe'))
 
-        # setup dirs
-        if os.path.exists(built_wheel_dir):
-            shutil.rmtree(built_wheel_dir)
-        os.makedirs(built_wheel_dir)
-
         env = os.environ.copy()
         # set up environment variables for run_with_env
         env['PYTHON_VERSION'] = config.version
@@ -140,8 +135,23 @@ def shell(args, env=None, cwd=None):
             shell([before_build_prepared], env=env)
 
         # build the wheel
+        if os.path.exists(built_wheel_dir):
+            shutil.rmtree(built_wheel_dir)
+        os.makedirs(built_wheel_dir)
         shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
-        built_wheel = glob(built_wheel_dir+'/*.whl')[0]
+        built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
+
+        # repair the wheel
+        if os.path.exists(repaired_wheel_dir):
+            shutil.rmtree(repaired_wheel_dir)
+        os.makedirs(repaired_wheel_dir)
+        if built_wheel.endswith('none-any.whl') or not repair_command:
+            # pure Python wheel or empty repair command
+            shutil.move(built_wheel, repaired_wheel_dir)
+        else:
+            repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
+            shell([repair_command_prepared], env=env)
+        repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
 
         if test_command:
             # set up a virtual environment to install and test from, to make sure
@@ -160,7 +170,7 @@ def shell(args, env=None, cwd=None):
             shell(['which', 'python'], env=virtualenv_env)
 
             # install the wheel
-            shell(['pip', 'install', built_wheel + test_extras], env=virtualenv_env)
+            shell(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
 
             # test the wheel
             if test_requires:
@@ -176,7 +186,7 @@ def shell(args, env=None, cwd=None):
             shutil.rmtree(venv_dir)
 
         # we're all done here; move it to output (remove if already exists)
-        dst = os.path.join(output_dir, os.path.basename(built_wheel))
+        dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
         if os.path.isfile(dst):
             os.remove(dst)
-        shutil.move(built_wheel, dst)
+        shutil.move(repaired_wheel, dst)
diff --git a/docs/options.md b/docs/options.md
