In place cmake build + rpath on linux, at least

Author: wesm

CMakeLists.txt

@@ -228,7 +228,9 @@ if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
 EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
   ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
 else()
-  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+  # set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+
 endif()
 
 # where to put generated archives (.a files)
@@ -587,6 +589,9 @@ foreach(pyx_api_file
     set_source_files_properties(${pyx_api_file} PROPERTIES CYTHON_API 1)
 endforeach(pyx_api_file)
 
+set(USE_RELATIVE_RPATH ON)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+
 foreach(module native)
     string(REPLACE "." ";" directories ${module})
     list(GET directories -1 module_name)
@@ -617,6 +622,10 @@ foreach(module native)
         set(module_install_rpath "${module_install_rpath}/..")
         math(EXPR i "${i} - 1" )
     endwhile(${i} GREATER 0)
+
+	# for inplace development for now
+    set(module_install_rpath "${CMAKE_SOURCE_DIR}/pandas/")
+
     set_target_properties(${module_name} PROPERTIES
       INSTALL_RPATH ${module_install_rpath})
     target_link_libraries(${module_name} pandas)

setup.py

@@ -6,13 +6,17 @@
 BSD license. Parts are from lxml (https://github.com/lxml/lxml)
 """
 
+import glob
 import os
+import os.path as osp
 import sys
 import shutil
-import warnings
 import re
 import platform
 from distutils.version import LooseVersion
+from distutils import sysconfig
+
+from os.path import join as pjoin
 
 # versioning
 import versioneer
@@ -26,6 +30,7 @@
 except ImportError:
     _CYTHON_INSTALLED = False
 
+
 try:
     import pkg_resources
     from setuptools import setup, Command
@@ -35,6 +40,7 @@
     from distutils.core import setup, Command
     _have_setuptools = False
 
+
 setuptools_kwargs = {}
 min_numpy_ver = '1.7.0'
 if sys.version_info[0] >= 3:
@@ -53,46 +59,199 @@
 else:
     setuptools_kwargs = {
         'install_requires': ['python-dateutil',
-                            'pytz >= 2011k',
+                             'pytz >= 2011k',
                              'numpy >= %s' % min_numpy_ver],
         'setup_requires': ['numpy >= %s' % min_numpy_ver],
         'zip_safe': False,
     }
 
     if not _have_setuptools:
         try:
-            import numpy
-            import dateutil
+            import numpy  # noqa
+            import dateutil  # noqa
             setuptools_kwargs = {}
         except ImportError:
             sys.exit("install requires: 'python-dateutil < 2','numpy'."
                      "  use pip or easy_install."
                      "\n   $ pip install 'python-dateutil < 2' 'numpy'")
 
+
+# Check if we're running 64-bit Python
+is_64_bit = sys.maxsize > 2**32
+
+# Check if this is a debug build of Python.
+if hasattr(sys, 'gettotalrefcount'):
+    build_type = 'Debug'
+else:
+    build_type = 'Release'
+
+
 from distutils.extension import Extension
 from distutils.command.build import build
 from distutils.command.build_ext import build_ext as _build_ext
 
 try:
     if not _CYTHON_INSTALLED:
         raise ImportError('No supported version of Cython installed.')
-    from Cython.Distutils import build_ext as _build_ext
+    from Cython.Distutils import build_ext as _build_ext  # noqa
     cython = True
 except ImportError:
     cython = False
 
-from os.path import join as pjoin
-
 
 class build_ext(_build_ext):
+
     def build_extensions(self):
         numpy_incl = pkg_resources.resource_filename('numpy', 'core/include')
 
         for ext in self.extensions:
-            if hasattr(ext, 'include_dirs') and not numpy_incl in ext.include_dirs:
+            if (hasattr(ext, 'include_dirs') and
+                    numpy_incl not in ext.include_dirs):
                 ext.include_dirs.append(numpy_incl)
         _build_ext.build_extensions(self)
 
+    def run(self):
+        self._run_cmake()
+        _build_ext.run(self)
+
+    # adapted from cmake_build_ext in dynd-python
+    # github.com/libdynd/dynd-python
+
+    description = "Build the C-extension for libpandas and regular pandas"
+    user_options = ([('extra-cmake-args=', None,
+                      'extra arguments for CMake')] +
+                    _build_ext.user_options)
+
+    def initialize_options(self):
+        _build_ext.initialize_options(self)
+        self.extra_cmake_args = ''
+
+    def _run_cmake(self):
+        # The directory containing this setup.py
+        source = osp.dirname(osp.abspath(__file__))
+
+        # The staging directory for the module being built
+        build_temp = pjoin(os.getcwd(), self.build_temp)
+
+        # Change to the build directory
+        saved_cwd = os.getcwd()
+        if not os.path.isdir(self.build_temp):
+            self.mkpath(self.build_temp)
+        os.chdir(self.build_temp)
+
+        # Detect if we built elsewhere
+        if os.path.isfile('CMakeCache.txt'):
+            cachefile = open('CMakeCache.txt', 'r')
+            cachedir = re.search('CMAKE_CACHEFILE_DIR:INTERNAL=(.*)',
+                                 cachefile.read()).group(1)
+            cachefile.close()
+            if (cachedir != build_temp):
+                return
+
+        pyexe_option = '-DPYTHON_EXECUTABLE=%s' % sys.executable
+        static_lib_option = ''
+        build_tests_option = ''
+
+        if sys.platform != 'win32':
+            cmake_command = ['cmake', self.extra_cmake_args, pyexe_option,
+                             build_tests_option,
+                             static_lib_option, source]
+
+            self.spawn(cmake_command)
+            self.spawn(['make'])
+        else:
+            import shlex
+            cmake_generator = 'Visual Studio 14 2015'
+            if is_64_bit:
+                cmake_generator += ' Win64'
+            # Generate the build files
+            extra_cmake_args = shlex.split(self.extra_cmake_args)
+            cmake_command = (['cmake'] + extra_cmake_args +
+                             [source, pyexe_option,
+                              static_lib_option,
+                              build_tests_option,
+                             '-G', cmake_generator])
+            if "-G" in self.extra_cmake_args:
+                cmake_command = cmake_command[:-2]
+
+            self.spawn(cmake_command)
+            # Do the build
+            self.spawn(['cmake', '--build', '.', '--config', build_type])
+
+        if self.inplace:
+            # a bit hacky
+            build_lib = saved_cwd
+        else:
+            build_lib = pjoin(os.getcwd(), self.build_lib)
+
+        # Move the built libpandas library to the place expected by the Python
+        # build
+        if sys.platform != 'win32':
+            name, = glob.glob('libpandas.*')
+            try:
+                os.makedirs(pjoin(build_lib, 'pandas'))
+            except OSError:
+                pass
+            shutil.move(name, pjoin(build_lib, 'pandas', name))
+        else:
+            shutil.move(pjoin(build_type, 'pandas.dll'),
+                        pjoin(build_lib, 'pandas', 'pandas.dll'))
+
+        # Move the built C-extension to the place expected by the Python build
+        self._found_names = []
+        for name in self.get_cmake_cython_names():
+            built_path = self.get_ext_built(name)
+            if not os.path.exists(built_path):
+                raise RuntimeError('libpandas C-extension failed to build:',
+                                   os.path.abspath(built_path))
+
+            ext_path = pjoin(build_lib, self._get_cmake_ext_path(name))
+            if os.path.exists(ext_path):
+                os.remove(ext_path)
+            self.mkpath(os.path.dirname(ext_path))
+            print('Moving built libpandas C-extension', built_path,
+                  'to build path', ext_path)
+            shutil.move(self.get_ext_built(name), ext_path)
+            self._found_names.append(name)
+
+        os.chdir(saved_cwd)
+
+    def _get_inplace_dir(self):
+        pass
+
+    def _get_cmake_ext_path(self, name):
+        # Get the package directory from build_py
+        build_py = self.get_finalized_command('build_py')
+        package_dir = build_py.get_package_dir('pandas')
+        # This is the name of the pandas C-extension
+        suffix = sysconfig.get_config_var('EXT_SUFFIX')
+        if suffix is None:
+            suffix = sysconfig.get_config_var('SO')
+        filename = name + suffix
+        return pjoin(package_dir, filename)
+
+    def get_ext_built(self, name):
+        if sys.platform == 'win32':
+            head, tail = os.path.split(name)
+            suffix = sysconfig.get_config_var('SO')
+            return pjoin(head, build_type, tail + suffix)
+        else:
+            suffix = sysconfig.get_config_var('SO')
+            return name + suffix
+
+    def get_cmake_cython_names(self):
+        return ['native']
+
+    def get_names(self):
+        return self._found_names
+
+    def get_outputs(self):
+        # Just the C extensions
+        cmake_exts = [self._get_cmake_ext_path(name)
+                      for name in self.get_names()]
+        regular_exts = _build_ext.get_outputs(self)
+        return regular_exts + cmake_exts
+
 
 DESCRIPTION = ("Powerful data structures for data analysis, time series,"
                "and statistics")
@@ -186,6 +345,7 @@ def build_extensions(self):
     'Topic :: Scientific/Engineering',
 ]
 
+
 class CleanCommand(Command):
     """Custom distutils command to clean the .so and .pyc files."""
 
@@ -196,22 +356,22 @@ def initialize_options(self):
         self._clean_me = []
         self._clean_trees = []
 
-        base = pjoin('pandas','src')
-        dt = pjoin(base,'datetime')
+        base = pjoin('pandas', 'src')
+        dt = pjoin(base, 'datetime')
         src = base
-        parser = pjoin(base,'parser')
-        ujson_python = pjoin(base,'ujson','python')
-        ujson_lib = pjoin(base,'ujson','lib')
-        self._clean_exclude = [pjoin(dt,'np_datetime.c'),
-                               pjoin(dt,'np_datetime_strings.c'),
-                               pjoin(src,'period_helper.c'),
-                               pjoin(parser,'tokenizer.c'),
-                               pjoin(parser,'io.c'),
-                               pjoin(ujson_python,'ujson.c'),
-                               pjoin(ujson_python,'objToJSON.c'),
-                               pjoin(ujson_python,'JSONtoObj.c'),
-                               pjoin(ujson_lib,'ultrajsonenc.c'),
-                               pjoin(ujson_lib,'ultrajsondec.c'),
+        parser = pjoin(base, 'parser')
+        ujson_python = pjoin(base, 'ujson', 'python')
+        ujson_lib = pjoin(base, 'ujson', 'lib')
+        self._clean_exclude = [pjoin(dt, 'np_datetime.c'),
+                               pjoin(dt, 'np_datetime_strings.c'),
+                               pjoin(src, 'period_helper.c'),
+                               pjoin(parser, 'tokenizer.c'),
+                               pjoin(parser, 'io.c'),
+                               pjoin(ujson_python, 'ujson.c'),
+                               pjoin(ujson_python, 'objToJSON.c'),
+                               pjoin(ujson_python, 'JSONtoObj.c'),
+                               pjoin(ujson_lib, 'ultrajsonenc.c'),
+                               pjoin(ujson_lib, 'ultrajsondec.c'),
                                ]
 
         for root, dirs, files in os.walk('pandas'):
@@ -252,6 +412,7 @@ def run(self):
 # class as it encodes the version info
 sdist_class = cmdclass['sdist']
 
+
 class CheckSDist(sdist_class):
     """Custom sdist that ensures Cython has compiled all pyx files to c."""
 
@@ -469,13 +630,13 @@ def pxd(name):
 extensions.extend([sparse_ext])
 
 testing_ext = Extension('pandas._testing',
-                       sources=[srcpath('testing', suffix=suffix)],
-                       include_dirs=[],
-                       libraries=libraries)
+                        sources=[srcpath('testing', suffix=suffix)],
+                        include_dirs=[],
+                        libraries=libraries)
 
 extensions.extend([testing_ext])
 
-#----------------------------------------------------------------------
+# ---------------------------------------------------------------------
 # msgpack stuff here
 
 if sys.byteorder == 'big':
@@ -484,31 +645,35 @@ def pxd(name):
     macros = [('__LITTLE_ENDIAN__', '1')]
 
 packer_ext = Extension('pandas.msgpack._packer',
-                        depends=['pandas/src/msgpack/pack.h',
-                                 'pandas/src/msgpack/pack_template.h'],
-                        sources = [srcpath('_packer',
-                                   suffix=suffix if suffix == '.pyx' else '.cpp',
+                       depends=['pandas/src/msgpack/pack.h',
+                                'pandas/src/msgpack/pack_template.h'],
+                       sources=[
+                           srcpath('_packer', suffix=suffix
+                                   if suffix == '.pyx' else '.cpp',
                                    subdir='msgpack')],
-                        language='c++',
-                        include_dirs=['pandas/src/msgpack'] + common_include,
-                        define_macros=macros)
+                       language='c++',
+                       include_dirs=['pandas/src/msgpack'] + common_include,
+                       define_macros=macros)
+
 unpacker_ext = Extension('pandas.msgpack._unpacker',
-                        depends=['pandas/src/msgpack/unpack.h',
-                                 'pandas/src/msgpack/unpack_define.h',
-                                 'pandas/src/msgpack/unpack_template.h'],
-                        sources = [srcpath('_unpacker',
-                                   suffix=suffix if suffix == '.pyx' else '.cpp',
-                                   subdir='msgpack')],
-                        language='c++',
-                        include_dirs=['pandas/src/msgpack'] + common_include,
-                        define_macros=macros)
+                         depends=['pandas/src/msgpack/unpack.h',
+                                  'pandas/src/msgpack/unpack_define.h',
+                                  'pandas/src/msgpack/unpack_template.h'],
+                         sources=[
+                             srcpath('_unpacker',
+                                     suffix=suffix
+                                     if suffix == '.pyx' else '.cpp',
+                                     subdir='msgpack')],
+                         language='c++',
+                         include_dirs=['pandas/src/msgpack'] + common_include,
+                         define_macros=macros)
 extensions.append(packer_ext)
 extensions.append(unpacker_ext)
 
 if suffix == '.pyx' and 'setuptools' in sys.modules:
     # undo dumb setuptools bug clobbering .pyx sources back to .c
     for ext in extensions:
-        if ext.sources[0].endswith(('.c','.cpp')):
+        if ext.sources[0].endswith(('.c', '.cpp')):
             root, _ = os.path.splitext(ext.sources[0])
             ext.sources[0] = root + suffix