From 76667659a6d43a895bc87e3def6da4ff1ca3701f Mon Sep 17 00:00:00 2001 From: Matthew Kirk Date: Thu, 15 Feb 2018 07:54:47 +0700 Subject: [PATCH 1/6] Fix the non cython build for cpp extensions --- setup.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index c66979dd19ef0..e99ead7216e02 100755 --- a/setup.py +++ b/setup.py @@ -311,7 +311,6 @@ class CheckSDist(sdist_class): 'pandas/_libs/missing.pyx', 'pandas/_libs/reduction.pyx', 'pandas/_libs/testing.pyx', - 'pandas/_libs/window.pyx', 'pandas/_libs/skiplist.pyx', 'pandas/_libs/sparse.pyx', 'pandas/_libs/parsers.pyx', @@ -331,6 +330,8 @@ class CheckSDist(sdist_class): 'pandas/_libs/writers.pyx', 'pandas/io/sas/sas.pyx'] + _cpp_pyxfiles =['pandas/_libs/window.pyx'] + def initialize_options(self): sdist_class.initialize_options(self) @@ -338,12 +339,15 @@ def run(self): if 'cython' in cmdclass: self.run_command('cython') else: - for pyxfile in self._pyxfiles: - cfile = pyxfile[:-3] + 'c' - msg = ("C-source file '{source}' not found.\n" - "Run 'setup.py cython' before sdist.".format( - source=cfile)) - assert os.path.isfile(cfile), msg + pyx_files = [(self._pyxfiles, 'c'), (self._cpp_pyxfiles, 'cpp')] + + for pyxfiles, extension in pyx_files: + for pyxfile in pyxfiles: + sourcefile = pyxfile[:-3] + extension + msg = ("{extension}-source file '{source}' not found.\n" + "Run 'setup.py cython' before sdist.".format( + source=cfile, extension=extension)) + assert os.path.isfile(sourcefile), msg sdist_class.run(self) @@ -618,7 +622,8 @@ def pxd(name): '_libs.window': { 'pyxfile': '_libs/window', 'pxdfiles': ['_libs/skiplist', '_libs/src/util'], - 'language': 'c++'}, + 'language': 'c++', + 'suffix': '.cpp'}, '_libs.writers': { 'pyxfile': '_libs/writers', 'pxdfiles': ['_libs/src/util']}, @@ -628,7 +633,9 @@ def pxd(name): extensions = [] for name, data in ext_data.items(): - sources = [srcpath(data['pyxfile'], suffix=suffix, subdir='')] + source_suffix = suffix if suffix == '.pyx' else data.get('suffix', '.c') + + sources = [srcpath(data['pyxfile'], suffix=source_suffix, subdir='')] pxds = [pxd(x) for x in data.get('pxdfiles', [])] if suffix == '.pyx' and pxds: sources.extend(pxds) From 0b0f31861ced990e5f370fab580bea674a9b73e2 Mon Sep 17 00:00:00 2001 From: Matthew Kirk Date: Thu, 15 Feb 2018 07:58:53 +0700 Subject: [PATCH 2/6] sourcefile should be here not cfile --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e99ead7216e02..b2ea149dd5b26 100755 --- a/setup.py +++ b/setup.py @@ -346,7 +346,7 @@ def run(self): sourcefile = pyxfile[:-3] + extension msg = ("{extension}-source file '{source}' not found.\n" "Run 'setup.py cython' before sdist.".format( - source=cfile, extension=extension)) + source=sourcefile, extension=extension)) assert os.path.isfile(sourcefile), msg sdist_class.run(self) From ea7c9e2860a72bb1d9f082c84bf3bead51f77e94 Mon Sep 17 00:00:00 2001 From: Matthew Kirk Date: Thu, 15 Feb 2018 07:59:41 +0700 Subject: [PATCH 3/6] Fix linting error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2ea149dd5b26..a0046ddbc5bbf 100755 --- a/setup.py +++ b/setup.py @@ -330,7 +330,7 @@ class CheckSDist(sdist_class): 'pandas/_libs/writers.pyx', 'pandas/io/sas/sas.pyx'] - _cpp_pyxfiles =['pandas/_libs/window.pyx'] + _cpp_pyxfiles = ['pandas/_libs/window.pyx'] def initialize_options(self): sdist_class.initialize_options(self) From d1e1a2ee62c52d2b06cf8d1044a0ffe478e0b4bf Mon Sep 17 00:00:00 2001 From: Matthew Kirk Date: Sat, 17 Feb 2018 09:07:11 +0700 Subject: [PATCH 4/6] Build msgpack in the same extension loop --- setup.py | 67 +++++++++++++++++++++++++------------------------------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/setup.py b/setup.py index a0046ddbc5bbf..524d44ea2490e 100755 --- a/setup.py +++ b/setup.py @@ -421,6 +421,11 @@ def get_tag(self): cmdclass['build_src'] = DummyBuildSrc cmdclass['build_ext'] = CheckingBuildExt +if sys.byteorder == 'big': + endian_macro = [('__BIG_ENDIAN__', '1')] +else: + endian_macro = [('__LITTLE_ENDIAN__', '1')] + lib_depends = ['inference'] @@ -457,6 +462,7 @@ def pxd(name): 'pandas/_libs/src/datetime/np_datetime_strings.h'] np_datetime_sources = ['pandas/_libs/src/datetime/np_datetime.c', 'pandas/_libs/src/datetime/np_datetime_strings.c'] + tseries_depends = np_datetime_headers + ['pandas/_libs/tslibs/np_datetime.pxd'] # some linux distros require it @@ -628,7 +634,28 @@ def pxd(name): 'pyxfile': '_libs/writers', 'pxdfiles': ['_libs/src/util']}, 'io.sas._sas': { - 'pyxfile': 'io/sas/sas'}} + 'pyxfile': 'io/sas/sas'}, + 'io.msgpack._packer': { + 'macros': endian_macro, + 'depends': ['pandas/_libs/src/msgpack/pack.h', + 'pandas/_libs/src/msgpack/pack_template.h'], + 'include': ['pandas/_libs/src/msgpack'] + common_include, + 'language': 'c++', + 'suffix': '.cpp', + 'pyxfile': 'io/msgpack/_packer', + 'subdir': 'io/msgpack'}, + 'io.msgpack._unpacker': { + 'depends': ['pandas/_libs/src/msgpack/unpack.h', + 'pandas/_libs/src/msgpack/unpack_define.h', + 'pandas/_libs/src/msgpack/unpack_template.h'], + 'macros': endian_macro, + 'include': ['pandas/_libs/src/msgpack'] + common_include, + 'language': 'c++', + 'suffix': '.cpp', + 'pyxfile': 'io/msgpack/_unpacker', + 'subdir': 'io/msgpack' + } +} extensions = [] @@ -636,6 +663,7 @@ def pxd(name): source_suffix = suffix if suffix == '.pyx' else data.get('suffix', '.c') sources = [srcpath(data['pyxfile'], suffix=source_suffix, subdir='')] + pxds = [pxd(x) for x in data.get('pxdfiles', [])] if suffix == '.pyx' and pxds: sources.extend(pxds) @@ -649,46 +677,11 @@ def pxd(name): depends=data.get('depends', []), include_dirs=include, language=data.get('language', 'c'), + define_macros=data.get('macros', []), extra_compile_args=extra_compile_args) extensions.append(obj) -# ---------------------------------------------------------------------- -# msgpack - -if sys.byteorder == 'big': - macros = [('__BIG_ENDIAN__', '1')] -else: - macros = [('__LITTLE_ENDIAN__', '1')] - -msgpack_include = ['pandas/_libs/src/msgpack'] + common_include -msgpack_suffix = suffix if suffix == '.pyx' else '.cpp' -unpacker_depends = ['pandas/_libs/src/msgpack/unpack.h', - 'pandas/_libs/src/msgpack/unpack_define.h', - 'pandas/_libs/src/msgpack/unpack_template.h'] - -packer_ext = Extension('pandas.io.msgpack._packer', - depends=['pandas/_libs/src/msgpack/pack.h', - 'pandas/_libs/src/msgpack/pack_template.h'], - sources=[srcpath('_packer', - suffix=msgpack_suffix, - subdir='io/msgpack')], - language='c++', - include_dirs=msgpack_include, - define_macros=macros, - extra_compile_args=extra_compile_args) -unpacker_ext = Extension('pandas.io.msgpack._unpacker', - depends=unpacker_depends, - sources=[srcpath('_unpacker', - suffix=msgpack_suffix, - subdir='io/msgpack')], - language='c++', - include_dirs=msgpack_include, - define_macros=macros, - extra_compile_args=extra_compile_args) -extensions.append(packer_ext) -extensions.append(unpacker_ext) - # ---------------------------------------------------------------------- # ujson From d3a8757ef871cfb76c4bed68094b9627767be897 Mon Sep 17 00:00:00 2001 From: Matthew Kirk Date: Sat, 17 Feb 2018 09:08:46 +0700 Subject: [PATCH 5/6] add comment --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 524d44ea2490e..29388bb3fe7b7 100755 --- a/setup.py +++ b/setup.py @@ -330,7 +330,9 @@ class CheckSDist(sdist_class): 'pandas/_libs/writers.pyx', 'pandas/io/sas/sas.pyx'] - _cpp_pyxfiles = ['pandas/_libs/window.pyx'] + _cpp_pyxfiles = ['pandas/_libs/window.pyx', + 'pandas/io/msgpack/_packer.pyx', + 'pandas/io/msgpack/_unpacker.pyx'] def initialize_options(self): sdist_class.initialize_options(self) @@ -339,6 +341,7 @@ def run(self): if 'cython' in cmdclass: self.run_command('cython') else: + # If we are not running cython then compile the extensions correctly pyx_files = [(self._pyxfiles, 'c'), (self._cpp_pyxfiles, 'cpp')] for pyxfiles, extension in pyx_files: From 42a80c377756aef1a339194cd46ac915cee94f03 Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Mon, 19 Feb 2018 18:37:53 -0500 Subject: [PATCH 6/6] lint --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 29388bb3fe7b7..c7784260d79ca 100755 --- a/setup.py +++ b/setup.py @@ -341,7 +341,8 @@ def run(self): if 'cython' in cmdclass: self.run_command('cython') else: - # If we are not running cython then compile the extensions correctly + # If we are not running cython then + # compile the extensions correctly pyx_files = [(self._pyxfiles, 'c'), (self._cpp_pyxfiles, 'cpp')] for pyxfiles, extension in pyx_files: