diff --git a/projects/hello-complex/CMakeLists.txt b/projects/hello-complex/CMakeLists.txt new file mode 100644 index 0000000..0be80a9 --- /dev/null +++ b/projects/hello-complex/CMakeLists.txt @@ -0,0 +1,48 @@ +cmake_minimum_required(VERSION 3.13.0) +project(my_skb_mod LANGUAGES C) + + +find_package(PythonInterp REQUIRED) +find_package(PythonLibs REQUIRED) + + +### +# Private helper function to execute `python -c ""` +# +# Runs a python command and populates an outvar with the result of stdout. +# Be careful of indentation if `cmd` is multiline. +# +function(pycmd outvar cmd) + execute_process( + COMMAND "${PYTHON_EXECUTABLE}" -c "${cmd}" + RESULT_VARIABLE _exitcode + OUTPUT_VARIABLE _output) + if(NOT ${_exitcode} EQUAL 0) + message(ERROR "Failed when running python code: \"\"\" +${cmd}\"\"\"") + message(FATAL_ERROR "Python command failed with error code: ${_exitcode}") + endif() + # Remove supurflous newlines (artifacts of print) + string(STRIP "${_output}" _output) + set(${outvar} "${_output}" PARENT_SCOPE) +endfunction() + +### +# Find scikit-build and include its cmake resource scripts +# +if (NOT SKBUILD) + pycmd(skbuild_location "import os, skbuild; print(os.path.dirname(skbuild.__file__))") + set(skbuild_cmake_dir "${skbuild_location}/resources/cmake") + # If skbuild is not the driver, then we need to include its utilities in our CMAKE_MODULE_PATH + list(APPEND CMAKE_MODULE_PATH ${skbuild_cmake_dir}) +endif() + +find_package(PythonExtensions REQUIRED) +find_package(Cython REQUIRED) +find_package(NumPy REQUIRED) + +# Backend C library +add_subdirectory("src/cxx") + +# Cython library +add_subdirectory("src/python/my_skb_mod") diff --git a/projects/hello-complex/demo_issue.sh b/projects/hello-complex/demo_issue.sh new file mode 100644 index 0000000..cf4e48f --- /dev/null +++ b/projects/hello-complex/demo_issue.sh @@ -0,0 +1,79 @@ + +run_common_module_tests(){ + echo "Running common tests of the installed module" + + echo "Package Version: $(python -c 'import my_skb_mod; print(my_skb_mod.__version__)')" + echo "Python Package Location: $(python -c 'import my_skb_mod; print(my_skb_mod.__file__)')" + echo "Compiled Cython Module: $(python -c 'from my_skb_mod import _myalgo_cython; print(_myalgo_cython.__file__)')" + # Test that we can access the module resource + python -c "import my_skb_mod; print(my_skb_mod.submod.get_module_resouce())" + + echo "Finished common tests of the installed module" + + python -c "import my_skb_mod; print(my_skb_mod.__doc__)" + + +} + + +####################### +# WHEEL MODE TEST +####################### + +# First test with a regular wheel +pip wheel . +WHEEL_FPATH="$(ls my_skb_mod-*.whl)" +echo "WHEEL_FPATH = $WHEEL_FPATH" +# Should see the submodule resource here, but we don't +unzip -l "$WHEEL_FPATH" + +# Install the wheel +pip uninstall -y "my_skb_mod" || echo "already uninstalled" +pip install "$WHEEL_FPATH" + +# Run the module tests +# This does not include the package data correctly +run_common_module_tests + + +####################### +# DEVELOPMENT MODE TEST +####################### + +# Clean up +pip uninstall -y "my_skb_mod" || echo "already uninstalled" +rm -rf _skbuild || echo "already clean" +rm -rf src/python/my_skb_mod/*.so || echo "already clean" + +# Test in development mode +# FIXME: running ``pip install -e .`` multiple times breaks! +pip install --verbose -e . + +# The egg link should point to +# .../issue-xxx-multi-package-with-package-dir/src/python +# +# NOT +# +# .../issue-xxx-multi-package-with-package-dir/ +SITE_PACKAGE_DPATH="$(python -c 'import distutils.sysconfig; print(distutils.sysconfig.get_python_lib())')" +echo "SITE_PACKAGE_DPATH = $SITE_PACKAGE_DPATH" +cat "$SITE_PACKAGE_DPATH/my_skb_mod.egg-link" +# but even with the packages=[''] hack, there seems to be an extra ../../.. term +# not sure what's going on. + +run_common_module_tests + + + +####################### +# DEMO MULTIPLE EDIBALE PIP INSTALL ISSUE +####################### + +# Maybe this isn't an issue here? It is an issue in the project I'm basing this +# on cant run this command multiple times. I dont know why. +# This only seeps to break if the "packages" directory is specified as +# `packages=['my_skb_mod']`. Otherwise it does seem to work +pip uninstall -y "my_skb_mod" || echo "already uninstalled" +pip uninstall -y "my_skb_mod" || echo "already uninstalled" +pip install -e . +pip install -e . diff --git a/projects/hello-complex/setup.py b/projects/hello-complex/setup.py new file mode 100644 index 0000000..9bc4052 --- /dev/null +++ b/projects/hello-complex/setup.py @@ -0,0 +1,24 @@ +""" +MWE for issue +""" +from skbuild import setup + +if __name__ == '__main__': + setup( + package_dir={ + '': 'src/python/', + }, + name='my_skb_mod', + version='0.1.0', + # develop mode doesnt work without this hack + # packages=['', 'my_skb_mod', 'my_skb_mod.submod'], # sort of works? but also warns? + packages=['my_skb_mod', 'my_skb_mod.submod'], # works for wheel install + # packages=['my_skb_mod'], # works for develop install + + # This package data seems like it is not respected. + package_data={ + 'my_skb_mod.submod': [ + 'module_resouce.json' + ], + }, + ) diff --git a/projects/hello-complex/src/cxx/CMakeLists.txt b/projects/hello-complex/src/cxx/CMakeLists.txt new file mode 100644 index 0000000..e322aac --- /dev/null +++ b/projects/hello-complex/src/cxx/CMakeLists.txt @@ -0,0 +1,67 @@ +# TODO: Can we make this work in either shared or static mode? + +option(BUILD_MECL "Enable MECL (my extern c lib) library" TRUE) +if (BUILD_MECL) + + set(THREADS_PREFER_PTHREAD_FLAG ON) + + # TODO: include external deps in mecl + + # Find required external libraries + find_package(Threads REQUIRED) + #find_package(ZLIB REQUIRED) + #find_package(GSL REQUIRED) + #find_package(OpenMP) + + # Corresponds to FFLAGS + #set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -fPIC") + + # Hacks: + #set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + + #add_library(GLMnet STATIC "GLMnet.f") + + set(MECL_MODULE_NAME "mecl") + # Add other C sources + list(APPEND mecl_sources "my_extern_clib.c") + ## Create C++ library. Specify include dirs and link libs as normal + #add_library(${MECL_MODULE_NAME} SHARED ${mecl_sources}) + + # HACK: statically link the library to the cython module because + # I'm having trouble making the shared library work. + add_library(${MECL_MODULE_NAME} STATIC ${mecl_sources}) + + #target_include_directories( + # ${MECL_MODULE_NAME} + # PUBLIC + # ${NumPy_INCLUDE_DIRS} + # ${PYTHON_INCLUDE_DIRS} + # ${CMAKE_CURRENT_SOURCE_DIR} + #) + + #message(STATUS "OpenMP::OpenMP_C = ${OpenMP::OpenMP_C}") + #message(STATUS "ZLIB::ZLIB = ${ZLIB::ZLIB}") + #message(STATUS "Threads::Threads = ${Threads::Threads}") + message(STATUS "Threads = ${Threads}") + + target_link_libraries( + ${MECL_MODULE_NAME} PUBLIC + #ZLIB::ZLIB + Threads::Threads + ) + + #target_compile_definitions(${SCCD_MODULE_NAME} PUBLIC + # "NPY_NO_DEPRECATED_API" + # #"NPY_1_7_API_VERSION=0x00000007" + # ) + + # Transform the C++ library into an importable python module + #python_extension_module(${SCCD_MODULE_NAME}) + + # Install the C++ module to the correct relative location + # (this will be an inplace build if you use `pip install -e`) + #file(RELATIVE_PATH _install_dest "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}") + # Is this right? + #install(TARGETS ${SCCD_MODULE_NAME} LIBRARY DESTINATION "${_install_dest}../tool/python/pycold") + +endif() diff --git a/projects/hello-complex/src/cxx/my_extern_clib.c b/projects/hello-complex/src/cxx/my_extern_clib.c new file mode 100644 index 0000000..87b814e --- /dev/null +++ b/projects/hello-complex/src/cxx/my_extern_clib.c @@ -0,0 +1,9 @@ +#include "my_extern_clib.h" + +long external_c_function(long* data_view, int size){ + long result = 0; + for (int i = 0; i < size; i++){ + result += data_view[i]; + } + return result; +} diff --git a/projects/hello-complex/src/cxx/my_extern_clib.h b/projects/hello-complex/src/cxx/my_extern_clib.h new file mode 100644 index 0000000..92d60bc --- /dev/null +++ b/projects/hello-complex/src/cxx/my_extern_clib.h @@ -0,0 +1,6 @@ +#ifndef MYEXTERN_CLIB_H +#define MYEXTERN_CLIB_H + +long external_c_function(long* data_view, int size); + +#endif diff --git a/projects/hello-complex/src/python/my_skb_mod/CMakeLists.txt b/projects/hello-complex/src/python/my_skb_mod/CMakeLists.txt new file mode 100644 index 0000000..db17859 --- /dev/null +++ b/projects/hello-complex/src/python/my_skb_mod/CMakeLists.txt @@ -0,0 +1,47 @@ +# Setup basic python stuff and ensure we have skbuild + +option(BUILD_MYALGO "Enable _myalgo_cython module" TRUE) +if (BUILD_MYALGO) + + set(cython_source "_myalgo_cython.pyx") + set(MYALGO_MODULE_NAME "_myalgo_cython") + + # Translate Cython into C/C++ + add_cython_target(${MYALGO_MODULE_NAME} "${cython_source}" C OUTPUT_VAR sources) + + # Add other C sources + list(APPEND sources ) + + # Create C++ library. Specify include dirs and link libs as normal + add_library(${MYALGO_MODULE_NAME} MODULE ${sources}) + target_include_directories( + ${MYALGO_MODULE_NAME} + PUBLIC + ${NumPy_INCLUDE_DIRS} + ${PYTHON_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} + ) + + # TODO: not sure why this isn't set in the global scope? + # Hack around it: just hard code the module name + set(MECL_MODULE_NAME "mecl") + + # TODO: linking to the SCCD shared object isn't working 100% yet. + target_link_libraries(${MYALGO_MODULE_NAME} ${MECL_MODULE_NAME}) + + target_compile_definitions(${MYALGO_MODULE_NAME} PUBLIC + "NPY_NO_DEPRECATED_API" + #"NPY_1_7_API_VERSION=0x00000007" + ) + + # Transform the C++ library into an importable python module + python_extension_module(${MYALGO_MODULE_NAME}) + + # My "normal" method of setting install targets does not seem to work here. Hacking it. + # NOTE: skbuild *seems* to place libraries in a data dir *unless* the install destination + # corresponds exactly to the / specified implicitly in setup.py + set(my_skb_mod_install_dest "src/python/my_skb_mod") + + install(TARGETS ${MYALGO_MODULE_NAME} LIBRARY DESTINATION "${my_skb_mod_install_dest}") + +endif() diff --git a/projects/hello-complex/src/python/my_skb_mod/__init__.py b/projects/hello-complex/src/python/my_skb_mod/__init__.py new file mode 100644 index 0000000..1af66eb --- /dev/null +++ b/projects/hello-complex/src/python/my_skb_mod/__init__.py @@ -0,0 +1,18 @@ +""" +My ScikitBuild Module Example +""" +__version__ = '0.1.0' + +try: + from . import submod +except Exception: + print('ERROR submod NOT INCLUDED IN PACAKGE!') + submod = None + +try: + from ._my_skb_mod_cython import cython_func +except Exception: + print('ERROR _my_skb_mod_cython NOT INCLUDED IN PACAKGE!') + cython_func = None + +__all__ = ['cython_func', 'submod'] diff --git a/projects/hello-complex/src/python/my_skb_mod/_myalgo_cython.pyx b/projects/hello-complex/src/python/my_skb_mod/_myalgo_cython.pyx new file mode 100644 index 0000000..b5784f3 --- /dev/null +++ b/projects/hello-complex/src/python/my_skb_mod/_myalgo_cython.pyx @@ -0,0 +1,17 @@ +import numpy as np +cimport numpy as np + + +cdef extern from "../../cxx/my_extern_clib.h": + cdef long external_c_function(long* data_view, int size); + + +def cython_func(np.ndarray[np.int64_t, ndim=1] data): + """ + This is a docstring + """ + cdef long [:] data_view = data + cdef int size = len(data) + cdef long result = 0; + result = external_c_function(&data_view[0], size) + return result diff --git a/projects/hello-complex/src/python/my_skb_mod/submod/__init__.py b/projects/hello-complex/src/python/my_skb_mod/submod/__init__.py new file mode 100644 index 0000000..e141431 --- /dev/null +++ b/projects/hello-complex/src/python/my_skb_mod/submod/__init__.py @@ -0,0 +1,6 @@ +def get_module_resource(): + from importlib import resources as importlib_resources + import json + file = importlib_resources.open_text('my_skb_mod.submod', 'module_resource.json') + data = json.load(file) + return data diff --git a/projects/hello-complex/src/python/my_skb_mod/submod/module_resource.json b/projects/hello-complex/src/python/my_skb_mod/submod/module_resource.json new file mode 100644 index 0000000..0e4d0f0 --- /dev/null +++ b/projects/hello-complex/src/python/my_skb_mod/submod/module_resource.json @@ -0,0 +1,3 @@ +{ + "some-key": "some-value" +}