Skip to content

Commit 0d576f5

Browse files
sc1fsc1f
committed
Build minimal arrow for python
1 parent a2841d4 commit 0d576f5

File tree

3 files changed

+59
-69
lines changed

3 files changed

+59
-69
lines changed

‎cmake/arrow/CMakeLists.txt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,22 @@ set(ARROW_SRCS
101101
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/options.cc
102102
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/writer.cc)
103103

104+
if (PSP_PYTHON_BUILD)
105+
set(ARROW_SRCS
106+
${ARROW_SRCS}
107+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/datum.cc
108+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/file.cc
109+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/coo_converter.cc
110+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csf_converter.cc
111+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csx_converter.cc
112+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/formatting.cc
113+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/time.cc
114+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum-dtoa.cc
115+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fast-dtoa.cc
116+
${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fixed-dtoa.cc
117+
)
118+
endif()
119+
104120
set_property(SOURCE util/io_util.cc
105121
APPEND_STRING
106122
PROPERTY COMPILE_FLAGS " -Wno-unused-macros -stdlib=libc++")

‎cpp/perspective/CMakeLists.txt‎

Lines changed: 26 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ function (psp_build_message message)
4242
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${message}")
4343
endfunction()
4444

45-
###############################
46-
# Helper to grab dependencies #
47-
###############################
45+
###################################################
46+
# Helper to grab dependencies from remote sources #
47+
###################################################
4848
function (psp_build_dep name cmake_file)
4949
if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build)
5050
psp_build_message("${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
@@ -69,7 +69,7 @@ function (psp_build_dep name cmake_file)
6969
endif()
7070

7171
if(${name} STREQUAL arrow)
72-
# Overwrite arrow's CMakeLists with our custom, minimal one
72+
# Overwrite arrow's CMakeLists with our custom, minimal CMakeLists.
7373
configure_file(${PSP_CMAKE_MODULE_PATH}/arrow/CMakeLists.txt ${CMAKE_BINARY_DIR}/arrow-src/cpp/ COPYONLY)
7474
configure_file(${PSP_CMAKE_MODULE_PATH}/arrow/config.h ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/ COPYONLY)
7575
add_subdirectory(${CMAKE_BINARY_DIR}/arrow-src/cpp/
@@ -222,9 +222,6 @@ if (PSP_WASM_BUILD)
222222
# Include this docker-only directory.
223223
include_directories("/boost_includes")
224224

225-
# Build Rapidjson as it is used in the minimal arrow to be built later.
226-
psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in")
227-
228225
set(EXTENDED_FLAGS " \
229226
--bind \
230227
--source-map-base ./build/ \
@@ -315,15 +312,20 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
315312
endif()
316313
endif()
317314

315+
# Boost is a system dependency and must be present and built on the system.
318316
find_package(Boost REQUIRED)
317+
319318
if(NOT Boost_FOUND)
320319
message(FATAL_ERROR "${Red}Boost could not be located${ColorReset}")
321320
else()
322321
psp_build_message("${Cyan}Found Boost: `Boost_INCLUDE_DIRS`: ${Boost_INCLUDE_DIRS}, `Boost_LIBRARY_DIRS` - ${Boost_LIBRARY_DIRS} ${ColorReset}")
323322
include_directories( ${Boost_INCLUDE_DIRS} )
324323
endif()
325324

325+
# TBB is a dependency that we can build as part of our build process, so it
326+
# does not necessarily have to be present before Perspective is built.
326327
find_package(TBB)
328+
327329
if(NOT TBB_FOUND)
328330
psp_build_message("${Red}TBB could not be located - building TBB from external source ${ColorReset}")
329331
psp_build_dep("tbb" "${PSP_CMAKE_MODULE_PATH}/TBB.txt.in")
@@ -415,17 +417,6 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
415417
psp_build_message("${Cyan}Numpy found: ${PYTHON_NUMPY_INCLUDE_DIR}${ColorReset}")
416418
include_directories( ${PYTHON_NUMPY_INCLUDE_DIR})
417419
endif()
418-
419-
find_package(PyArrow REQUIRED)
420-
421-
if(NOT PYTHON_PYARROW_FOUND)
422-
message(FATAL_ERROR "${Red}PyArrow could not be located${ColorReset}")
423-
else()
424-
psp_build_message("${Cyan}PyArrow found: PYTHON_PYARROW_INCLUDE_DIR - ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}")
425-
psp_build_message("${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}")
426-
include_directories(${PYTHON_PYARROW_INCLUDE_DIR})
427-
link_directories(${PYTHON_PYARROW_LIBRARY_DIR})
428-
endif()
429420
#####################
430421
endif()
431422
endif()
@@ -435,21 +426,22 @@ psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in")
435426
psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in")
436427
psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in")
437428

438-
# For WASM/CPP build, build minimal arrow from source
439-
if (NOT PSP_PYTHON_BUILD)
440-
# build arrow + dependencies from source for Emscripten and C++
441-
message("${Cyan}Building minimal Apache Arrow${ColorReset}")
429+
# Build minimal arrow for both Emscripten and Python
430+
message("${Cyan}Building minimal Apache Arrow${ColorReset}")
442431

443-
psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in")
444-
psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in")
432+
# Build arrow dependencies
433+
psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in")
434+
psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in")
445435

446-
find_package(Flatbuffers)
447-
if(NOT FLATBUFFERS_FOUND)
448-
message(FATAL_ERROR"${Red}Flatbuffers could not be located${ColorReset}")
449-
else()
450-
psp_build_message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}")
451-
include_directories( ${FLATBUFFERS_INCLUDE_DIR} )
452-
endif()
436+
# Build minimal arrow itself
437+
psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in")
438+
439+
find_package(Flatbuffers)
440+
if(NOT FLATBUFFERS_FOUND)
441+
message(FATAL_ERROR"${Red}Flatbuffers could not be located${ColorReset}")
442+
else()
443+
psp_build_message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}")
444+
include_directories( ${FLATBUFFERS_INCLUDE_DIR} )
453445
endif()
454446

455447
#####################
@@ -635,25 +627,12 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
635627

636628
# .dll not importable
637629
set_property(TARGET binding PROPERTY SUFFIX .pyd)
638-
elseif (MACOS OR NOT MANYLINUX)
639-
target_compile_options(binding PRIVATE -Wdeprecated-declarations)
640-
641-
# Add a relative path to search for PyArrow - when Perspective is
642-
# installed from a wheel, PyArrow may not be in the same directory
643-
# as the PyArrow which was used to build the wheel.
644-
#
645-
# Assuming that both Pyarrow and Perspective are installed in
646-
# `site-packages`, the relative search path should be able to pick
647-
# up pyarrow. This is only enabled for *nix (non-manylinux), as `auditwheel`
648-
# will not delocate libarrow properly if it is in the rpath.
649-
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${module_origin_path}../../pyarrow/ ${PYTHON_PYARROW_LIBRARY_DIR} ${PSP_PYTHON_ARROWINSTALLDIR})
650-
set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${module_origin_path}../../pyarrow/ ${PYTHON_PYARROW_LIBRARY_DIR} ${PSP_PYTHON_ARROWINSTALLDIR})
651630
else()
652631
target_compile_options(binding PRIVATE -Wdeprecated-declarations)
653632
endif()
654633

655-
target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
656-
target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES})
634+
target_link_libraries(psp arrow)
635+
target_link_libraries(binding arrow)
657636

658637
target_link_libraries(psp tbb)
659638
target_link_libraries(binding tbb)
@@ -669,7 +648,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
669648

670649
if(WIN32)
671650
# inline arrow dlls
672-
file(GLOB ARROW_DLLS "${PYTHON_PYARROW_LIBRARY_DIR}/*.dll")
651+
# file(GLOB ARROW_DLLS "${PYTHON_PYARROW_LIBRARY_DIR}/*.dll")
673652

674653
add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_DLLS} ${PSP_PYTHON_SRC}/table/)
675654
if(NOT TBB_FOUND)

‎docs/md/development.md‎

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -47,29 +47,32 @@ yarn start simple
4747
#### Docker
4848

4949
[Docker](https://docs.docker.com/install/) images with pre-built development
50-
environments are provided for the Javascript and Python libraries.
50+
environments are provided for the Python libraries.
5151

5252
To build Perspective using Docker, select the option in `yarn setup`.
5353

54+
### System Dependencies
55+
56+
`Perspective.js` and `perspective-python` **require** the following system dependencies to be installed:
57+
58+
- [CMake](https://cmake.org/) (version 3.15.4 or higher)
59+
- [Boost](https://www.boost.org/) (version 1.67 or higher, must be built - not header-only)
60+
- [Flatbuffers](https://google.github.io/flatbuffers/flatbuffers_guide_building.html)
61+
5462
## `Perspective.js`
5563

5664
To build the JavaScript library, which includes WebAssembly compilation,
5765
[Emscripten](https://github.com/kripken/emscripten) and its prerequisites are
58-
required. A Docker image is provided with the correct environment and
59-
prerequisites.
66+
required.
6067

61-
#### System Dependencies
62-
63-
Perspective requires some system dependencies to be installed before it can be
64-
built using Emscripten:
65-
66-
- [CMake](https://cmake.org/) (version 3.15.4 or higher)
67-
- [Boost](https://www.boost.org/) (version 1.67 or higher)
68-
- [Flatbuffers](https://google.github.io/flatbuffers/flatbuffers_guide_building.html)
68+
`Perspective.js` specifies its Emscripten version dependency in `package.json`,
69+
and the correct version of Emscripten will be installed with other JS
70+
dependencies by running `yarn`.
6971

7072
#### Building via local EMSDK
7173

72-
To build using local Emscripten,
74+
To build using an Emscripten install on your local system and not the
75+
Emscripten bundled with Perspective in its `package.json`,
7376
[install](https://emscripten.org/docs/getting_started/downloads.html) the
7477
Emscripten SDK, then activate and export the latest `emsdk` environment via
7578
[`emsdk_env.sh`](https://github.com/juj/emsdk):
@@ -78,14 +81,14 @@ Emscripten SDK, then activate and export the latest `emsdk` environment via
7881
source emsdk/emsdk_env.sh
7982
```
8083

81-
We currently use Emscripten version `1.39.13` — deviating from this specific
84+
We currently use Emscripten version `2.0.6` — deviating from this specific
8285
version of Emscripten can introduce various errors that are extremely difficult
8386
to debug.
8487

8588
To install this specific version of Emscripten:
8689

8790
```bash
88-
./emsdk install 1.39.13
91+
./emsdk install 2.0.6
8992
```
9093

9194
## `perspective-python`
@@ -104,14 +107,6 @@ To build the Python 2 version of the library, use the `--python2` flag:
104107
yarn build --python2
105108
```
106109

107-
### System Dependencies
108-
109-
`perspective-python` requires the following system dependencies to be installed before it can be
110-
built from source:
111-
112-
- [CMake](https://cmake.org/) (version 3.15.4 or higher)
113-
- [Boost](https://www.boost.org/) (version 1.67 or higher)
114-
115110
## System-Specific Instructions
116111

117112
### MacOS/OSX

0 commit comments

Comments
 (0)