Add pyodide build

Author: tomjakubowski

.github/workflows/pyodide.yml

@@ -0,0 +1,67 @@
+# Adapted from https://raw.githubusercontent.com/numpy/numpy/main/.github/workflows/emscripten.yml
+name: Pyodide wheel build
+
+on:
+  push:
+    branches:
+      - '*pyodide*'
+  pull_request:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
+jobs:
+  build-wasm-emscripten:
+    runs-on: ubuntu-22.04
+    env:
+      PYODIDE_VERSION: 0.23.2
+      # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION.
+      # The appropriate versions can be found in the Pyodide repodata.json
+      # "info" field, or in Makefile.envs:
+      # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2
+      PYTHON_VERSION: 3.11.2
+      EMSCRIPTEN_VERSION: 3.1.32
+    steps:
+      - name: Checkout perspective
+        uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
+
+      - name: set up python
+        id: setup-python
+        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b # v4.6.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - uses: mymindstorm/setup-emsdk@ab889da2abbcbb280f91ec4c215d3bb4f3a8f775 # v12
+        with:
+          version: ${{ env.EMSCRIPTEN_VERSION }}
+          actions-cache-folder: emsdk-cache
+
+      - name: copy files
+        run: |
+          npm install -g yarn
+          yarn --frozen-lockfile
+          yarn build_python --setup-only
+
+      - name: Linux init steps
+        run: sudo node scripts/install_tools.js
+
+      - name: Install pyodide-build
+        run: pip install pyodide-build==$PYODIDE_VERSION
+
+      - name: Build
+        # Without --exports=pyinit, pyodide-build tries to export all symbols from every .o,
+        # which causes the em++ linker command line invocation to be so long
+        # that the command fails.
+        run: pyodide build python/perspective --exports=pyinit
+
+      - name: Upload pyodide wheel
+        uses: actions/upload-artifact@v3
+        with:
+            name: perspective-python-dist-pyodide
+            path: dist/*.whl

.gitignore

@@ -215,3 +215,4 @@ test-results/
 playwright-report/
 playwright/.cache/
 
+.pyodide-xbuildenv

cmake/arrow/CMakeLists.txt

@@ -109,7 +109,7 @@ set(ARROW_SRCS
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/tracing.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/trie.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/unreachable.cc
-    ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/uri.cc
+    # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/uri.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/utf8.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/value_parsing.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/base64.cpp
@@ -277,11 +277,7 @@ target_compile_definitions(arrow PUBLIC ARROW_STATIC)
 
 # will need built boost filesystem and system .lib to work, even though
 # perspective itself does not use those dependencies
-target_link_libraries(arrow
-    ${double-conversion_LIBRARIES}
-    ${Boost_FILESYSTEM_LIBRARY}
-    ${Boost_SYSTEM_LIBRARY}
-    ${ARROW_TEST_LINK_TOOLCHAIN})
+target_link_libraries(arrow)
 
 # find_package(Flatbuffers)
 

cmake/modules/FindNumPy.cmake

@@ -20,6 +20,13 @@ execute_process(
 
 find_path(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h
   HINTS "${__numpy_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH)
+if(PSP_PYODIDE)
+  # In a pyodide build, find_path fails for some reason, even though
+  #    ${__numpy_path}/numpy/arrayobject.h
+  # exists
+  # execute_process (COMMAND bash -c "ls -l ${__numpy_path}/numpy/arrayobject.h")
+  set(PYTHON_NUMPY_INCLUDE_DIR "${__numpy_path}")
+endif()
 
 if(PYTHON_NUMPY_INCLUDE_DIR)
   set(PYTHON_NUMPY_FOUND 1 CACHE INTERNAL "Python numpy found")

cpp/perspective/CMakeLists.txt

@@ -1,8 +1,7 @@
-cmake_minimum_required(VERSION 3.7.2)
+cmake_minimum_required(VERSION 3.18.2)
 project(psp)
 include(CheckCCompilerFlag)
 
-set(CMAKE_BUILD_TYPE "Release")
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
@@ -108,12 +107,14 @@ endif()
 if(NOT DEFINED PSP_PYTHON_BUILD)
     set(PSP_PYTHON_BUILD OFF)
 elseif(PSP_PYTHON_BUILD)
-    # PSP_PYTHON_BUILD will always run PSP_CPP_BUILD
-    set(PSP_CPP_BUILD ON)
-
     if(NOT DEFINED PSP_PYTHON_VERSION)
         set(PSP_PYTHON_VERSION 3.10)
     endif()
+    if($ENV{PYODIDE})
+        set(PSP_PYODIDE 1)
+    else()
+        set(PSP_PYODIDE 0)
+    endif()
 endif()
 
 if(NOT DEFINED PSP_CPP_BUILD_STRICT)
@@ -212,10 +213,10 @@ if(PSP_WASM_BUILD)
     list(APPEND CMAKE_PREFIX_PATH /usr/local)
 
     # Assumes that Boost includes will be in this folder.
-    include_directories("/usr/local/include" SYSTEM)
+    include_directories(SYSTEM "/usr/local/include")
 
     # Include this docker-only directory.
-    include_directories("/boost_includes")
+    include_directories(SYSTEM "/boost_includes")
 
     set(EXTENDED_FLAGS " \
         -Wall \
@@ -258,28 +259,45 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
         endif()
     else()
         if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
-            set(OPT_FLAGS " \
-                -O0 \
-                -g3 \
-                ")
+            if (PSP_PYODIDE)
+              set(OPT_FLAGS " \
+                  -O0 \
+                  -g3 \
+                  -gsource-map \
+                  --profiling \
+                  -Wcast-align \
+                  -Wover-aligned \
+                  ")
+            else ()
+              set(OPT_FLAGS " \
+                  -O0 \
+                  -g3 \
+                  ")
+            endif ()
         else()
             set(OPT_FLAGS " \
                 -O3 \
                 -g1 \
                 ")
+            if (PSP_PYODIDE)
+              set(OPT_FLAGS "${OPT_FLAGS} -flto")
+            endif ()
         endif()
     endif()
 
     set(ASYNC_MODE_FLAGS "")
 
     # Boost is a system dependency and must be present and built on the system.
+    if(PSP_PYODIDE)
+        set(CMAKE_FIND_ROOT_PATH "${CMAKE_FIND_ROOT_PATH};/usr/local/")
+    endif()
     find_package(Boost REQUIRED)
 
     if(NOT Boost_FOUND)
         message(FATAL_ERROR "${Red}Boost could not be located${ColorReset}")
     else()
         psp_build_message("${Cyan}Found Boost: `Boost_INCLUDE_DIRS`: ${Boost_INCLUDE_DIRS}, `Boost_LIBRARY_DIRS` - ${Boost_LIBRARY_DIRS} ${ColorReset}")
-        include_directories(${Boost_INCLUDE_DIRS})
+        include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 
         if(WIN32)
             add_definitions(-DBOOST_UUID_FORCE_AUTO_LINK)
@@ -316,16 +334,13 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
             psp_build_message("${Red}Manylinux build has no python shared libraries${ColorReset}")
             find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter)
             find_package(PythonHeaders ${PSP_PYTHON_VERSION} EXACT REQUIRED)
-
-            # Run with exact version so its cached for pybind
-            find_package(PythonInterp ${PSP_PYTHON_VERSION} EXACT REQUIRED)
         else()
             psp_build_message("${Cyan}Use python shared libraries${ColorReset}")
-            find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development)
-
-            # Run with exact version so its cached for pybind
-            find_package(PythonInterp ${PSP_PYTHON_VERSION} EXACT REQUIRED)
-            find_package(PythonLibs ${PSP_PYTHON_VERSION} EXACT REQUIRED)
+            if(PSP_PYODIDE)
+                find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter)
+            else()
+                find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module)
+            endif()
 
             link_directories(${Python_LIBRARY_DIRS})
         endif()
@@ -335,7 +350,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 
         psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in")
 
-        find_package(NumPy)
+        find_package(NumPy REQUIRED)
 
         if(NOT PYTHON_NUMPY_FOUND)
             message(FATAL_ERROR "${Red}Numpy could not be located${ColorReset}")
@@ -373,14 +388,18 @@ psp_build_dep("re2" "${PSP_CMAKE_MODULE_PATH}/re2.txt.in")
 psp_build_dep("exprtk" "${PSP_CMAKE_MODULE_PATH}/exprtk.txt.in")
 
 # ####################
-set(CMAKE_C_FLAGS_RELEASE " \
+set(CMAKE_C_FLAGS_DEBUG "")
+set(CMAKE_C_FLAGS " \
     ${CMAKE_C_FLAGS} \
     ${EXTENDED_FLAGS} \
     ${OPT_FLAGS} \
     ")
 
-set(CMAKE_CXX_FLAGS_RELEASE " \
-    ${CMAKE_C_FLAGS} \
+# prevents the default debug flags from overriding the debug flags we
+# set in OPT_FLAGS
+set(CMAKE_CXX_FLAGS_DEBUG "")
+set(CMAKE_CXX_FLAGS " \
+    ${CMAKE_CXX_FLAGS} \
     ${EXTENDED_FLAGS} \
     ${OPT_FLAGS} \
     ")
@@ -410,6 +429,7 @@ set(SOURCE_FILES
     ${PSP_CPP_SRC}/src/cpp/compat.cpp
     ${PSP_CPP_SRC}/src/cpp/compat_impl_linux.cpp
     ${PSP_CPP_SRC}/src/cpp/compat_impl_osx.cpp
+    ${PSP_CPP_SRC}/src/cpp/compat_impl_wasm.cpp
     ${PSP_CPP_SRC}/src/cpp/compat_impl_win.cpp
     ${PSP_CPP_SRC}/src/cpp/computed_expression.cpp
     ${PSP_CPP_SRC}/src/cpp/computed_function.cpp
@@ -545,6 +565,27 @@ if(PSP_WASM_BUILD)
     set_target_properties(perspective_esm PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./web/")
     set_target_properties(perspective_esm PROPERTIES OUTPUT_NAME "perspective.cpp")
 elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
+    if (PSP_PYODIDE)
+      # TODO(tom): this is a straight copy of PSP_WASM_BUILD, merge them
+      # TODO(tom): determine which of these we shouldn't bother setting
+      # because pyodide's pywasmcross.py overrides them
+      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
+          --bind \
+          --source-map-base \"\" \
+          --memory-init-file 0 \
+          --no-entry \
+          -s EXPORT_ES6=1 \
+          -s NO_EXIT_RUNTIME=1 \
+          -s NO_FILESYSTEM=1 \
+          -s ALLOW_MEMORY_GROWTH=1 \
+          -s MODULARIZE=1 \
+          -s EXPORT_NAME=\"load_perspective\" \
+          -s MAXIMUM_MEMORY=4gb \
+          -s USE_ES6_IMPORT_META=0 \
+          -s ERROR_ON_UNDEFINED_SYMBOLS=1 \
+          -s ENVIRONMENT=web,webview,worker \
+      ")
+    endif ()
     if(NOT WIN32)
         set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
 
@@ -569,8 +610,15 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
         # #######################
         # Python extra targets #
         # #######################
-        add_library(psp SHARED ${PYTHON_SOURCE_FILES})
+        add_library(psp STATIC ${PYTHON_SOURCE_FILES})
         add_library(binding SHARED ${PYTHON_BINDING_SOURCE_FILES})
+        if(PSP_PYODIDE)
+            set(PSP_PYTHON_DEFS PSP_ENABLE_WASM=1)
+        else()
+            set(PSP_PYTHON_DEFS PSP_ENABLE_PYTHON_THREADING=1)
+        endif()
+        target_compile_definitions(psp PRIVATE ${PSP_PYTHON_DEFS})
+        target_compile_definitions(binding PRIVATE ${PSP_PYTHON_DEFS})
 
         include_directories(${PSP_PYTHON_SRC}/include)
 
@@ -583,13 +631,16 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 
             # .dll not importable
             set_property(TARGET binding PROPERTY SUFFIX .pyd)
+        elseif(PSP_PYODIDE)
+            target_compile_options(psp PRIVATE -fvisibility=hidden)
+            target_compile_options(binding PRIVATE -fvisibility=hidden)
         elseif(MACOS OR NOT MANYLINUX)
             target_compile_options(binding PRIVATE -Wdeprecated-declarations)
             set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
             set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
 
             target_compile_options(psp PRIVATE -fvisibility=hidden)
-            target_compile_options(binding PRIVATE -fvisibility=hidden)    
+            target_compile_options(binding PRIVATE -fvisibility=hidden)
         else()
             target_compile_options(binding PRIVATE -Wdeprecated-declarations)
         endif()
@@ -612,15 +663,15 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
     endif()
 
     if(PSP_CPP_BUILD_STRICT AND NOT WIN32)
-        target_compile_options(psp PRIVATE -Wall -Werror)
-        target_compile_options(psp PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
+        # target_compile_options(psp PRIVATE -Wall -Werror)
+        # target_compile_options(psp PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
 
-        if(PSP_PYTHON_BUILD)
+        if(PSP_PYTHON_BUILD AND NOT PSP_PYODIDE)
             target_compile_options(binding PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
         endif()
     elseif(WIN32)
-        target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1)
-        target_compile_definitions(psp PRIVATE WIN32=1)
-        target_compile_definitions(psp PRIVATE _WIN32=1)
+        # target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1)
+        # target_compile_definitions(psp PRIVATE WIN32=1)
+        # target_compile_definitions(psp PRIVATE _WIN32=1)
     endif()
 endif()

cpp/perspective/src/cpp/base.cpp

@@ -11,7 +11,7 @@
 #include <perspective/base.h>
 #include <cstdint>
 #include <limits>
-#ifdef PSP_ENABLE_WASM
+#if defined PSP_ENABLE_WASM
 #include <emscripten.h>
 #else
 #include <perspective/exception.h>
@@ -21,7 +21,7 @@ namespace perspective {
 
 void
 psp_abort(const std::string& message) {
-#ifdef PSP_ENABLE_WASM
+#if defined PSP_ENABLE_WASM
     std::string error = "Abort(): " + message;
     const char* error_cstr = error.c_str();
 

cpp/perspective/src/cpp/column.cpp

@@ -348,20 +348,20 @@ t_column::reserve(t_uindex size) {
 // object storage, specialize only for std::uint64_t
 template <>
 void
-t_column::object_copied<std::uint64_t>(std::uint64_t ptr) const {}
+t_column::object_copied<std::uint64_t>(t_uindex ptr) const {}
 
 void
-t_column::notify_object_copied(std::uint64_t idx) const {
+t_column::notify_object_copied(t_uindex idx) const {
     if (*get_nth_status(idx) == STATUS_VALID)
         object_copied<PSP_OBJECT_TYPE>(*(get_nth<std::uint64_t>(idx)));
 }
 
 template <>
 void
-t_column::object_cleared<std::uint64_t>(std::uint64_t ptr) const {}
+t_column::object_cleared<std::uint64_t>(t_uindex ptr) const {}
 
 void
-t_column::notify_object_cleared(std::uint64_t idx) const {
+t_column::notify_object_cleared(t_uindex idx) const {
     if (*get_nth_status(idx) == STATUS_VALID)
         object_cleared<PSP_OBJECT_TYPE>(*(get_nth<std::uint64_t>(idx)));
 }