vendor all C++ dependencies at build time, fixes #2075, closes #2076,

.github/workflows/build.yml

@@ -193,6 +193,8 @@ jobs:
                     - 3.9
                 node-version: [16.x]
 
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
         ##################################
         # Setup matrix job configuration #
         ##################################
@@ -432,6 +434,8 @@ jobs:
                 python-version:
                     - 3.9
 
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
         ##################################
         # Setup matrix job configuration #
         ##################################
@@ -546,6 +550,8 @@ jobs:
                     - 3.9
                 node-version: [16.x]
 
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
         ##################################
         # Setup matrix job configuration #
         ##################################
@@ -778,6 +784,8 @@ jobs:
                     - is-full-run: false
                       python-version: "3.10"
 
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
         ##################################
         # Setup matrix job configuration #
         ##################################
@@ -1045,6 +1053,8 @@ jobs:
                 python-version:
                     - 3.9
                 node-version: [16.x]
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
 
         ##################################
         # Setup matrix job configuration #
@@ -1251,6 +1261,8 @@ jobs:
 
                     - is-full-run: false
                       python-version: "3.10"
+            # NOTE: don't fail fast as sometimes npm blocks the burst of fetches from GHA
+            fail-fast: false
 
         ##################################
         # Setup matrix job configuration #

cmake/modules/FindInstallDependency.cmake

@@ -0,0 +1,66 @@
+cmake_minimum_required(VERSION 3.7.2)
+
+# ##################################################
+# Helper to grab dependencies from remote sources #
+# ##################################################
+function(psp_build_dep name cmake_file)
+    if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build)
+        psp_build_message("${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
+    else()
+        configure_file(${cmake_file} ${name}-download/CMakeLists.txt)
+
+        execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+            RESULT_VARIABLE result
+            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download)
+
+        if(result)
+            message(FATAL_ERROR "CMake step for ${name} failed: ${result}")
+        endif()
+
+        execute_process(COMMAND ${CMAKE_COMMAND} --build .
+            RESULT_VARIABLE result
+            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download)
+
+        if(result)
+            message(FATAL_ERROR "Build step for ${name} failed: ${result}")
+        endif()
+    endif()
+
+    if(${name} STREQUAL arrow)
+        # Overwrite arrow's CMakeLists with our custom, minimal CMakeLists.
+        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/cpp/ COPYONLY)
+        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/config.h ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/arrow/util/ COPYONLY)
+        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
+            ${CMAKE_BINARY_DIR}/${name}-build
+            EXCLUDE_FROM_ALL)
+
+        include_directories(${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
+    elseif(${name} STREQUAL exprtk)
+        # no cmakelists - just include the header
+        include_directories(${CMAKE_BINARY_DIR}/${name}-src)
+    elseif(${name} STREQUAL re2)
+        # Overwrite re2's CMakeLists with our custom CMakeLists.
+        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/ COPYONLY)
+        include_directories(${CMAKE_BINARY_DIR}/${name}-src)
+
+        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
+            ${CMAKE_BINARY_DIR}/${name}-build
+            EXCLUDE_FROM_ALL)
+    else()
+        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
+            ${CMAKE_BINARY_DIR}/${name}-build
+            EXCLUDE_FROM_ALL)
+
+        include_directories(${CMAKE_BINARY_DIR}/${name}-src/extras/${name}/include)
+        include_directories(${CMAKE_BINARY_DIR}/${name}-src/include)
+        include_directories(${CMAKE_BINARY_DIR}/${name}-src)
+    endif()
+
+    if(NOT PSP_WASM_BUILD AND (MACOS OR NOT MANYLINUX))
+        if(${name} STREQUAL arrow OR ${name} STREQUAL flatbuffers OR ${name} STREQUAL double-conversion OR ${name} STREQUAL re2)
+            target_compile_options(${name} PRIVATE -fvisibility=hidden)
+        endif()
+    endif()
+endfunction()
+
+# #############################

cpp/perspective/CMakeLists.txt

@@ -59,69 +59,11 @@ function(psp_build_message message)
     set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${message}")
 endfunction()
 
-# ##################################################
-# Helper to grab dependencies from remote sources #
-# ##################################################
-function(psp_build_dep name cmake_file)
-    if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build)
-        psp_build_message("${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
-    else()
-        configure_file(${cmake_file} ${name}-download/CMakeLists.txt)
-
-        execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
-            RESULT_VARIABLE result
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download)
-
-        if(result)
-            message(FATAL_ERROR "CMake step for ${name} failed: ${result}")
-        endif()
-
-        execute_process(COMMAND ${CMAKE_COMMAND} --build .
-            RESULT_VARIABLE result
-            WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download)
-
-        if(result)
-            message(FATAL_ERROR "Build step for ${name} failed: ${result}")
-        endif()
-    endif()
-
-    if(${name} STREQUAL arrow)
-        # Overwrite arrow's CMakeLists with our custom, minimal CMakeLists.
-        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/cpp/ COPYONLY)
-        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/config.h ${CMAKE_BINARY_DIR}/${name}-src/cpp/src/arrow/util/ COPYONLY)
-        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
-            ${CMAKE_BINARY_DIR}/${name}-build
-            EXCLUDE_FROM_ALL)
-
-        include_directories(${CMAKE_BINARY_DIR}/${name}-src/cpp/src/)
-    elseif(${name} STREQUAL exprtk)
-        # no cmakelists - just include the header
-        include_directories(${CMAKE_BINARY_DIR}/${name}-src)
-    elseif(${name} STREQUAL re2)
-        # Overwrite re2's CMakeLists with our custom CMakeLists.
-        configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/ COPYONLY)
-        include_directories(${CMAKE_BINARY_DIR}/${name}-src)
-
-        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
-            ${CMAKE_BINARY_DIR}/${name}-build
-            EXCLUDE_FROM_ALL)
-    else()
-        add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
-            ${CMAKE_BINARY_DIR}/${name}-build
-            EXCLUDE_FROM_ALL)
-
-        include_directories(${CMAKE_BINARY_DIR}/${name}-src/extras/${name}/include)
-        include_directories(${CMAKE_BINARY_DIR}/${name}-src/include)
-        include_directories(${CMAKE_BINARY_DIR}/${name}-src)
-    endif()
-endfunction()
-
-# #############################
-
 # ######################
 # BUILD CONFIGURATION #
 # ######################
 find_package(Color)
+find_package(InstallDependency)
 
 # OPTIONS
 option(CMAKE_BUILD_TYPE "Release/Debug build" RELEASE)
@@ -391,29 +333,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
         psp_build_message("${Cyan}Using Python ${Python_VERSION}, \nPython_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}, \nPython_LIBRARIES: ${Python_LIBRARIES}, \nPython_EXECUTABLE: ${Python_EXECUTABLE} ${ColorReset}")
         include_directories(${Python_INCLUDE_DIRS})
 
-        if(MACOS)
-            # on mac, use the vanilla pybind11 finder
-            find_package(pybind11)
-
-            if(pybind11_FOUND)
-                # Found PyBind installed by Homebrew
-                set(PYTHON_PYBIND_FOUND pybind11_FOUND)
-            else()
-                # Check if pip installed PyBind is available
-                find_package(Pybind)
-            endif()
-        else()
-            # On Linux/Docker, look for pip installed PyBind only
-            find_package(Pybind)
-        endif()
-
-        if(NOT PYTHON_PYBIND_FOUND)
-            psp_build_message("${Red}PyBind11 could not be located - building from external source${ColorReset}")
-            psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in")
-        else()
-            psp_build_message("${Cyan}Found PyBind11 in ${PYTHON_PYBIND_INCLUDE_DIR}${ColorReset}")
-            include_directories(${PYTHON_PYBIND_INCLUDE_DIR})
-        endif()
+        psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in")
 
         find_package(NumPy)
 
@@ -439,47 +359,18 @@ message("${Cyan}Building minimal Apache Arrow${ColorReset}")
 # Build arrow dependencies
 psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in")
 psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in")
-
-# FIXME: this is a hack to get Flatbuffers working on Azure Win64 by making the
-# headers accessible. The actual flatc executable is installed using
-# Chocolatey for our Azure Windows job.
-find_package(Flatbuffers)
-
-if(NOT FLATBUFFERS_FOUND)
-    psp_build_message("${Cyan}Could not find Flatbuffers${ColorReset}")
-    psp_build_dep("flatbuffers" "${PSP_CMAKE_MODULE_PATH}/flatbuffers.txt.in")
-else()
-    psp_build_message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}")
-    include_directories(${FLATBUFFERS_INCLUDE_DIR})
-endif()
+psp_build_dep("flatbuffers" "${PSP_CMAKE_MODULE_PATH}/flatbuffers.txt.in")
 
 # Build minimal arrow itself
 psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in")
 
 # Build re2 as our regex library
-find_package(re2)
-
-if(NOT re2_FOUND)
-    psp_build_message("${Cyan}Could not find Re2${ColorReset}")
-
-    # this is a workaround for some re2-specific weirdness
-    add_definitions(-DTARGET_OS_OSX=1)
-    psp_build_dep("re2" "${PSP_CMAKE_MODULE_PATH}/re2.txt.in")
-else()
-    psp_build_message("${Cyan}Found re2 in ${RE2_INCLUDE_DIR}${ColorReset}")
-    include_directories(${RE2_INCLUDE_DIR})
-endif()
+# this is a workaround for some re2-specific weirdness
+add_definitions(-DTARGET_OS_OSX=1)
+psp_build_dep("re2" "${PSP_CMAKE_MODULE_PATH}/re2.txt.in")
 
 # Build exprtk for expression parsing
-find_package(exprtk)
-
-if(NOT exprtk_FOUND)
-    psp_build_message("${Cyan}Could not find exprtk${ColorReset}")
-    psp_build_dep("exprtk" "${PSP_CMAKE_MODULE_PATH}/exprtk.txt.in")
-else()
-    psp_build_message("${Cyan}Found exprtk in ${EXPRTK_INCLUDE_DIR}${ColorReset}")
-    include_directories(${EXPRTK_INCLUDE_DIR})
-endif()
+psp_build_dep("exprtk" "${PSP_CMAKE_MODULE_PATH}/exprtk.txt.in")
 
 # ####################
 set(CMAKE_C_FLAGS_RELEASE " \
@@ -689,10 +580,6 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
             set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
             set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
 
-            target_compile_options(arrow PRIVATE -fvisibility=hidden)
-            target_compile_options(flatbuffers PRIVATE -fvisibility=hidden)
-            target_compile_options(double-conversion PRIVATE -fvisibility=hidden)
-            target_compile_options(re2 PRIVATE -fvisibility=hidden)
             target_compile_options(psp PRIVATE -fvisibility=hidden)
             target_compile_options(binding PRIVATE -fvisibility=hidden)    
         else()

python/perspective/bench/runtime/bench.py

@@ -144,7 +144,6 @@ def after_each(self):
 
 
 class Runner(object):
-
     ITERATIONS = 10
 
     def __init__(self, suite):
@@ -165,12 +164,12 @@ def __init__(self, suite):
         class_attrs = self._suite.__class__.__dict__.items()
         instance_attrs = self._suite.__dict__.items()
 
-        for (k, v) in class_attrs:
+        for k, v in class_attrs:
             if hasattr(v, "benchmark") and getattr(v, "benchmark") is True:
                 logging.info("Registering {0}".format(k))
                 self._benchmarks.append(v)
 
-        for (k, v) in instance_attrs:
+        for k, v in instance_attrs:
             if hasattr(v, "benchmark") and getattr(v, "benchmark") is True:
                 logging.info("Registering {0}".format(k))
                 self._benchmarks.append(v)

python/perspective/bench/runtime/perspective_benchmark.py

@@ -38,7 +38,6 @@ def empty_callback(port_id):
 
 
 class PerspectiveBenchmark(Suite):
-
     AGG_OPTIONS = [
         [{"column": "Sales", "op": "sum"}],
         [{"column": "State", "op": "dominant"}],

python/perspective/perspective/client/websocket.py

@@ -46,7 +46,6 @@ async def close(self):
 
 
 class PerspectiveWebsocketClient(PerspectiveClient):
-
     # Ping the server every 30 seconds
     PING_TIMEOUT = 15 * 1000
 

python/perspective/perspective/manager/manager_internal.py

@@ -34,7 +34,6 @@ def default(self, obj):
 
 
 class _PerspectiveManagerInternal(object):
-
     # Commands that should be blocked from execution when the manager is in
     # `locked` mode, i.e. its tables and views made immutable from remote
     # modification.

python/perspective/perspective/table/table.py

@@ -200,13 +200,13 @@ def validate_expressions(self, expressions, as_string=False):
             # full expression string in the UI.
             validated["expression_alias"][expression[0]] = expression[1]
 
-        for (alias, dtype) in expression_schema.items():
+        for alias, dtype in expression_schema.items():
             if not as_string:
                 dtype = _str_to_pythontype(dtype)
 
             validated["expression_schema"][alias] = expression_schema[alias]
 
-        for (alias, error) in expression_errors.items():
+        for alias, error in expression_errors.items():
             error_dict = {}
             error_dict["error_message"] = error.error_message
             error_dict["line"] = error.line