Fixes issues which occur when building on Apple Silicon

Link to xeus-zmq
Add option to define where cxxopts folder is (Macbook unable to locate folder without)
Change header file xeus/xserver_zmq.hpp to xeus-zmq/xserver_zmq.hpp
Add #include <iostream> to include/xeus-clang-repl/xmanager.hpp otherwise errors due to use of std::cerr and others
Fixes dockerfile so builds on arm based machines
Changes CI to include extra dependencies

Author: mcbarton

.github/workflows/ci.yml

@@ -136,10 +136,12 @@ jobs:
         #
         conda update --all
         conda install -y -q -c conda-forge \
-          'xeus>=2.0,<3.0' \
+          'xeus>=2.0' \
+          xeus-zmq \
           'nlohmann_json>=3.9.1,<3.10' \
           'cppzmq>=4.6.0,<5' \
           'xtl>=0.7,<0.8' \
+          'openssl<4' \
           pugixml \
           'cxxopts>=2.2.1,<2.3' \
           libuuid \

CMakeLists.txt

@@ -89,6 +89,7 @@ set(xeus_REQUIRED_VERSION 1.0.0)
 set(cppzmq_REQUIRED_VERSION 4.3.0)
 
 find_package(xeus ${xeus_REQUIRED_VERSION} REQUIRED)
+find_package(xeus-zmq  REQUIRED) 
 find_package(cppzmq REQUIRED CONFIG)
 find_package(pugixml REQUIRED)
 
@@ -103,6 +104,10 @@ if(POLICY CMP0077)
 endif()
 
 find_package(cxxopts REQUIRED)
+
+# Added as system unable to locate cxxopts include folder on MacOS without
+include_directories(${cxxopts_DIR}/../include/)
+
 set(Python_FIND_VIRTUALENV ONLY)
 find_package(Python COMPONENTS Interpreter Development)
 
@@ -251,7 +256,7 @@ target_include_directories(xeus-clang-repl SYSTEM
                            PUBLIC
                            $<BUILD_INTERFACE:${XEUS_CLANG-REPL_INCLUDE_DIR}>
                            $<INSTALL_INTERFACE:include>)
-target_link_libraries(xeus-clang-repl PUBLIC clangCppInterOp xeus pugixml cxxopts::cxxopts)
+target_link_libraries(xeus-clang-repl PUBLIC clangCppInterOp xeus xeus-zmq pugixml)
 
 if(APPLE)
     target_link_libraries(xeus-clang-repl PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup)

Dockerfile

@@ -45,6 +45,7 @@ RUN \
       ###libomp-dev \
     # Other "our" apt installs (development and testing)
       build-essential \
+      clang-15 \
       git \
       nano-tiny \
       less \
@@ -54,7 +55,9 @@ RUN \
       #cuda \
       $([ -n "$_CUDA_" ] && echo nvidia-cuda-toolkit) \
     && \
-    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean &&  \
+    apt-get autoremove && \
+    rm -rf /var/lib/apt/lists/* && \
     echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
     locale-gen
 
@@ -130,13 +133,16 @@ RUN \
     # Other "our" conda installs
     cmake \
     #"clangdev=$LLVM_REQUIRED_VERSION" \
-    'xeus>=2.0,<3.0' \
+    'xeus>=2.0' \
+    xeus-zmq \
     'nlohmann_json>=3.9.1,<3.10' \
     'cppzmq>=4.6.0,<5' \
     'xtl>=0.7,<0.8' \
-    'openssl<2' \
+    'openssl<4' \
     ipykernel \
     pugixml \
+    zlib \
+    libxml2 \
     'cxxopts>=2.2.1,<2.3' \
     libuuid \
     # Test dependencies
@@ -159,6 +165,9 @@ RUN \
     set -x && \
     source /home/jovyan/.conda.init && \
     conda activate .venv && \
+    export ARCHITECHURE=$(uname -m) && \
+    if [ "$ARCHITECHURE" == "x86_64" ] ; \ 
+    then \
     #
     artifact_name="clang-dev" && \
     #
@@ -167,9 +176,10 @@ RUN \
     echo $PWD && git_remote_origin_url=$(git config --get remote.origin.url) && \
     echo "Debug: Remote origin url: $git_remote_origin_url" && \
     arr=(${git_remote_origin_url//\// }) && \
-    gh_repo_owner=${arr[2]} && \
+    arr2=(${arr[0]//:/ }) && \
+    gh_repo_owner=${arr2[1]} && \
     gh_f_repo_owner="compiler-research" && \
-    arr=(${arr[3]//./ }) && \
+    arr=(${arr[1]//./ }) && \
     gh_repo_name=${arr[0]} && \
     gh_repo="${gh_repo_owner}/${gh_repo_name}" && \
     gh_f_repo_name=${gh_repo_name} && \
@@ -193,7 +203,6 @@ RUN \
     pushd /home/runner/work/xeus-clang-repl/xeus-clang-repl && \
     # repo
     echo "Debug: Repo owner/name: ${gh_repo_owner} / ${gh_repo_name}" && \
-
 #RUN \
 #    set -x && \
     source /home/jovyan/.conda.init && \
@@ -231,26 +240,45 @@ RUN \
     popd && \
     #
     echo "Debug clang path: $PATH_TO_CLANG_DEV" && \
-    export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/inst && \
+    export PATH_TO_LLVM_BUILD=$PATH_TO_CLANG_DEV/inst/ && \
+    export PATH_TO_LLVM_CONFIG=$PATH_TO_CLANG_DEV/inst/lib/cmake/llvm/ && \
+    export PATH_TO_CLANG_CONFIG=$PATH_TO_CLANG_DEV/inst/lib/cmake/clang/ ; \
+    fi  && \
     export VENV=${CONDA_DIR}/envs/.venv && \
-    export PATH=${VENV}/bin:${CONDA_DIR}/bin:$PATH_TO_LLVM_BUILD/bin:$PATH && \
-    export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH && \
     echo "export VENV=$VENV" >> ~/.profile && \
-    echo "export PATH=$PATH" >> ~/.profile && \
     echo "export EDITOR=emacs" >> ~/.profile && \
     #
-    # Build CppInterOp
-    #
+    # Build LLVM (if on arm) & CppInterOp 
+    #  
     sys_incs=$(LC_ALL=C c++ -xc++ -E -v /dev/null 2>&1 | LC_ALL=C sed -ne '/starts here/,/End of/p' | LC_ALL=C sed '/^ /!d' | cut -c2- | tr '\n' ':') && \
     export CPLUS_INCLUDE_PATH="${PATH_TO_LLVM_BUILD}/include/llvm:${PATH_TO_LLVM_BUILD}/include/clange:$CPLUS_INCLUDE_PATH:${sys_incs%:}" && \
     git clone https://github.com/compiler-research/CppInterOp.git && \
+    if [ "$ARCHITECHURE" == "aarch64" ] ; \ 
+    then \
+    export CC=/usr/bin/clang-15 && \
+    export CXX=/usr/bin/clang++-15 && \
+	git clone  --depth=1 --branch release/16.x  https://github.com/llvm/llvm-project.git && \
+	cd ./llvm-project/ && \
+	mkdir build && \
+	export LLVM_DIR=$(pwd) && \
+    compgen -G "../CppInterOp/patches/llvm/clang16-*.patch" > /dev/null && find ../CppInterOp/patches/llvm/clang16-*.patch -printf "%f\n" && git apply ../CppInterOp/patches/llvm/clang16-*.patch && \
+    cd build && \
+    cmake -DLLVM_ENABLE_PROJECTS=clang -DLLVM_TARGETS_TO_BUILD="host;NVPTX" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_LINKER=gold -DCLANG_ENABLE_STATIC_ANALYZER=OFF -DCLANG_ENABLE_ARCMT=OFF -DCLANG_ENABLE_FORMAT=OFF -DCLANG_ENABLE_BOOTSTRAP=OFF ../llvm && \
+    cmake --build . --target clang clang-repl --parallel $(nproc --all) && \
+    export PATH_TO_LLVM_CONFIG=$LLVM_DIR/build/lib/cmake/llvm/ && \
+    export PATH_TO_CLANG_CONFIG=$LLVM_DIR/build/lib/cmake/clang/ && \
+    export PATH_TO_LLVM_BUILD=$LLVM_DIR/build && \
+    cd ../../ ; \
+    fi && \
+    export PATH=${VENV}/bin:${CONDA_DIR}/bin:$PATH_TO_LLVM_BUILD/bin:$PATH && \
+    export LD_LIBRARY_PATH=$PATH_TO_LLVM_BUILD/lib:$LD_LIBRARY_PATH && \
+    echo "export PATH=$PATH" >> ~/.profile && \
     export CB_PYTHON_DIR="$PWD/cppyy-backend/python" && \
     export CPPINTEROP_DIR="$CB_PYTHON_DIR/cppyy_backend" && \
-    cd CppInterOp && \
-    mkdir build && \
-    cd build && \
+    mkdir CppInterOp/build && \
+    cd CppInterOp/build && \
     export CPPINTEROP_BUILD_DIR=$PWD && \
-    cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_CLING=OFF -DUSE_REPL=ON -DLLVM_DIR=$PATH_TO_LLVM_BUILD -DLLVM_USE_LINKER=gold -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. && \
+    cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DUSE_CLING=OFF -DUSE_REPL=ON -DLLVM_DIR=$PATH_TO_LLVM_CONFIG -DClang_DIR=$PATH_TO_CLANG_CONFIG -DLLVM_USE_LINKER=gold -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR .. && \
     cmake --build . --parallel $(nproc --all) && \
     #make install -j$(nproc --all)
     export CPLUS_INCLUDE_PATH="$CPPINTEROP_DIR/include:$CPLUS_INCLUDE_PATH" && \
@@ -260,9 +288,10 @@ RUN \
     #
     # Build and Install cppyy-backend
     #
-    git clone https://github.com/compiler-research/cppyy-backend.git && \
+    git clone --depth=1 https://github.com/compiler-research/cppyy-backend.git && \
     cd cppyy-backend && \
-    mkdir -p $CPPINTEROP_DIR/lib build && cd build && \
+    mkdir -p $CPPINTEROP_DIR/lib build && \
+    cd build && \
     # Install CppInterOp
     (cd $CPPINTEROP_BUILD_DIR && cmake --build . --target install --parallel $(nproc --all)) && \
     # Build and Install cppyy-backend
@@ -274,9 +303,9 @@ RUN \
     # Build and Install CPyCppyy
     #
     # Install CPyCppyy
-    git clone https://github.com/compiler-research/CPyCppyy.git && \
-    cd CPyCppyy && \
-    mkdir build && cd build && \
+    git clone --depth=1 https://github.com/compiler-research/CPyCppyy.git && \
+    mkdir CPyCppyy/build && \
+    cd CPyCppyy/build  && \
     cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. && \
     cmake --build . --parallel $(nproc --all) && \
     export CPYCPPYY_DIR=$PWD && \
@@ -285,7 +314,7 @@ RUN \
     # Build and Install cppyy
     #
     # Install cppyy
-    git clone https://github.com/compiler-research/cppyy.git && \
+    git clone --depth=1 https://github.com/compiler-research/cppyy.git && \
     cd cppyy && \
     python -m pip install --upgrade . --no-deps && \
     cd .. && \
@@ -294,9 +323,9 @@ RUN \
     export PYTHONPATH=$PYTHONPATH:$CPYCPPYY_DIR:$CB_PYTHON_DIR:/home/jovyan && \
     echo "export PYTHONPATH=$PYTHONPATH" >> ~/.profile && \
     export CPLUS_INCLUDE_PATH="/home/jovyan/CPyCppyy/include/:$CPLUS_INCLUDE_PATH" && \
-    # FIXME: Remove the hardcoded version of python here.
+    export PYTHON_VERSION=$(python --version | cut -c8- | cut -f1,2 -d'.') && \
     export CPLUS_INCLUDE_PATH="/home/jovyan/clad/include:$CPLUS_INCLUDE_PATH" && \
-    export CPLUS_INCLUDE_PATH="${VENV}/include:${VENV}/include/python3.10:$CPLUS_INCLUDE_PATH" && \
+    export CPLUS_INCLUDE_PATH="${VENV}/include:${VENV}/include/python${PYTHON_VERSION}:$CPLUS_INCLUDE_PATH" && \
     python -c "import cppyy" && \
     #
     # Build and Install xeus-clang-repl
@@ -312,13 +341,11 @@ RUN \
     # Build and Install Clad
     #
     git clone --depth=1 https://github.com/vgvassilev/clad.git && \
-    cd clad && \
-    mkdir build && \
-    cd build && \
-    cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. -DClang_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/clang/ -DLLVM_DIR=${PATH_TO_LLVM_BUILD}/lib/cmake/llvm/ -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} -DLLVM_EXTERNAL_LIT="$(which lit)" && \
-    #make -j$(nproc --all) && \
-    make && \
-    make install && \
+    mkdir clad/build && \
+    cd clad/build && \
+    cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. -DClang_DIR=$PATH_TO_CLANG_CONFIG -DLLVM_DIR=$PATH_TO_LLVM_CONFIG -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} -DLLVM_EXTERNAL_LIT="$(which lit)" && \
+    make -j$(nproc --all) && \
+    make install -j$(nproc --all) && \
     ### install clad in all exist kernels
     ##for i in "$KERNEL_PYTHON_PREFIX"/share/jupyter/kernels/*; do if [[ $i =~ .*/clad-xcpp.* ]]; then jq '.argv += ["-fplugin=$KERNEL_PYTHON_PREFIX/lib/clad.so"] | .display_name += " (with clad)"' "$i"/kernel.json > tmp.$$.json && mv tmp.$$.json "$i"/kernel.json; fi; done && \
     ###
@@ -339,8 +366,8 @@ RUN \
     k="/opt/conda/share/jupyter/kernels/python3/kernel.json" && \
     jq ".argv[0] = \"${VENV}/bin/python\"" $k > $k.$$.tmp && mv $k.$$.tmp $k && \
     # xtensor
-    git clone https://github.com/xtensor-stack/xtensor.git && \
-    cd xtensor && \
-    mkdir build && cd build && \
+    git clone --depth=1 https://github.com/xtensor-stack/xtensor.git && \
+    mkdir xtensor/build && \
+    cd xtensor/build && \
     cmake -DCMAKE_INSTALL_PREFIX=$KERNEL_PYTHON_PREFIX .. && \
-    make install
+    make install -j$(nproc --all)

README.md

@@ -7,8 +7,6 @@ link:
 
 ### Xeus-Clang-REPL
 
-Clone the repository locally and execute ./run-docker.sh
-
 ## Description
 
 `xeus-clang-repl` integrates clang-repl with the xeus protocol and is a platform for C++ usage in Jupyter Notebooks. The demo developed in this repository shows a Python - CPP integraton in Jupyter Notebooks, where variables can be transfered between Python and CPP.
@@ -26,7 +24,7 @@ fresh environment. It is also needed to use a
 You will first need to install dependencies
 
 ```bash
-mamba install cmake cxx-compiler nlohmann_json cppzmq xtl jupyterlab clangdev=14 cxxopts pugixml -c conda-forge
+mamba install xeus xeus-zmq cmake cxx-compiler nlohmann_json cppzmq xtl jupyterlab clangdev=14 cxxopts pugixml -c conda-forge
 ```
 
 **Note:** Use a mamba environment with python version >= 3.11 for fetching clang-versions
@@ -47,9 +45,9 @@ mamba install`xeus-clang-repl` notebook -c conda-forge
 ``` -->
 
 ```bash
-git clone https://github.com/llvm/llvm-project
+git clone --depth=1 --branch release/15.0x https://github.com/llvm/llvm-project
 
-git checkout -b release/15.0x
+cd llvm-project
 
 git apply patches/llvm/clang15-D127284.patch
 
@@ -75,6 +73,18 @@ cmake ../ -DClang_DIR=/usr/lib/llvm-15/build/lib/cmake/clang\
 make -j n
 ```
 
+## Docker
+It is possible to build xeus-clang-repl within a Docker container. On an arm based Apple machine with docker, docker-compose and colima installed, you can try out xeus-clang-repl using the following commands (cpu represents number of cores, and memory is in GiB, so adjust according to your machine)
+```bash
+git clone https://github.com/compiler-research/xeus-clang-repl.git
+cd xeus-clang-repl
+colima start --cpu 8 --memory 8 --arch aarch64 --vm-type=vz --vz-rosetta
+chmod a+x ./build-and-run-docker.sh
+./build-and-run-docker.sh
+```
+Once the bash script finishes running, you can experiment with xeus-clang-repl by opening up your browser and visiting http://127.0.0.1:8888/lab
+
+
 ## Try it online (for developers)
 
 To try out xeus-clang-repl interactively in your web browser, just click on the binder

build-and-run-docker.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+docker build . -t xeus-clang-repl
+
+export OS=$(uname -s)
+
+if [ "$OS" == "Darwin" ]
+then
+	docker run --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh"
+else
+	docker container run --rm -i hadolint/hadolint hadolint - < Dockerfile
+
+jupyter-repo2docker \
+    --no-run \
+    --user-name=jovyan \
+    --image-name xeus-clang-repl \
+    .
+
+docker run --gpus all --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh"
+#docker run --rm --runtime=nvidia --gpus all --publish 8888:8888 --name xeus-clang-repl-c -i -t xeus-clang-repl "start-notebook.sh"
+
+#    --editable \
+#    --ref InterOpIntegration \
+#    https://github.com/alexander-penev/xeus-clang-repl.git \
+
+fi

include/xeus-clang-repl/xmanager.hpp

@@ -20,6 +20,7 @@
 
 #include "nlohmann/json.hpp"
 
+#include <iostream>
 #include <map>
 #include <memory>
 #include <regex>

run-docker.sh

@@ -14,7 +14,7 @@
 
 #include "xeus/xkernel.hpp"
 #include "xeus/xkernel_configuration.hpp"
-#include "xeus/xserver_zmq.hpp"
+#include "xeus-zmq/xserver_zmq.hpp"
 
 #include "xeus-clang-repl/xeus_clang-repl_config.hpp"
 #include "xeus-clang-repl/xinterpreter.hpp"