Skip to content

Commit 0d819fa

Browse files
PhatcatPhatcat
committed
Cached native buildrunners
Hitting all 3 target platforms with debug builds targeting both x64 and arm64, just like the upcoming manual build guide. - The windows and linux build tests the vcpkg configuration and reveals any issues in that install path. - macOS uses a homebrew approach and only needs to get the binaries from mysql qhich takes about a second. Add a simple check in the top level cmakefile and add homebrew libraries if we're on apple so the buildbot and people can use natively installed libraries. - The linux docker already tests a native dpendency path, and most packages can be got from apt-get with the exception of a proper mysql, but that takes but a second to download and install. all caches are about 1.1gb in size, and with the current 4 native builders that's about 4gb in total so there's room for expansion
1 parent e814a84 commit 0d819fa

File tree

6 files changed

+417
-1
lines changed

6 files changed

+417
-1
lines changed

‎.github/workflows/buildrunner_native.yml

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Copyright (c) 2025 Ember
2+
#
3+
# This Source Code Form is subject to the terms of the Mozilla Public
4+
# License, v. 2.0. If a copy of the MPL was not distributed with this
5+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
7+
name: Native Buildrunner
8+
9+
on:
10+
push:
11+
branches: [ "development" ]
12+
pull_request:
13+
branches: [ "development" ]
14+
15+
jobs:
16+
# Windows – Cached Debug Build
17+
Windows_x64_debug:
18+
runs-on: windows-2025
19+
steps:
20+
- name: Checkout Code
21+
uses: actions/[email protected]
22+
23+
- name: Cache Build Folder
24+
uses: actions/cache@v3
25+
with:
26+
path: build
27+
key: win-build-${{ hashFiles('CMakeLists.txt') }}
28+
restore-keys: win-build-
29+
30+
- name: Run Windows Build Script
31+
shell: pwsh
32+
run: .\build_windows.ps1
33+
34+
# Windows ARM – Cached Debug Build
35+
Windows_ARM64_debug:
36+
runs-on: windows-11-arm
37+
steps:
38+
- name: Checkout Code
39+
uses: actions/[email protected]
40+
41+
- name: Cache Build Folder
42+
uses: actions/cache@v3
43+
with:
44+
path: build
45+
key: win_arm-build-${{ hashFiles('CMakeLists.txt') }}
46+
restore-keys: win_arm-build-
47+
48+
- name: Run Windows Build Script
49+
shell: pwsh
50+
run: .\build_windows.ps1
51+
52+
# macOS – Cached Debug Build
53+
macOS_Apple_Silicon_debug:
54+
runs-on: macos-15
55+
steps:
56+
- name: Checkout Code
57+
uses: actions/[email protected]
58+
59+
- name: Cache Dependencies and Homebrew
60+
uses: actions/cache@v3
61+
with:
62+
path: |
63+
/Users/runner/Library/Caches/Homebrew
64+
dependencies
65+
key: macOS-deps
66+
restore-keys: macOS-deps
67+
68+
- name: Cache Build Folder
69+
uses: actions/cache@v3
70+
with:
71+
path: build
72+
key: macOS-build-${{ hashFiles('CMakeLists.txt') }}
73+
restore-keys: macOS-build-
74+
75+
- name: Run macOS Build Script
76+
run: |
77+
chmod +x ./build_macos.sh
78+
./build_macos.sh
79+
80+
# Linux – Cached Debug Build
81+
Linux_Ubuntu_debug:
82+
runs-on: ubuntu-24.04
83+
steps:
84+
- name: Checkout Code
85+
uses: actions/[email protected]
86+
87+
- name: Cache Dependencies and Homebrew
88+
uses: actions/cache@v3
89+
with:
90+
path: |
91+
~/.cache/Homebrew
92+
dependencies
93+
key: linux-deps
94+
restore-keys: linux-deps
95+
96+
- name: Cache Build Folder
97+
uses: actions/cache@v3
98+
with:
99+
path: build
100+
key: linux-build-${{ hashFiles('CMakeLists.txt') }}
101+
restore-keys: linux-build-
102+
103+
- name: Run Linux Build Script
104+
run: |
105+
chmod +x ./build_linux.sh
106+
./build_linux.sh

‎.github/workflows/docker-image.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
# Copyright (c) 2025 Ember
2+
#
3+
# This Source Code Form is subject to the terms of the Mozilla Public
4+
# License, v. 2.0. If a copy of the MPL was not distributed with this
5+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
17
name: Docker Image CI
28

39
on:
@@ -10,7 +16,7 @@ jobs:
1016

1117
build:
1218

13-
runs-on: ubuntu-latest
19+
runs-on: ubuntu-24.04-arm
1420

1521
steps:
1622
- name: Checkout Code

‎CMakeLists.txt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,17 @@ include_directories(${CMAKE_SOURCE_DIR}/deps)
165165
include(BuildDBCLoaders)
166166
include(BuildSparkServices)
167167

168+
# For Apple: Amend homebrew installed libraries to cmake's search path
169+
if(APPLE)
170+
file(GLOB HOME_BREW_LIB_DIRS "/opt/homebrew/opt/*/lib")
171+
set(HOMEBREW_LINKER_FLAGS_LIST "")
172+
foreach(libdir ${HOME_BREW_LIB_DIRS})
173+
list(APPEND HOMEBREW_LINKER_FLAGS_LIST "-L${libdir}")
174+
endforeach()
175+
message(STATUS "Appending Homebrew linker flags: ${HOMEBREW_LINKER_FLAGS_LIST}")
176+
add_link_options(${HOMEBREW_LINKER_FLAGS_LIST})
177+
endif()
178+
168179
add_subdirectory(schemas)
169180
set(cmake_ctest_arguments "CTEST_OUTPUT_ON_FAILURE")
170181
enable_testing()

‎build_linux.sh

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Copyright (c) 2025 Ember
2+
#
3+
# This Source Code Form is subject to the terms of the Mozilla Public
4+
# License, v. 2.0. If a copy of the MPL was not distributed with this
5+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
7+
#!/bin/bash
8+
set -e
9+
10+
#####################################################################################################
11+
# Install build tools
12+
# Already pre-installed: (software-properties-common, wget, gcc-14, g++-14, libstdc++-14-dev, git)
13+
# GCC is already the default compiler and up-to-date - libtirpc-dev is for libmysqlconncpp via vcpkg
14+
#####################################################################################################
15+
echo "installing apt-get dependencies..."
16+
sudo apt-get install -y build-essential cmake
17+
sudo update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-14 100
18+
sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-14 100
19+
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
20+
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
21+
22+
###################################################################################
23+
# Install homebrew and get dependencies through here until available in apt-get
24+
###################################################################################
25+
sudo apt-get install -y procps curl file libpcre3-dev
26+
27+
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
28+
29+
echo 'eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"' >>~/.profile
30+
eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
31+
32+
brew update
33+
brew upgrade
34+
brew install boost
35+
brew install botan
36+
brew install mysql-client
37+
brew install flatbuffers
38+
39+
#######################################
40+
# Install MySQL Connector/C++
41+
#######################################
42+
CACHE_TARBALL="dependencies/mysql-connector.tar.gz"
43+
44+
if [ -f "$CACHE_TARBALL" ]; then
45+
echo "Cached MySQL Connector tarball found."
46+
else
47+
arch=$(uname -m) && case "$arch" in \
48+
x86_64) url="https://dev.mysql.com/get/Downloads/Connector-C++/mysql-connector-c++-9.3.0-linux-glibc2.28-x86-64bit.tar.gz" ;; \
49+
aarch64) url="https://dev.mysql.com/get/Downloads/Connector-C++/mysql-connector-c++-9.3.0-linux-glibc2.28-aarch64.tar.gz" ;; \
50+
*) echo "Unsupported architecture: $arch" && exit 1 ;; esac
51+
echo "Downloading MySQL Connector/C++ from ${url}"
52+
sudo mkdir -p dependencies
53+
sudo wget "$url" -O "$CACHE_TARBALL"
54+
fi
55+
56+
echo "Extracting MySQL Connector tarball..."
57+
sudo mkdir -p /usr/lib/cmake/mysql-concpp
58+
sudo tar -zxf "$CACHE_TARBALL" -C /usr/lib/cmake/mysql-concpp --strip-components=1
59+
60+
echo "Installing headers and libraries..."
61+
sudo mkdir -p /usr/include/mysql-cppconn
62+
sudo cp -r /usr/lib/cmake/mysql-concpp/include/* /usr/include/mysql-cppconn/
63+
sudo cp -r /usr/lib/cmake/mysql-concpp/lib64/* /usr/local/lib/
64+
sudo ldconfig
65+
echo "MySQL Connector/C++ installed."
66+
67+
###############################
68+
# Configure and Build Ember
69+
###############################
70+
echo "=== Configuring project with CMake ==="
71+
72+
sudo rm -rf build
73+
74+
BUILD_OPTIONAL_TOOLS=-1
75+
DISABLE_THREADS=0
76+
BUILD_DIR="build"
77+
INSTALL_DIR="./build/bin"
78+
BUILD_TYPE="Debug"
79+
80+
cmake -S . -B ${BUILD_DIR} \
81+
-DBUILD_OPT_TOOLS=${BUILD_OPTIONAL_TOOLS} \
82+
-DDISABLE_EMBER_THREADS=${DISABLE_THREADS} \
83+
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}
84+
85+
echo "Building and installing the project..."
86+
cmake --build ${BUILD_DIR} --target install --config ${BUILD_TYPE}
87+
88+
###############################################
89+
# Run the unit_tests for regression control
90+
###############################################
91+
echo "=== Switching to installed directory and running tests ==="
92+
cd ${INSTALL_DIR}
93+
if [ -x "./unit_tests" ]; then
94+
echo "Running installed test executable..."
95+
./unit_tests
96+
else
97+
echo "Error: Installed test executable not found in ${INSTALL_DIR}. Aborting."
98+
exit 1
99+
fi
100+
101+
echo "=== Build, install, and test complete ==="

‎build_macos.sh

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# Copyright (c) 2025 Ember
2+
#
3+
# This Source Code Form is subject to the terms of the Mozilla Public
4+
# License, v. 2.0. If a copy of the MPL was not distributed with this
5+
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6+
7+
#!/bin/bash
8+
set -e
9+
10+
###############################################
11+
# Install build tools
12+
# Already pre-installed: (git, cmake, ninja)
13+
###############################################
14+
# Force xcode 16.3 or it'll default to 16.0
15+
LATEST_XCODE="/Applications/Xcode_16.3.app/Contents/Developer"
16+
CURRENT_XCODE=$(xcode-select -p)
17+
18+
if [ "$CURRENT_XCODE" != "$LATEST_XCODE" ]; then
19+
echo "Switching Xcode from $CURRENT_XCODE to $LATEST_XCODE"
20+
sudo xcode-select -s "$LATEST_XCODE"
21+
else
22+
echo "Using the correct Xcode: $LATEST_XCODE"
23+
fi
24+
25+
brew update
26+
brew upgrade
27+
brew install llvm
28+
29+
#############################################################
30+
# Install dependencies through homebrew
31+
# (Github macOS runner already come with openssl and zlib)
32+
#############################################################
33+
brew install boost
34+
brew install botan
35+
brew install mysql-client
36+
brew install pcre
37+
brew install flatbuffers
38+
39+
#######################################
40+
# Install MySQL Connector/C++
41+
#######################################
42+
CACHE_TARBALL="dependencies/mysql-connector.tar.gz"
43+
44+
if [ -f "$CACHE_TARBALL" ]; then
45+
echo "Cached MySQL Connector tarball found."
46+
else
47+
echo "Downloading MySQL Connector/C++ Library from Oracle..."
48+
sudo mkdir -p dependencies
49+
sudo wget https://dev.mysql.com/get/Downloads/Connector-C++/mysql-connector-c++-9.3.0-macos15-arm64.tar.gz -O "$CACHE_TARBALL"
50+
fi
51+
52+
echo "Extracting MySQL Connector tarball..."
53+
sudo mkdir -p /usr/local/lib/cmake/mysql-concpp
54+
sudo tar -zxf "$CACHE_TARBALL" -C /usr/local/lib/cmake/mysql-concpp --strip-components=1
55+
56+
echo "Installing headers and libraries..."
57+
sudo mkdir -p /usr/local/include/mysql-concpp
58+
sudo cp -r /usr/local/lib/cmake/mysql-concpp/include/. /usr/local/include/mysql-concpp/
59+
sudo cp -r /usr/local/lib/cmake/mysql-concpp/lib64/. /usr/local/lib/
60+
echo "MySQL Connector/C++ installed."
61+
62+
echo "Patching MySQL Connector dependencies to reference absolute OpenSSL paths..."
63+
for libfile in /usr/local/lib/libmysqlcppconn*.dylib; do
64+
if [ -f "$libfile" ]; then
65+
sudo install_name_tool -change libssl.3.dylib /opt/homebrew/opt/openssl@3/lib/libssl.3.dylib "$libfile"
66+
sudo install_name_tool -change libcrypto.3.dylib /opt/homebrew/opt/openssl@3/lib/libcrypto.3.dylib "$libfile"
67+
echo "Patched $libfile"
68+
fi
69+
done
70+
71+
###############################
72+
# Configure and Build Ember
73+
###############################
74+
echo "=== Configuring project with CMake ==="
75+
76+
# Set the DYLD_LIBRARY_PATH for all subsequent build and runtime steps.
77+
export DYLD_LIBRARY_PATH="$(brew --prefix openssl@3)/lib:$DYLD_LIBRARY_PATH"
78+
echo "DYLD_LIBRARY_PATH set to: $DYLD_LIBRARY_PATH"
79+
80+
C_COMPILER="/opt/homebrew/opt/llvm/bin/clang"
81+
CXX_COMPILER="/opt/homebrew/opt/llvm/bin/clang++"
82+
83+
CMAKE_C_FLAGS="-isystem /opt/homebrew/opt/llvm/include/c++/v1 -isysroot $(xcrun --show-sdk-path)"
84+
CMAKE_CXX_FLAGS="-isystem /opt/homebrew/opt/llvm/include/c++/v1 -isysroot $(xcrun --show-sdk-path)"
85+
86+
BUILD_OPTIONAL_TOOLS=-1
87+
DISABLE_THREADS=0
88+
BUILD_DIR="build"
89+
INSTALL_DIR="./build/bin"
90+
BUILD_TYPE="Debug"
91+
92+
cmake -S . -B ${BUILD_DIR} \
93+
-DCMAKE_C_COMPILER=${C_COMPILER} \
94+
-DCMAKE_CXX_COMPILER=${CXX_COMPILER} \
95+
-DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
96+
-DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
97+
-DCMAKE_OSX_SYSROOT=$(xcrun --show-sdk-path) \
98+
-DBUILD_OPT_TOOLS=${BUILD_OPTIONAL_TOOLS} \
99+
-DDISABLE_EMBER_THREADS=${DISABLE_THREADS} \
100+
-DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}
101+
102+
echo "Building and installing the project..."
103+
cmake --build ${BUILD_DIR} --target install --config ${BUILD_TYPE}
104+
105+
###############################################
106+
# Run the unit_tests for regression control
107+
###############################################
108+
echo "=== Switching to installed directory and running tests ==="
109+
cd ${INSTALL_DIR}
110+
if [ -x "./unit_tests" ]; then
111+
echo "Running installed test executable..."
112+
# openssl is linked dynamically so we need to point the unit_tests to the library
113+
sudo install_name_tool -change libssl.3.dylib "$(brew --prefix openssl@3)/lib/libssl.3.dylib" ./unit_tests
114+
sudo install_name_tool -change libcrypto.3.dylib "$(brew --prefix openssl@3)/lib/libcrypto.3.dylib" ./unit_tests
115+
sudo -E ./unit_tests
116+
else
117+
echo "Error: Installed test executable not found in ${INSTALL_DIR}. Aborting."
118+
exit 1
119+
fi
120+
121+
echo "=== Build, install, and test complete ==="

0 commit comments

Comments
 (0)