Skip to content

Commit 7d17061

Browse files
JaifroidJaifroid
authored
Add options to build release and nightly wasm versions from kiwix-build binaries #12 (#37)
1 parent 5f0f280 commit 7d17061

File tree

3 files changed

+78
-10
lines changed

3 files changed

+78
-10
lines changed

‎.github/workflows/build_libzim_wasm.yml

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
name: Build and publish release artefacts (Docker)
66

77
on:
8+
schedule:
9+
# Nightly run at 02:21 UTC
10+
- cron: '21 02 * * *'
811
push:
912
branches: [ main ]
1013
tags:
@@ -19,12 +22,26 @@ on:
1922
If left blank or incorrect format, archives will be archived instead of being uploaded to Releases.
2023
required: false
2124
default: ''
22-
25+
buildtype:
26+
description: |
27+
Choose the build type - 'source' (from source code, i.e., built from scratch - this takes a long time),
28+
'release' (from libzim released binary and dependencies - recommended), or 'nightly' (from the latest libzim
29+
nightly release)
30+
type: choice
31+
options:
32+
- source
33+
- release
34+
- nightly
35+
default: 'release'
36+
required: true
37+
2338
# Define top-level environment vars we can refer to below
2439
env:
2540
VERSION: ${{ github.ref_name }}
2641
DISPATCH_VERSION: ${{ github.event.inputs.version }}
42+
DISPATCH_TYPE: ${{ github.event.inputs.buildtype }}
2743
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44+
SSH_KEY: ${{ secrets.SSH_KEY }}
2845

2946
jobs:
3047
build:
@@ -35,8 +52,21 @@ jobs:
3552
# Customizes the Emscripten docker container via the Dockerfile in this repo
3653
- name: Build the Docker image
3754
run: docker build -t "docker-emscripten-libzim:v3" ./docker
55+
# If we're building release version
56+
- name: Build release from libzim binaries
57+
if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event.inputs.buildtype == 'release'
58+
run: |
59+
make libzim_release
60+
docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) docker-emscripten-libzim:v3 make release
61+
# If we're building nightly version
62+
- name: Build nightly from libzim binaries
63+
if: github.event.schedule || github.event.inputs.buildtype == 'nightly'
64+
run: |
65+
make libzim_nightly
66+
docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) docker-emscripten-libzim:v3 make nightly
3867
# Creates the ASM and WASM artefacts, and the JS wrappers, using the Makefile in this repo
39-
- name: Compile the libzim WASM artefacts
68+
- name: Compile the libzim WASM artefacts from source
69+
if: github.event.inputs.buildtype == 'source'
4070
run: docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) docker-emscripten-libzim:v3 make
4171
- name: List directories with updated archives
4272
run: |
@@ -50,15 +80,15 @@ jobs:
5080
- name: Archive build artefacts
5181
if: |
5282
github.event_name == 'pull_request' || github.event_name == 'push' && ! startsWith(github.ref_name, 'v')
53-
|| ! startsWith(github.event.inputs.version, 'v')
83+
|| github.event.inputs.buildtype != 'nightly' && ! github.event.schedule && ! startsWith(github.event.inputs.version, 'v')
5484
uses: actions/upload-artifact@v3
5585
with:
5686
name: libzim-wasm-artefacts
5787
path: |
5888
libzim-wasm.*
5989
libzim-asm.*
6090
tests/test_large_file_access/large_file_access.*
61-
# Otherwise, zip the artefacts into respective packages (asm and wasm), create and upload releases
91+
# If it's a release, zip the artefacts into respective packages (asm and wasm), create and upload releases
6292
- name: Zip the artefacts and create draft release
6393
id: zip-release
6494
if: github.event_name == 'push' && startsWith(github.ref_name, 'v') || startsWith(github.event.inputs.version, 'v')
@@ -69,3 +99,18 @@ jobs:
6999
# Create a draft release and upload zipped artefacts as release assets
70100
chmod +x ./scripts/create_draft_release.sh
71101
./scripts/create_draft_release.sh
102+
# If it's a nightly build, zip artefacts and upload releases
103+
- name: Zip the artefacts and upload to nightly
104+
if: github.event.schedule || github.event.inputs.buildtype == 'nightly'
105+
run: |
106+
echo "$SSH_KEY" > ./scripts/ssh_key
107+
chmod 600 ./scripts/ssh_key
108+
CURRENT_DATE=$(date +'%Y-%m-%d')
109+
target="/data/openzim/nightly/$CURRENT_DATE"
110+
zip libzim-javascript_wasm_$CURRENT_DATE.zip libzim-wasm.*
111+
zip libzim-javascript_asm_$CURRENT_DATE.zip libzim-asm.*
112+
for FILE in "libzim-javascript_wasm_$CURRENT_DATE.zip" "libzim-javascript_asm_$CURRENT_DATE.zip"
113+
do
114+
echo "Copying $FILE to $target"
115+
scp -P 30022 -o StrictHostKeyChecking=no -i ./scripts/ssh_key "$FILE" [email protected]:$target
116+
done

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@
1313
/icu4c-*.tgz
1414
/xapian-core-*.tar.xz
1515
/libzim-*
16+
/libzim_wasm-*.tar.gz
1617
ssh_key
17-
github_token
18+
github_token

‎Makefile

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,26 @@
1-
all: libzim-wasm.dev.js libzim-asm.dev.js libzim-wasm.js libzim-asm.js large_file_access.js
1+
SHELL := /bin/bash
2+
3+
all: build/lib/libzim.a libzim-wasm.dev.js libzim-asm.dev.js libzim-wasm.js libzim-asm.js large_file_access.js
4+
5+
release: libzim-asm.js libzim-wasm.js libzim-asm.dev.js libzim-wasm.dev.js large_file_access.js
6+
7+
nightly: libzim-asm.js libzim-wasm.js libzim-asm.dev.js libzim-wasm.dev.js large_file_access.js
8+
9+
libzim_release:
10+
wget -N $$(wget -q https://download.openzim.org/release/libzim/feed.xml -O - | grep -E -o -m1 "<link>[^<]+wasm-emscripten[^<]+</link>" | sed -E "s:</?link>::g")
11+
tar xf libzim_wasm-emscripten-*.tar.gz
12+
mkdir build
13+
mkdir build/lib
14+
cp -r libzim_wasm-emscripten-*/include/ build/include/
15+
cp -r libzim_wasm-emscripten-*/lib/x86_64-linux-gnu/*.* build/lib/
16+
17+
libzim_nightly:
18+
wget -N https://download.openzim.org/nightly/$$(date +'%Y-%m-%d')/$$(wget -q https://download.openzim.org/nightly/$$(date +'%Y-%m-%d') -O - | grep -E -o -m1 '"libzim_wasm-emscripten[^"]+"' | sed -E 's/"//g')
19+
tar xf libzim_wasm-emscripten-$$(date +'%Y-%m-%d').tar.gz
20+
mkdir build
21+
mkdir build/lib
22+
cp -r libzim_wasm-emscripten-$$(date +'%Y-%m-%d')/include/ build/include/
23+
cp -r libzim_wasm-emscripten-$$(date +'%Y-%m-%d')/lib/x86_64-linux-gnu/*.* build/lib/
224

325
build/lib/liblzma.so :
426
# Origin: https://tukaani.org/xz/xz-5.2.4.tar.gz
@@ -56,20 +78,20 @@ build/lib/libzim.a : build/lib/liblzma.so build/lib/libz.a build/lib/libzstd.a b
5678
cd libzim-8.1.0; ninja -C build install
5779

5880
# Development WASM version for testing, completely unoptimized
59-
libzim-wasm.dev.js: build/lib/libzim.a libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
81+
libzim-wasm.dev.js: libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
6082
em++ -o libzim-wasm.dev.js --bind libzim_bindings.cpp -I/src/build/include -L/src/build/lib -lzim -llzma -lzstd -lxapian -lz -licui18n -licuuc -licudata -lpthread -lm -fdiagnostics-color=always -pipe -Wall -Winvalid-pch -Wnon-virtual-dtor -Werror -std=c++11 -O0 -g --pre-js prejs_file_api.js --post-js postjs_file_api.js -s WASM=1 -s DISABLE_EXCEPTION_CATCHING=0 -s "EXPORTED_RUNTIME_METHODS=['ALLOC_NORMAL','printErr','ALLOC_STACK','print']" -s DEMANGLE_SUPPORT=1 -s INITIAL_MEMORY=83886080 -s ALLOW_MEMORY_GROWTH=1 -lworkerfs.js
6183
cp libzim-wasm.dev.* tests/prototype/
6284

6385
# Development ASM version for testing, completely unoptimized
64-
libzim-asm.dev.js: build/lib/libzim.a libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
86+
libzim-asm.dev.js: libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
6587
em++ -o libzim-asm.dev.js --bind libzim_bindings.cpp -I/src/build/include -L/src/build/lib -lzim -llzma -lzstd -lxapian -lz -licui18n -licuuc -licudata -lpthread -lm -fdiagnostics-color=always -pipe -Wall -Winvalid-pch -Wnon-virtual-dtor -Werror -std=c++11 -O0 -g --pre-js prejs_file_api.js --post-js postjs_file_api.js -s WASM=0 --memory-init-file 0 -s DISABLE_EXCEPTION_CATCHING=0 -s "EXPORTED_RUNTIME_METHODS=['ALLOC_NORMAL','printErr','ALLOC_STACK','print']" -s DEMANGLE_SUPPORT=1 -s INITIAL_MEMORY=83886080 -s ALLOW_MEMORY_GROWTH=1 -lworkerfs.js
6688

6789
# Production WASM version, optimized and packed
68-
libzim-wasm.js: build/lib/libzim.a libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
90+
libzim-wasm.js: libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
6991
em++ -o libzim-wasm.js --bind libzim_bindings.cpp -I/src/build/include -L/src/build/lib -lzim -llzma -lzstd -lxapian -lz -licui18n -licuuc -licudata -O3 --pre-js prejs_file_api.js --post-js postjs_file_api.js -s WASM=1 -s "EXPORTED_RUNTIME_METHODS=['ALLOC_NORMAL','printErr','ALLOC_STACK','print']" -s INITIAL_MEMORY=83886080 -s ALLOW_MEMORY_GROWTH=1 -lworkerfs.js
7092

7193
# Production ASM version, optimized and packed
72-
libzim-asm.js: build/lib/libzim.a libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
94+
libzim-asm.js: libzim_bindings.cpp prejs_file_api.js postjs_file_api.js
7395
em++ -o libzim-asm.js --bind libzim_bindings.cpp -I/src/build/include -L/src/build/lib -lzim -llzma -lzstd -lxapian -lz -licui18n -licuuc -licudata -O3 --pre-js prejs_file_api.js --post-js postjs_file_api.js -s WASM=0 --memory-init-file 0 -s MIN_EDGE_VERSION=40 -s "EXPORTED_RUNTIME_METHODS=['ALLOC_NORMAL','printErr','ALLOC_STACK','print']" -s INITIAL_MEMORY=83886080 -s ALLOW_MEMORY_GROWTH=1 -lworkerfs.js
7496

7597
# Test case: for testing large files

0 commit comments

Comments
 (0)