Skip to content

Commit 50eefd6

Browse files
roshkhatriroshkhatri
authored
[Cherry-Pick]Adds workflows to build release binaries and push to S3 (#315) (#857)
[related to](#230) Adds workflows to build Valkey binaries and push to S3 to make it available to download from the website The Workflows can be triggered by pushing a release to the repo and the other option is manually by one of the Maintainers. Once the workflow triggers, it will generate a matrix of Jobs for the platforms we need to build from `utils/releasetools/build-config.json` and then the respective Jobs are triggered. These jobs make Valkey with respect to the platform binaries we want to release and would push to a private S3 bucket. --------- Signed-off-by: Roshan Khatri <[email protected]>
1 parent 68d8876 commit 50eefd6

File tree

5 files changed

+313
-0
lines changed

5 files changed

+313
-0
lines changed

‎.github/actions/generate-package-build-matrix/action.yml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
name: Generate target matrix.
2+
description: Matrix creation for building Valkey for different architectures and platforms.
3+
4+
inputs:
5+
ref:
6+
description: The commit, tag or branch of Valkey to checkout to determine what version to use.
7+
required: true
8+
outputs:
9+
x86_64-build-matrix:
10+
description: The x86_64 build matrix.
11+
value: ${{ steps.set-matrix.outputs.x86matrix }}
12+
arm64-build-matrix:
13+
description: The arm64 build matrix.
14+
value: ${{ steps.set-matrix.outputs.armmatrix }}
15+
16+
runs:
17+
using: "composite"
18+
steps:
19+
- name: Checkout code for version check
20+
uses: actions/checkout@v4
21+
with:
22+
ref: ${{ inputs.ref }}
23+
path: version-check
24+
25+
- name: Get targets
26+
run: |
27+
x86_arch=$(jq -c '[.linux_targets[] | select(.arch=="x86_64")]' utils/releasetools/build-config.json)
28+
x86_matrix=$(echo "{ \"distro\" : $x86_arch }" | jq -c .)
29+
echo "X86_MATRIX=$x86_matrix" >> $GITHUB_ENV
30+
31+
arm_arch=$(jq -c '[.linux_targets[] | select(.arch=="arm64")]' utils/releasetools/build-config.json)
32+
arm_matrix=$(echo "{ \"distro\" : $arm_arch }" | jq -c .)
33+
echo "ARM_MATRIX=$arm_matrix" >> $GITHUB_ENV
34+
shell: bash
35+
36+
- id: set-matrix
37+
run: |
38+
echo $X86_MATRIX
39+
echo $X86_MATRIX| jq .
40+
echo "x86matrix=$X86_MATRIX" >> $GITHUB_OUTPUT
41+
echo $ARM_MATRIX
42+
echo $ARM_MATRIX| jq .
43+
echo "armmatrix=$ARM_MATRIX" >> $GITHUB_OUTPUT
44+
shell: bash

‎.github/workflows/build-release-packages.yml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
name: Build Release Packages
2+
3+
on:
4+
release:
5+
types: [published]
6+
7+
workflow_dispatch:
8+
inputs:
9+
version:
10+
description: Version of Valkey to build
11+
required: true
12+
13+
permissions:
14+
contents: read
15+
16+
jobs:
17+
# This job provides the version metadata from the tag for the other jobs to use.
18+
release-build-get-meta:
19+
name: Get metadata to build
20+
runs-on: ubuntu-latest
21+
outputs:
22+
version: ${{ steps.get_version.outputs.VERSION }}
23+
steps:
24+
25+
- run: |
26+
echo "Version: ${{ inputs.version || github.ref_name }}"
27+
shell: bash
28+
29+
# This step is to consolidate the three different triggers into a single "version"
30+
# 1. If manual dispatch - use the version provided.
31+
# 3. If tag trigger, use that tag.
32+
- name: Get the version
33+
id: get_version
34+
run: |
35+
VERSION="${INPUT_VERSION}"
36+
if [ -z "${VERSION}" ]; then
37+
exit 1
38+
fi
39+
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
40+
shell: bash
41+
env:
42+
# Use the dispatch variable in preference, if empty use the context ref_name which should
43+
# only ever be a tag
44+
INPUT_VERSION: ${{ inputs.version || github.ref_name }}
45+
46+
generate-build-matrix:
47+
name: Generating build matrix
48+
runs-on: ubuntu-latest
49+
outputs:
50+
x86_64-build-matrix: ${{ steps.set-matrix.outputs.x86_64-build-matrix }}
51+
arm64-build-matrix: ${{ steps.set-matrix.outputs.arm64-build-matrix }}
52+
steps:
53+
- name: Checkout repository
54+
uses: actions/checkout@v4
55+
# Set up the list of target to build so we can pass the JSON to the reusable job
56+
- uses: ./.github/actions/generate-package-build-matrix
57+
id: set-matrix
58+
with:
59+
ref: ${{ inputs.version || github.ref_name }}
60+
61+
release-build-linux-x86-packages:
62+
needs:
63+
- release-build-get-meta
64+
- generate-build-matrix
65+
uses: ./.github/workflows/call-build-linux-x86-packages.yml
66+
with:
67+
version: ${{ needs.release-build-get-meta.outputs.version }}
68+
ref: ${{ inputs.version || github.ref_name }}
69+
build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }}
70+
secrets:
71+
token: ${{ secrets.GITHUB_TOKEN }}
72+
bucket: ${{ secrets.AWS_S3_BUCKET }}
73+
access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
74+
secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
75+
76+
release-build-linux-arm-packages:
77+
needs:
78+
- release-build-get-meta
79+
- generate-build-matrix
80+
uses: ./.github/workflows/call-build-linux-arm-packages.yml
81+
with:
82+
version: ${{ needs.release-build-get-meta.outputs.version }}
83+
ref: ${{ inputs.version || github.ref_name }}
84+
build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }}
85+
secrets:
86+
token: ${{ secrets.GITHUB_TOKEN }}
87+
bucket: ${{ secrets.AWS_S3_BUCKET }}
88+
access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
89+
secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}

‎.github/workflows/call-build-linux-arm-packages.yml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: Builds Linux arm binary packages into S3 bucket.
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
version:
7+
description: The version of Valkey to create.
8+
type: string
9+
required: true
10+
ref:
11+
description: The commit, tag or branch of Valkey to checkout for building that creates the version above.
12+
type: string
13+
required: true
14+
build_matrix:
15+
description: The build targets to produce as a JSON matrix.
16+
type: string
17+
required: true
18+
secrets:
19+
token:
20+
description: The Github token or similar to authenticate with.
21+
required: true
22+
bucket:
23+
description: The name of the S3 bucket to push packages into.
24+
required: false
25+
access_key_id:
26+
description: The S3 access key id for the bucket.
27+
required: false
28+
secret_access_key:
29+
description: The S3 secret access key for the bucket.
30+
required: false
31+
32+
permissions:
33+
contents: read
34+
35+
jobs:
36+
build-valkey:
37+
# Capture source tarball and generate checksum for it
38+
name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
39+
runs-on: 'ubuntu-latest'
40+
strategy:
41+
fail-fast: false
42+
matrix: ${{ fromJSON(inputs.build_matrix) }}
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v4
46+
with:
47+
ref: ${{ inputs.version }}
48+
49+
- name: Make Valkey
50+
uses: uraimo/run-on-arch-action@v2
51+
with:
52+
arch: aarch64
53+
distro: ${{matrix.distro.target}}
54+
install: apt-get update && apt-get install -y build-essential libssl-dev
55+
run: make -C src all BUILD_TLS=yes
56+
57+
- name: Create Tarball and SHA256sums
58+
run: |
59+
TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}}
60+
mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share
61+
cp -rfv src/valkey-* $TAR_FILE_NAME/bin
62+
cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE
63+
tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME
64+
sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256
65+
mkdir -p packages-files
66+
cp -rfv $TAR_FILE_NAME.tar* packages-files/
67+
68+
- name: Install AWS cli.
69+
run: |
70+
sudo apt-get install -y awscli
71+
72+
- name: Configure AWS credentials
73+
run: |
74+
aws configure set region us-west-2
75+
aws configure set aws_access_key_id ${{ secrets.access_key_id }}
76+
aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
77+
78+
- name: Sync to S3
79+
run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/

‎.github/workflows/call-build-linux-x86-packages.yml

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: Builds Linux X86 binary packages into S3 bucket.
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
version:
7+
description: The version of Valkey to create.
8+
type: string
9+
required: true
10+
ref:
11+
description: The commit, tag or branch of Valkey to checkout for building that creates the version above.
12+
type: string
13+
required: true
14+
build_matrix:
15+
description: The build targets to produce as a JSON matrix.
16+
type: string
17+
required: true
18+
secrets:
19+
token:
20+
description: The Github token or similar to authenticate with.
21+
required: true
22+
bucket:
23+
description: The name of the S3 bucket to push packages into.
24+
required: false
25+
access_key_id:
26+
description: The S3 access key id for the bucket.
27+
required: false
28+
secret_access_key:
29+
description: The S3 secret access key for the bucket.
30+
required: false
31+
32+
permissions:
33+
contents: read
34+
35+
jobs:
36+
build-valkey:
37+
# Capture source tarball and generate checksum for it
38+
name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
39+
runs-on: 'ubuntu-latest'
40+
strategy:
41+
fail-fast: false
42+
matrix: ${{ fromJSON(inputs.build_matrix) }}
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v4
46+
with:
47+
ref: ${{ inputs.version }}
48+
49+
- name: Install dependencies
50+
run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev jq wget awscli
51+
52+
- name: Make Valkey
53+
run: make -C src all BUILD_TLS=yes
54+
55+
- name: Create Tarball and SHA256sums
56+
run: |
57+
TAR_FILE_NAME=valkey-${{inputs.version}}-${{matrix.distro.platform}}-${{ matrix.distro.arch}}
58+
mkdir -p $TAR_FILE_NAME/bin $TAR_FILE_NAME/share
59+
cp -rfv src/valkey-* $TAR_FILE_NAME/bin
60+
cp -v /home/runner/work/valkey/valkey/COPYING $TAR_FILE_NAME/share/LICENSE
61+
tar -czvf $TAR_FILE_NAME.tar.gz $TAR_FILE_NAME
62+
sha256sum $TAR_FILE_NAME.tar.gz > $TAR_FILE_NAME.tar.gz.sha256
63+
mkdir -p packages-files
64+
cp -rfv $TAR_FILE_NAME.tar* packages-files/
65+
66+
- name: Configure AWS credentials
67+
run: |
68+
aws configure set region us-west-2
69+
aws configure set aws_access_key_id ${{ secrets.access_key_id }}
70+
aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
71+
72+
- name: Sync to S3
73+
run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/

‎utils/releasetools/build-config.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"linux_targets": [
3+
{
4+
"arch": "x86_64",
5+
"target": "ubuntu18.04",
6+
"type": "deb",
7+
"platform": "bionic"
8+
},
9+
{
10+
"arch": "x86_64",
11+
"target": "ubuntu20.04",
12+
"type": "deb",
13+
"platform": "focal"
14+
},
15+
{
16+
"arch": "arm64",
17+
"target": "ubuntu18.04",
18+
"type": "deb",
19+
"platform": "bionic"
20+
},
21+
{
22+
"arch": "arm64",
23+
"target": "ubuntu20.04",
24+
"type": "deb",
25+
"platform": "focal"
26+
}
27+
]
28+
}

0 commit comments

Comments
 (0)