Skip to content

Commit f67899d

Browse files
cjc7373cjc7373
committed
feat: milvus backup and restore (#2252)
(cherry picked from commit 7ddff5d)
1 parent 3862ca5 commit f67899d

File tree

8 files changed

+183
-0
lines changed

8 files changed

+183
-0
lines changed

‎addons/milvus/dataprotection/backup.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
# if the script exits with a non-zero exit code, touch a file to indicate that the backup failed,
4+
# the sync progress container will check this file and exit if it exists
5+
function handle_exit() {
6+
exit_code=$?
7+
if [ $exit_code -ne 0 ]; then
8+
echo "failed with exit code $exit_code"
9+
touch "${DP_BACKUP_INFO_FILE}.exit"
10+
exit $exit_code
11+
fi
12+
}
13+
14+
trap handle_exit EXIT
15+
setStorageConfig
16+
17+
./milvus-backup create -n "$BACKUP_NAME"
18+
19+
# use datasafed to get backup size
20+
# if we do not write into $DP_BACKUP_INFO_FILE, the backup job will stuck
21+
TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}')
22+
DP_save_backup_status_info "$TOTAL_SIZE"

‎addons/milvus/dataprotection/common.sh‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
5+
function getToolConfigValue() {
6+
local var=$1
7+
grep "$var" < "$TOOL_CONFIG" | awk '{print $NF}'
8+
}
9+
10+
# shellcheck disable=SC2034
11+
function setStorageConfig() {
12+
TOOL_CONFIG=/etc/datasafed/datasafed.conf
13+
14+
ACCESS_KEY_ID=$(getToolConfigValue access_key_id)
15+
SECRET_ACCESS_KEY=$(getToolConfigValue secret_access_key)
16+
ENDPOINT=$(getToolConfigValue endpoint)
17+
BUCKET=$(getToolConfigValue "root =")
18+
PROVIDER=$(getToolConfigValue provider)
19+
20+
if [[ "$PROVIDER" == "Alibaba" ]]; then
21+
ENDPOINT="https://${ENDPOINT}"
22+
fi
23+
24+
export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
25+
export DATASAFED_BACKEND_BASE_PATH=${DP_BACKUP_BASE_PATH}
26+
27+
# only underscores are allowed in backup name
28+
BACKUP_NAME=${DP_BACKUP_NAME//-/_}
29+
30+
BACKUP_CONFIG=configs/backup.yaml
31+
MILVUS_CONFIG=/milvus/configs/user.yaml
32+
33+
# connection config
34+
yq -i ".milvus.address = \"$DP_DB_HOST\"" "$BACKUP_CONFIG"
35+
yq -i ".milvus.port = $DP_DB_PORT" "$BACKUP_CONFIG"
36+
yq -i ".milvus.user = \"\"" "$BACKUP_CONFIG"
37+
yq -i ".milvus.password = \"\"" "$BACKUP_CONFIG"
38+
yq -i ".backup.gcPause.address = \"http://$DP_DB_HOST:9091\"" "$BACKUP_CONFIG"
39+
40+
# milvus storage config
41+
yq -i ".minio.address = \"$MINIO_HOST\"" "$BACKUP_CONFIG"
42+
yq -i ".minio.port = \"$MINIO_PORT\"" "$BACKUP_CONFIG"
43+
yq -i ".minio.accessKeyID = \"$MINIO_ACCESS_KEY\"" "$BACKUP_CONFIG"
44+
yq -i ".minio.secretAccessKey = \"$MINIO_SECRET_KEY\"" "$BACKUP_CONFIG"
45+
46+
yq -i ".minio.bucketName = (load(\"$MILVUS_CONFIG\") | .minio.bucketName)" "$BACKUP_CONFIG"
47+
yq -i ".minio.useSSL = (load(\"$MILVUS_CONFIG\") | .minio.useSSL)" "$BACKUP_CONFIG"
48+
yq -i ".minio.rootPath = (load(\"$MILVUS_CONFIG\") | .minio.rootPath)" "$BACKUP_CONFIG"
49+
# TODO: is this right?
50+
yq -i ".minio.storageType = (load(\"$MILVUS_CONFIG\") | .minio.cloudProvider)" "$BACKUP_CONFIG"
51+
52+
# backup storage config
53+
without_scheme=${ENDPOINT#http://}
54+
IFS=":" read -r -a parts <<< "$without_scheme"
55+
yq -i ".minio.backupAddress = \"${parts[0]}\"" "$BACKUP_CONFIG"
56+
# FIXME: will backupPort be empty?
57+
yq -i ".minio.backupPort = \"${parts[1]}\"" "$BACKUP_CONFIG"
58+
yq -i ".minio.backupAccessKeyID = \"$ACCESS_KEY_ID\"" "$BACKUP_CONFIG"
59+
yq -i ".minio.backupSecretAccessKey = \"$SECRET_ACCESS_KEY\"" "$BACKUP_CONFIG"
60+
yq -i ".minio.backupBucketName = \"$BUCKET\"" "$BACKUP_CONFIG"
61+
# eliminate the leading slash, or go-minio will return an empty list when listing
62+
BACKUP_ROOT_PATH=${DP_BACKUP_BASE_PATH#/}
63+
yq -i ".minio.backupRootPath = \"$BACKUP_ROOT_PATH\"" "$BACKUP_CONFIG"
64+
}
65+
66+
# Save backup status info file for syncing progress.
67+
# timeFormat: %Y-%m-%dT%H:%M:%SZ
68+
function DP_save_backup_status_info() {
69+
local totalSize=$1
70+
local startTime=$2
71+
local stopTime=$3
72+
local timeZone=$4
73+
local extras=$5
74+
local timeZoneStr=""
75+
if [ -n "${timeZone}" ]; then
76+
timeZoneStr=$(printf ',"timeZone":"%s"' "${timeZone}")
77+
fi
78+
if [ -z "${stopTime}" ]; then
79+
printf '{"totalSize":"%s"}' "${totalSize}" > "${DP_BACKUP_INFO_FILE}"
80+
elif [ -z "${startTime}" ]; then
81+
printf '{"totalSize":"%s","extras":[%s],"timeRange":{"end":"%s"%s}}' "${totalSize}" "${extras}" "${stopTime}" "${timeZoneStr}" > "${DP_BACKUP_INFO_FILE}"
82+
else
83+
printf '{"totalSize":"%s","extras":[%s],"timeRange":{"start":"%s","end":"%s"%s}}' "${totalSize}" "${extras}" "${startTime}" "${stopTime}" "${timeZoneStr}" > "${DP_BACKUP_INFO_FILE}"
84+
fi
85+
}

‎addons/milvus/dataprotection/restore.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
setStorageConfig
4+
5+
./milvus-backup restore -n "$BACKUP_NAME"

‎addons/milvus/templates/_helpers.tpl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,7 @@ Milvus cluster vars for external storage services reference
325325
name: milvus-object-storage
326326
optional: false
327327
host: Required
328+
host: Required
328329
- name: MINIO_PORT
329330
valueFrom:
330331
serviceRefVarRef:

‎addons/milvus/templates/actionset.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: dataprotection.kubeblocks.io/v1alpha1
2+
kind: ActionSet
3+
metadata:
4+
name: milvus-full
5+
labels:
6+
{{- include "milvus.labels" . | nindent 4 }}
7+
spec:
8+
backupType: Full
9+
backup:
10+
backupData:
11+
image: {{ .Values.images.registry | default "docker.io" }}/{{ .Values.images.milvusBackup.repository }}:v0.5.9-yq
12+
# runOnTargetPodNode is needed to let backup controller mount volumes for backup job
13+
runOnTargetPodNode: true
14+
syncProgress:
15+
enabled: false
16+
intervalSeconds: 5
17+
command:
18+
- bash
19+
- -c
20+
- |
21+
{{- .Files.Get "dataprotection/common.sh" | nindent 8 }}
22+
{{- .Files.Get "dataprotection/backup.sh" | nindent 8 }}
23+
restore:
24+
postReady:
25+
- job:
26+
image: {{ .Values.images.registry | default "docker.io" }}/{{ .Values.images.milvusBackup.repository }}:v0.5.9-yq
27+
runOnTargetPodNode: true
28+
command:
29+
- bash
30+
- -c
31+
- |
32+
{{- .Files.Get "dataprotection/common.sh" | nindent 12 }}
33+
{{- .Files.Get "dataprotection/restore.sh" | nindent 12 }}

‎addons/milvus/templates/backuppolicytemplate.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,35 @@ spec:
1717
- data
1818
schedules:
1919
- backupMethod: volume-snapshot
20+
enabled: false
21+
retentionPeriod: 7d
22+
cronExpression: "0 18 * * 0"
23+
24+
---
25+
26+
apiVersion: dataprotection.kubeblocks.io/v1alpha1
27+
kind: BackupPolicyTemplate
28+
metadata:
29+
name: milvus-cluster-backup-policy-template
30+
labels:
31+
{{- include "milvus.labels" . | nindent 4 }}
32+
spec:
33+
serviceKind: Milvus
34+
compDefs:
35+
- {{ include "milvus-proxy.cmpdRegexpPattern" . }}
36+
backupMethods:
37+
- name: full
38+
snapshotVolumes: false
39+
actionSetName: milvus-full
40+
targetVolumes:
41+
volumes:
42+
- milvus-config
43+
volumeMounts:
44+
- mountPath: /milvus/configs/
45+
name: milvus-config
46+
readOnly: true
47+
schedules:
48+
- backupMethod: full
2049
enabled: false
2150
retentionPeriod: 7d
2251
cronExpression: "0 18 * * 0"

‎addons/milvus/tools/Dockerfile.backup‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# build with
2+
# docker buildx build --platform linux/amd64,linux/arm64 --tag apecloud/milvus-backup:v0.5.9-yq -f Dockerfile.backup .
3+
4+
FROM milvusdb/milvus-backup:v0.5.9
5+
6+
RUN apk update && apk add --no-cache bash yq

‎addons/milvus/values.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ images:
2828
registry: ""
2929
repository: apecloud/os-shell
3030
tag: 11-debian-11-r90
31+
milvusBackup:
32+
repository: apecloud/milvus-backup
3133

3234
livenessProbe:
3335
enabled: true

0 commit comments

Comments
 (0)