chore(release-0.9): zk hscale

addons-cluster/zookeeper-cluster/templates/_helpers.tpl

@@ -0,0 +1,15 @@
+{{/*
+Create extra envs annotations
+*/}}
+{{- define "zookeeper-cluster.annotations.extra-envs" -}}
+"kubeblocks.io/extra-env": {{ include "zookeeper-cluster.extra-envs" . | nospace  | quote }}
+{{- end -}}
+
+{{/*
+Create extra env
+*/}}
+{{- define "zookeeper-cluster.extra-envs" -}}
+{
+"ZOOKEEPER_STANDALONE_ENABLED": "{{ .Values.standaloneEnabled }}",
+}
+{{- end -}}

addons-cluster/zookeeper-cluster/templates/cluster.yaml

@@ -5,6 +5,8 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     {{ include "kblib.clusterLabels" . | nindent 4 }}
+  annotations:
+    {{- include "zookeeper-cluster.annotations.extra-envs" . | nindent 4 }}
 spec:
   terminationPolicy: {{ .Values.extra.terminationPolicy }}
   {{- include "kblib.affinity" . | indent 2 }}

addons-cluster/zookeeper-cluster/values.yaml

@@ -29,4 +29,6 @@ memory: 2
 storage: 10
 
 ## @param log storage, the unit is Gi
-logStorage: 2
+logStorage: 2
+
+standaloneEnabled: false

addons/zookeeper/Chart.yaml

@@ -4,7 +4,7 @@ description: Apache ZooKeeper is a centralized service for maintaining configura
 
 type: application
 
-version: 0.9.0
+version: 0.9.1
 
 appVersion: "3.7.2"
 

addons/zookeeper/config/zookeeper-config.tpl

@@ -45,21 +45,20 @@ autopurge.purgeInterval=12
 # whitelist
 4lw.commands.whitelist=srvr, mntr, ruok, conf, stat, sync
 
-# cluster server list
-{{- printf "\n" }}
-{{- $fqnds := splitList "," .ZOOKEEPER_POD_FQDN_LIST }}
-{{- range $i, $fqdn := $fqnds }}
-  {{- $name := index (splitList "." $fqdn) 0 }}
-  {{- $tokens := splitList "-" $name }}
-  {{- $ordinal := index $tokens (sub (len $tokens) 1) }}
-  {{- if ge $i 3 }}
-    {{- printf "server.%s=%s:2888:3888:observer\n" $ordinal $fqdn }}
-  {{- else }}
-    {{- printf "server.%s=%s:2888:3888:participant\n" $ordinal $fqdn }}
-  {{- end }}
+{{- if hasKey $.cluster.metadata.annotations "kubeblocks.io/extra-env" -}}
+{{- $extraEnv := index $.cluster.metadata.annotations "kubeblocks.io/extra-env" | fromJson -}}
+{{- if hasKey $extraEnv "ZOOKEEPER_STANDALONE_ENABLED" }}
+standaloneEnabled={{ $extraEnv.ZOOKEEPER_STANDALONE_ENABLED }}
+{{- else }}
+standaloneEnabled=false
+{{- end -}}
+{{- else }}
+standaloneEnabled=false
 {{- end }}
 
-
+# dynamic config
+reconfigEnabled=true
+dynamicConfigFile={{ .ZOOKEEPER_DYNAMIC_CONFIG_FILE }}
 
 # logging
 audit.enable=true

addons/zookeeper/config/zookeeper-dynamic.tpl

@@ -0,0 +1,13 @@
+# cluster server list
+{{- printf "\n" }}
+{{- $fqnds := splitList "," .ZOOKEEPER_POD_FQDN_LIST }}
+{{- range $i, $fqdn := $fqnds }}
+  {{- $name := index (splitList "." $fqdn) 0 }}
+  {{- $tokens := splitList "-" $name }}
+  {{- $ordinal := index $tokens (sub (len $tokens) 1) }}
+  {{- if ge $i 3 }}
+    {{- printf "server.%s=%s:2888:3888:observer\n" $ordinal $fqdn }}
+  {{- else }}
+    {{- printf "server.%s=%s:2888:3888:participant\n" $ordinal $fqdn }}
+  {{- end }}
+{{- end }}

addons/zookeeper/scripts/member_join.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# ZOOKEEPER_POD_FQDN_LIST eg:  zk-zookeeper-0.zk-zookeeper-headless.default.svc.cluster.local,zk-zookeeper-1.zk-zookeeper-headless.default.svc.cluster.local
+zk_member_fqdns=${ZOOKEEPER_POD_FQDN_LIST//,/ }
+for member in $zk_member_fqdns; do
+    # if member contains CURRENT_POD_NAME
+    if [[ $member == *"$KB_POD_NAME"* ]]; then
+        zk_current_member_fqdn=$member
+    fi
+done
+
+if [ -z "$zk_current_member_fqdn" ]; then
+    echo "ERROR: Could not find current pod FQDN in ZOOKEEPER_POD_FQDN_LIST"
+    exit 1
+else
+    echo "Current pod FQDN: $zk_current_member_fqdn"
+    current_member_index=${KB_POD_NAME##*-}
+    zkCli.sh << EOF
+        addauth digest $ZK_ADMIN_USER:$ZK_ADMIN_PASSWORD
+        reconfig -add server.${current_member_index}=$zk_current_member_fqdn:2888:3888:participant;2181
+EOF
+fi

addons/zookeeper/scripts/member_leave.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+current_member_index=${KB_POD_NAME##*-}
+zkCli.sh << EOF
+    addauth digest $ZK_ADMIN_USER:$ZK_ADMIN_PASSWORD
+    reconfig -remove ${current_member_index}
+EOF

addons/zookeeper/scripts/startup.sh

@@ -51,23 +51,26 @@ function set_jvm_configuration() {
   gc_file="/opt/bitnami/zookeeper/logs/gc.log"
   java_major=$(java -XshowSettings:properties -version 2>&1 | grep "java.specification.version" | awk '{print $3}')
   if [ "${java_major}" -ge 9 ];then
-    export JVMFLAGS="$JVMFLAGS \
-          -XX:+UseG1GC \
-          -Xlog:gc:${gc_file}
-          -Xlog:gc* \
-          -XX:NewRatio=2 \
-          -Xms$MAX_HEAP_SIZE -Xmx$MAX_HEAP_SIZE"
+    JVMFLAGS="$JVMFLAGS \
+      -XX:+UseG1GC \
+      -Xlog:gc:${gc_file}
+      -Xlog:gc* \
+      -XX:NewRatio=2 \
+      -Xms$MAX_HEAP_SIZE -Xmx$MAX_HEAP_SIZE"
   else
-    export JVMFLAGS="$JVMFLAGS \
-          -XX:+UseG1GC \
-          -Xloggc:${gc_file} \
-          -XX:+PrintGCDetails \
-          -XX:+PrintGCDateStamps \
-          -XX:+PrintGCTimeStamps \
-          -XX:+UseGCLogFileRotation
-          -XX:NewRatio=2 \
-          -Xms$MAX_HEAP_SIZE -Xmx$MAX_HEAP_SIZE"
+    JVMFLAGS="$JVMFLAGS \
+      -XX:+UseG1GC \
+      -Xloggc:${gc_file} \
+      -XX:+PrintGCDetails \
+      -XX:+PrintGCDateStamps \
+      -XX:+PrintGCTimeStamps \
+      -XX:+UseGCLogFileRotation
+      -XX:NewRatio=2 \
+      -Xms$MAX_HEAP_SIZE -Xmx$MAX_HEAP_SIZE"
   fi
+
+  hash_str=$(echo -n ${ZK_ADMIN_USER}:${ZK_ADMIN_PASSWORD} | openssl dgst -binary -sha1 | openssl base64)
+  export JVMFLAGS="${JVMFLAGS} -Dzookeeper.DigestAuthenticationProvider.superDigest=${ZK_ADMIN_USER}:${hash_str}"
 }
 
 if [ -z "${ZOOKEEPER_IMAGE_VERSION}" ] ||  version_lt "3.6.0" "${ZOOKEEPER_IMAGE_VERSION%%-*}"  ; then

addons/zookeeper/templates/_helpers.tpl

@@ -49,3 +49,10 @@ Selector labels
 app.kubernetes.io/name: {{ include "zookeeper.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+
+{{/*
+Define zookeeper dynamic config template name
+*/}}
+{{- define "zookeeper.dynamicConfigTplName" -}}
+zookeeper-dynamic-config-template
+{{- end -}}

addons/zookeeper/templates/cmpd.yaml

@@ -44,6 +44,28 @@ spec:
       writable: false
       votable: false
   lifecycleActions:
+    memberJoin:
+      builtinHandler: custom
+      customHandler:
+        preCondition: RuntimeReady
+        container: zookeeper
+        exec:
+          command:
+            - /bin/bash
+            - -c
+            - |
+              /kubeblocks/scripts/member_join.sh
+    memberLeave:
+      builtinHandler: custom
+      customHandler:
+        preCondition: RuntimeReady
+        container: zookeeper
+        exec:
+          command:
+            - /bin/bash
+            - -c
+            - |
+              /kubeblocks/scripts/member_leave.sh
     roleProbe:
       builtinHandler: custom
       customHandler:
@@ -171,6 +193,9 @@ spec:
             subPath: log4j.properties
           - name: scripts
             mountPath: /kubeblocks/scripts
+          - name: dynamic-config
+            mountPath: {{ .Values.zookeeper.dynamicConfigDir }}
+
   vars:
     - name: ZOOKEEPER_POD_FQDN_LIST
       valueFrom:
@@ -185,6 +210,20 @@ spec:
       value: {{ .Values.zookeeper.dataDir }}
     - name: ZOOKEEPER_DATA_LOG_DIR
       value: {{ .Values.zookeeper.dataLogDir }}
+    - name: ZOOKEEPER_DYNAMIC_CONFIG_FILE
+      value: {{ .Values.zookeeper.dynamicConfigDir }}/zoo.cfg.dynamic
+    - name: ZK_ADMIN_USER
+      valueFrom:
+        credentialVarRef:
+          name: admin
+          optional: false
+          username: Required
+    - name: ZK_ADMIN_PASSWORD
+      valueFrom:
+        credentialVarRef:
+          name: admin
+          optional: false
+          password: Required
 
   scripts:
     - name: zookeeper-scripts
@@ -200,10 +239,18 @@ spec:
       constraintRef: {{ include "zookeeper.name" . }}-config-constraints
       volumeName: config
       defaultMode: 0755
+    - name: zookeeper-dynamic-config
+      namespace: {{ .Release.Namespace }}
+      templateRef: {{ include "zookeeper.dynamicConfigTplName" . }}
+      constraintRef: {{ include "zookeeper.name" . }}-dynamic-config-constraints
+      volumeName: dynamic-config
       reRenderResourceTypes:
         - hscale
     - name: zookeeper-log
       namespace: {{ .Release.Namespace }}
       templateRef: {{ include "zookeeper.name" . }}-configmap
       volumeName: config-log
-  updateStrategy: BestEffortParallel
+  updateStrategy: BestEffortParallel
+  systemAccounts:
+    - name: admin
+      initAccount: true

addons/zookeeper/templates/config-configmap.yaml

@@ -6,4 +6,15 @@ metadata:
     {{- include "zookeeper.labels" . | nindent 4 }}
 data:
   zoo.cfg: |-
-    {{- .Files.Get "config/zookeeper-config.tpl" | nindent 4 }}
+    {{- .Files.Get "config/zookeeper-config.tpl" | nindent 4 }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "zookeeper.dynamicConfigTplName" . }}
+  labels:
+    {{- include "zookeeper.labels" . | nindent 4 }}
+data:
+  zoo.cfg.dynamic: |-
+    {{- .Files.Get "config/zookeeper-dynamic.tpl" | nindent 4 }}

addons/zookeeper/templates/configconstraint.yaml

@@ -45,4 +45,26 @@ spec:
     topLevelKey: ZookeeperParameter
 
     cue: |-
-      {{- .Files.Get "config/zookeeper-config-constraint.cue" | nindent 6 }}
+      {{- .Files.Get "config/zookeeper-config-constraint.cue" | nindent 6 }}
+
+---
+apiVersion: apps.kubeblocks.io/v1beta1
+kind: ConfigConstraint
+metadata:
+  name: {{ include "zookeeper.name" . }}-dynamic-config-constraints
+  labels:
+    {{- include "zookeeper.labels" . | nindent 4 }}
+spec:
+  reloadAction:
+    autoTrigger:
+      processName: zookeeper
+
+  # zookeeper configuration file format
+  fileFormatConfig:
+    format: props-plus
+  dynamicParameters:
+    - server.1
+    - server.2
+    - server.3
+    - server.4
+    - server.5

addons/zookeeper/templates/scripts.yaml

@@ -6,4 +6,8 @@ metadata:
     {{- include "zookeeper.labels" . | nindent 4 }}
 data:
   startup.sh: |-
-    {{- .Files.Get "scripts/startup.sh" | nindent 4 }}
+    {{- .Files.Get "scripts/startup.sh" | nindent 4 }}
+  member_join.sh: |-
+    {{- .Files.Get "scripts/member_join.sh" | nindent 4 }}
+  member_leave.sh: |-
+    {{- .Files.Get "scripts/member_leave.sh" | nindent 4 }}

addons/zookeeper/values.yaml

@@ -20,4 +20,5 @@ metrics:
 
 zookeeper:
   dataDir: /bitnami/zookeeper/data
-  dataLogDir: /bitnami/zookeeper/log
+  dataLogDir: /bitnami/zookeeper/log
+  dynamicConfigDir: /opt/bitnami/zookeeper/dynamic