Skip to content

Commit 3cf7330

Browse files
benashzbenashz
committed
Helm: add support for auto upgrading CRDs
Introduces a pre-upgrade hook that will upgrade/create any of the CRD manifests that are bundled in the vso docker image. Other fixes: - make the component label test more reliable by checking all documents individually. - configure a backoff in the pre-delete "pdcc" Job.
1 parent 723a706 commit 3cf7330

File tree

12 files changed

+556
-26
lines changed

12 files changed

+556
-26
lines changed

‎Dockerfile

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,20 @@ ARG LD_FLAGS
3131
# Build
3232
RUN CGO_ENABLED=0 GOOS=$GOOS GOARCH=$GOARCH go build -ldflags "$LD_FLAGS" -a -o $BIN_NAME main.go
3333

34+
# setup scripts directory needed for upgrading CRDs.
35+
RUN mkdir scripts
36+
COPY chart/crds scripts/crds
37+
RUN ln -s ../$BIN_NAME scripts/upgrade-crds
38+
3439
# dev image
3540
# -----------------------------------
3641
# Use distroless as minimal base image to package the manager binary
3742
# Refer to https://github.com/GoogleContainerTools/distroless for more details
3843
FROM gcr.io/distroless/static:nonroot as dev
44+
ENV BIN_NAME=vault-secrets-operator
3945
WORKDIR /
4046
COPY --from=dev-builder /workspace/$BIN_NAME /
47+
COPY --from=dev-builder /workspace/scripts /scripts
4148
USER 65532:65532
4249

4350
ENTRYPOINT ["/vault-secrets-operator"]
@@ -59,7 +66,8 @@ LABEL revision=$PRODUCT_REVISION
5966

6067
WORKDIR /
6168

62-
COPY dist/$TARGETOS/$TARGETARCH/$BIN_NAME /
69+
COPY dist/$TARGETOS/$TARGETARCH/$BIN_NAME /$BIN_NAME
70+
COPY dist/$TARGETOS/$TARGETARCH/scripts /scripts
6371
COPY LICENSE /licenses/copyright.txt
6472

6573
USER 65532:65532
@@ -93,7 +101,8 @@ LABEL name="Vault Secrets Operator" \
93101

94102
WORKDIR /
95103

96-
COPY dist/$TARGETOS/$TARGETARCH/$BIN_NAME /
104+
COPY dist/$TARGETOS/$TARGETARCH/$BIN_NAME /$BIN_NAME
105+
COPY dist/$TARGETOS/$TARGETARCH/scripts /scripts
97106
COPY LICENSE /licenses/copyright.txt
98107
COPY --from=build-ubi /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/pki/ca-trust/extracted/pem/
99108

‎Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,10 @@ docker-push: ## Push docker image with the manager.
274274

275275
.PHONY: ci-build
276276
ci-build: ## Build operator binary (without generating assets).
277-
mkdir -p $(BUILD_DIR)/$(GOOS)/$(GOARCH)
277+
rm -rf $(BUILD_DIR)/$(GOOS)/$(GOARCH)/scripts
278+
mkdir -p $(BUILD_DIR)/$(GOOS)/$(GOARCH)/scripts
279+
cp -a chart/crds $(BUILD_DIR)/$(GOOS)/$(GOARCH)/scripts/.
280+
ln -s ../$(BIN_NAME) $(BUILD_DIR)/$(GOOS)/$(GOARCH)/scripts/upgrade-crds
278281
CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build \
279282
-ldflags "${LD_FLAGS} $(shell GOOS=$(GOOS) GOARCH=$(GOARCH) ./scripts/ldflags-version.sh)" \
280283
-a \

‎chart/templates/deployment.yaml

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ metadata:
99
name: {{ include "vso.chart.fullname" . }}-controller-manager
1010
namespace: {{ .Release.Namespace }}
1111
labels:
12+
app.kubernetes.io/component: controller-manager
1213
{{- include "vso.chart.labels" . | nindent 4 }}
1314
{{ include "vso.imagePullSecrets" .}}
1415
---
@@ -167,6 +168,7 @@ metadata:
167168
name: {{ printf "%s-%s" "pdcc" (include "vso.chart.fullname" .) | trunc 63 | trimSuffix "-" }}
168169
namespace: {{ .Release.Namespace }}
169170
labels:
171+
app.kubernetes.io/component: controller-manager
170172
{{- include "vso.chart.labels" . | nindent 4 }}
171173
annotations:
172174
# This is what defines this resource as a hook. Without this line, the
@@ -177,6 +179,7 @@ metadata:
177179
{{- toYaml .Values.controller.annotations | nindent 4 }}
178180
{{- end }}
179181
spec:
182+
backoffLimit: 5
180183
template:
181184
metadata:
182185
# This name is truncated because kubernetes applies labels to the job which contain the job and pod
@@ -195,15 +198,20 @@ spec:
195198
- --pre-delete-hook-timeout-seconds={{ .Values.controller.preDeleteHookTimeoutSeconds }}
196199
command:
197200
- /vault-secrets-operator
198-
resources: {{- toYaml .Values.controller.manager.resources | nindent 10 }}
201+
{{- with .Values.hooks.resources }}
202+
resources:
203+
{{- toYaml . | nindent 10 }}
204+
{{- end}}
205+
{{- with .Values.controller.securityContext }}
199206
securityContext:
200-
{{- toYaml .Values.controller.securityContext | nindent 10 }}
207+
{{- toYaml .| nindent 10 }}
208+
{{- end}}
201209
restartPolicy: Never
202-
{{- if .Values.controller.nodeSelector }}
210+
{{- with .Values.controller.nodeSelector }}
203211
nodeSelector:
204-
{{- toYaml .Values.controller.nodeSelector | nindent 8 }}
212+
{{- toYaml . | nindent 8 }}
205213
{{- end }}
206-
{{- if .Values.controller.tolerations }}
214+
{{- with .Values.controller.tolerations }}
207215
tolerations:
208-
{{- toYaml .Values.controller.tolerations | nindent 8 }}
216+
{{- toYaml .| nindent 8 }}
209217
{{- end }}

‎chart/templates/hook-upgrade-crds.yaml

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
{{- if .Values.hooks.upgradeCRDs.enabled -}}
2+
---
3+
apiVersion: v1
4+
kind: ServiceAccount
5+
metadata:
6+
name: {{ template "vso.chart.fullname" . }}-upgrade-crds
7+
namespace: {{ .Release.Namespace }}
8+
labels:
9+
app.kubernetes.io/component: controller-manager
10+
{{ include "vso.chart.labels" . | indent 4 }}
11+
annotations:
12+
helm.sh/hook: pre-upgrade
13+
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
14+
helm.sh/hook-weight: "1"
15+
---
16+
apiVersion: rbac.authorization.k8s.io/v1
17+
kind: ClusterRole
18+
metadata:
19+
name: {{ template "vso.chart.fullname" . }}-upgrade-crds
20+
labels:
21+
app.kubernetes.io/component: rbac
22+
{{ include "vso.chart.labels" . | indent 4 }}
23+
annotations:
24+
helm.sh/hook: pre-upgrade
25+
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
26+
helm.sh/hook-weight: "2"
27+
rules:
28+
- apiGroups:
29+
- apiextensions.k8s.io
30+
resources:
31+
- customresourcedefinitions
32+
verbs:
33+
- create
34+
- delete
35+
- get
36+
- list
37+
- patch
38+
- update
39+
---
40+
apiVersion: rbac.authorization.k8s.io/v1
41+
kind: ClusterRoleBinding
42+
metadata:
43+
name: {{ template "vso.chart.fullname" . }}-upgrade-crds
44+
labels:
45+
app.kubernetes.io/component: rbac
46+
{{ include "vso.chart.labels" . | indent 4 }}
47+
annotations:
48+
helm.sh/hook: pre-upgrade
49+
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
50+
helm.sh/hook-weight: "2"
51+
subjects:
52+
- kind: ServiceAccount
53+
name: {{ template "vso.chart.fullname" . }}-upgrade-crds
54+
namespace: {{ .Release.Namespace }}
55+
roleRef:
56+
kind: ClusterRole
57+
name: {{ template "vso.chart.fullname" . }}-upgrade-crds
58+
apiGroup: rbac.authorization.k8s.io
59+
---
60+
apiVersion: batch/v1
61+
kind: Job
62+
metadata:
63+
name: {{ printf "%s-%s" "upgrade-crds" (include "vso.chart.fullname" .) | trunc 63 | trimSuffix "-" }}
64+
namespace: {{ .Release.Namespace }}
65+
labels:
66+
app.kubernetes.io/component: controller-manager
67+
{{- include "vso.chart.labels" . | nindent 4 }}
68+
annotations:
69+
# This is what defines this resource as a hook. Without this line, the
70+
# job is considered part of the release.
71+
helm.sh/hook: pre-upgrade
72+
helm.sh/hook-delete-policy: "hook-succeeded,before-hook-creation"
73+
helm.sh/hook-weight: "99"
74+
{{- if .Values.controller.annotations }}
75+
{{- toYaml .Values.controller.annotations | nindent 4 }}
76+
{{- end }}
77+
spec:
78+
backoffLimit: {{ .Values.hooks.upgradeCRDs.backoffLimit | default 5 }}
79+
template:
80+
metadata:
81+
name: {{ printf "%s-%s" "upgrade-crds" (include "vso.chart.fullname" .) | trunc 63 | trimSuffix "-" }}
82+
spec:
83+
serviceAccountName: {{ template "vso.chart.fullname" . }}-upgrade-crds
84+
securityContext:
85+
{{- toYaml .Values.controller.podSecurityContext | nindent 8 }}
86+
containers:
87+
- name: pre-upgrade-crds
88+
image: {{ .Values.controller.manager.image.repository }}:{{ .Values.controller.manager.image.tag }}
89+
env:
90+
- name: VSO_UPGRADE_CRDS_TIMEOUT
91+
value: .Values.hooks.upgradeCRDs.executionTimeout
92+
command:
93+
- /scripts/upgrade-crds
94+
{{- with .Values.hooks.resources }}
95+
resources:
96+
{{- toYaml . | nindent 10 }}
97+
{{- end}}
98+
{{- with .Values.controller.securityContext }}
99+
securityContext:
100+
{{- toYaml .| nindent 10 }}
101+
{{- end}}
102+
restartPolicy: Never
103+
{{- with .Values.controller.nodeSelector }}
104+
nodeSelector:
105+
{{- toYaml . | nindent 8 }}
106+
{{- end }}
107+
{{- with .Values.controller.tolerations }}
108+
tolerations:
109+
{{- toYaml .| nindent 8 }}
110+
{{- end }}
111+
{{- end -}}

‎chart/templates/tests/test-runner.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ metadata:
99
name: {{ template "vso.chart.fullname" . }}-test
1010
namespace: {{ .Release.Namespace }}
1111
labels:
12+
app.kubernetes.io/component: controller-manager
1213
app: {{ template "vso.chart.name" . }}
1314
chart: {{ template "vso.chart.chart" . }}
1415
heritage: {{ .Release.Service }}

‎chart/values.yaml

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
# Top level configuration for the vault secrets operator deployment.
55
# This consists of a controller and a kube rbac proxy container.
66
controller:
7-
87
# Set the number of replicas for the operator.
98
# @type: integer
109
replicas: 1
@@ -753,6 +752,33 @@ telemetry:
753752
# @type: string
754753
scrapeTimeout: 10s
755754

755+
# Configure the behaviour of Helm hooks.
756+
hooks:
757+
# Resources common to all hooks.
758+
resources:
759+
limits:
760+
cpu: 500m
761+
memory: 128Mi
762+
requests:
763+
cpu: 10m
764+
memory: 64Mi
765+
# Configure the Helm pre-upgrade hook that handles custom resource definition (CRD) upgrades.
766+
upgradeCRDs:
767+
# Set to true to automatically upgrade the CRDs.
768+
# Disabling this will require manual intervention to upgrade the CRDs, so it is recommended to
769+
# always leave it enabled.
770+
# @type: boolean
771+
enabled: true
772+
773+
# Limit the number of retries for the CRD upgrade.
774+
# @type: integer
775+
backoffLimit: 5
776+
777+
# Set the timeout for the CRD upgrade. The operation should typically take less than 5s
778+
# to complete.
779+
# @type: string
780+
executionTimeout: 30s
781+
756782
## Used by unit tests, and will not be rendered except when using `helm template`, this can be safely ignored.
757783
tests:
758784
# @type: boolean

‎go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,13 @@ require (
2929
github.com/stretchr/testify v1.9.0
3030
golang.org/x/crypto v0.23.0
3131
google.golang.org/api v0.181.0
32-
gopkg.in/yaml.v3 v3.0.1
3332
k8s.io/api v0.30.1
33+
k8s.io/apiextensions-apiserver v0.30.1
3434
k8s.io/apimachinery v0.30.1
3535
k8s.io/client-go v0.30.1
3636
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
3737
sigs.k8s.io/controller-runtime v0.18.3
38+
sigs.k8s.io/yaml v1.4.0
3839
)
3940

4041
require (
@@ -167,10 +168,9 @@ require (
167168
gopkg.in/inf.v0 v0.9.1 // indirect
168169
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
169170
gopkg.in/yaml.v2 v2.4.0 // indirect
170-
k8s.io/apiextensions-apiserver v0.30.1 // indirect
171+
gopkg.in/yaml.v3 v3.0.1 // indirect
171172
k8s.io/klog/v2 v2.120.1 // indirect
172173
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
173174
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
174175
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
175-
sigs.k8s.io/yaml v1.4.0 // indirect
176176
)

0 commit comments

Comments
 (0)