Skip to content
This repository was archived by the owner on Apr 24, 2024. It is now read-only.

Commit 53203c4

Browse files
stevekuznetsovstevekuznetsov
committed
make: add an end-to-end test
Signed-off-by: Steve Kuznetsov <[email protected]>
1 parent 86b7a4b commit 53203c4

File tree

5 files changed

+548
-71
lines changed

5 files changed

+548
-71
lines changed

‎Makefile

Lines changed: 157 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,73 @@
1+
##@ General
2+
3+
# The help target prints out all targets with their descriptions organized
4+
# beneath their categories. The categories are represented by '##@' and the
5+
# target descriptions by '##'. The awk commands is responsible for reading the
6+
# entire set of makefiles included in this invocation, looking for lines of the
7+
# file as xyz: ## something, and then pretty-format the target and help. Then,
8+
# if there's a line with ##@ something, that gets pretty-printed as a category.
9+
# More info on the usage of ANSI control characters for terminal formatting:
10+
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
11+
# More info on the awk command:
12+
# http://linuxcommand.org/lc3_adv_awk.php
13+
14+
.PHONY: help
15+
help: ## Display this help.
16+
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
17+
18+
##@ Build Dependencies
19+
20+
## Location to install dependencies to
21+
LOCALBIN ?= $(shell pwd)/bin
22+
23+
## Tool Binaries
24+
KUSTOMIZE ?= $(LOCALBIN)/kustomize
25+
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
26+
ENVTEST ?= $(LOCALBIN)/setup-envtest
27+
KCP ?= $(LOCALBIN)/kcp
28+
KUBECTL_KCP ?= $(LOCALBIN)/kubectl-kcp
29+
YQ ?= $(LOCALBIN)/yq
30+
31+
## Tool Versions
32+
KUSTOMIZE_VERSION ?= v3.8.7
33+
CONTROLLER_TOOLS_VERSION ?= v0.8.0
34+
KCP_VERSION ?= 0.7.5
35+
YQ_VERSION ?= v4.27.2
36+
37+
KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
38+
$(KUSTOMIZE): ## Download kustomize locally if necessary.
39+
mkdir -p $(LOCALBIN)
40+
curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN)
41+
touch $(KUSTOMIZE) # we download an "old" file, so make will re-download to refresh it unless we make it newer than the owning dir
42+
43+
$(CONTROLLER_GEN): ## Download controller-gen locally if necessary.
44+
mkdir -p $(LOCALBIN)
45+
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
46+
47+
$(YQ): ## Download yq locally if necessary.
48+
mkdir -p $(LOCALBIN)
49+
GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION)
50+
51+
OS ?= $(shell go env GOOS )
52+
ARCH ?= $(shell go env GOARCH )
53+
54+
$(KCP): ## Download kcp locally if necessary.
55+
mkdir -p $(LOCALBIN)
56+
curl -L -s -o - https://github.com/kcp-dev/kcp/releases/download/v$(KCP_VERSION)/kcp_$(KCP_VERSION)_$(OS)_$(ARCH).tar.gz | tar --directory $(LOCALBIN)/../ -xvzf - bin/kcp
57+
touch $(KCP) # we download an "old" file, so make will re-download to refresh it unless we make it newer than the owning dir
58+
59+
$(KUBECTL_KCP): ## Download kcp kubectl plugins locally if necessary.
60+
mkdir -p $(LOCALBIN)
61+
curl -L -s -o - https://github.com/kcp-dev/kcp/releases/download/v$(KCP_VERSION)/kubectl-kcp-plugin_$(KCP_VERSION)_$(OS)_$(ARCH).tar.gz | tar --directory $(LOCALBIN)/../ -xvzf - bin
62+
touch $(KUBECTL_KCP) # we download an "old" file, so make will re-download to refresh it unless we make it newer than the owning dir
163

2-
# Image URL to use all building/pushing image targets
3-
IMG ?= controller:latest
64+
$(ENVTEST): ## Download envtest locally if necessary.
65+
mkdir -p $(LOCALBIN)
66+
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
67+
68+
# Image registry and URL to use all building/pushing image targets
69+
REGISTRY ?= localhost
70+
IMG ?= controller:test
471
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
572
ENVTEST_K8S_VERSION = 1.23
673

@@ -23,35 +90,18 @@ all: build
2390
# kcp specific
2491
APIEXPORT_PREFIX ?= today
2592

26-
##@ General
27-
28-
# The help target prints out all targets with their descriptions organized
29-
# beneath their categories. The categories are represented by '##@' and the
30-
# target descriptions by '##'. The awk commands is responsible for reading the
31-
# entire set of makefiles included in this invocation, looking for lines of the
32-
# file as xyz: ## something, and then pretty-format the target and help. Then,
33-
# if there's a line with ##@ something, that gets pretty-printed as a category.
34-
# More info on the usage of ANSI control characters for terminal formatting:
35-
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
36-
# More info on the awk command:
37-
# http://linuxcommand.org/lc3_adv_awk.php
38-
39-
.PHONY: help
40-
help: ## Display this help.
41-
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
42-
4393
##@ Development
4494

4595
.PHONY: manifests
46-
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
96+
manifests: $(CONTROLLER_GEN) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
4797
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
4898

4999
.PHONY: apiresourceschemas
50-
apiresourceschemas: kustomize ## Convert CRDs from config/crds to APIResourceSchemas. Specify APIEXPORT_PREFIX as needed.
100+
apiresourceschemas: $(KUSTOMIZE) ## Convert CRDs from config/crds to APIResourceSchemas. Specify APIEXPORT_PREFIX as needed.
51101
$(KUSTOMIZE) build config/crd | kubectl kcp crd snapshot -f - --prefix $(APIEXPORT_PREFIX) > config/kcp/$(APIEXPORT_PREFIX).apiresourceschemas.yaml
52102

53103
.PHONY: generate
54-
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
104+
generate: $(CONTROLLER_GEN) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
55105
$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
56106

57107
.PHONY: fmt
@@ -63,8 +113,76 @@ vet: ## Run go vet against code.
63113
go vet ./...
64114

65115
.PHONY: test
66-
test: manifests generate fmt vet envtest ## Run tests.
67-
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
116+
test: manifests generate fmt vet $(ENVTEST) ## Run tests.
117+
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./controllers/... -coverprofile cover.out
118+
119+
ARTIFACT_DIR ?= .test
120+
121+
.PHONY: test-e2e
122+
test-e2e: $(ARTIFACT_DIR)/kind.kubeconfig kcp-synctarget ready-deployment run-test-e2e## Set up prerequisites and run end-to-end tests on a cluster.
123+
124+
.PHONY: run-test-e2e
125+
run-test-e2e: ## Run end-to-end tests on a cluster.
126+
go test ./test/e2e/... --kubeconfig $(abspath $(ARTIFACT_DIR)/kcp.kubeconfig) --workspace $(shell $(KCP_KUBECTL) kcp workspace . --short)
127+
128+
.PHONY: ready-deployment
129+
ready-deployment: KUBECONFIG = $(ARTIFACT_DIR)/kcp.kubeconfig
130+
ready-deployment: kind-image install deploy apibinding ## Deploy the controller-manager and wait for it to be ready.
131+
$(KCP_KUBECTL) --namespace "controller-runtime-example-system" rollout status deployment/controller-runtime-example-controller-manager
132+
133+
# TODO(skuznets|ncdc): this APIBinding is not needed, but here only to work around https://github.com/kcp-dev/kcp/issues/1183 - remove it once that is fixed
134+
.PHONY: apibinding
135+
apibinding:
136+
$( eval WORKSPACE = $(shell $(KCP_KUBECTL) kcp workspace . --short))
137+
sed 's/WORKSPACE/$(WORKSPACE)/' ./test/e2e/apibinding.yaml | $(KCP_KUBECTL) apply -f -
138+
$(KCP_KUBECTL) wait --for=condition=Ready apibinding/data.my.domain
139+
140+
.PHONY: kind-image
141+
kind-image: docker-build ## Load the controller-manager image into the kind cluster.
142+
kind load docker-image $(REGISTRY)/$(IMG) --name controller-runtime-example
143+
144+
$(ARTIFACT_DIR)/kind.kubeconfig: $(ARTIFACT_DIR) ## Run a kind cluster and generate a $KUBECONFIG for it.
145+
@if ! kind get clusters --quiet | grep --quiet controller-runtime-example; then kind create cluster --name controller-runtime-example; fi
146+
kind get kubeconfig --name controller-runtime-example > $(ARTIFACT_DIR)/kind.kubeconfig
147+
148+
$(ARTIFACT_DIR): ## Create a directory for test artifacts.
149+
mkdir -p $(ARTIFACT_DIR)
150+
151+
KCP_KUBECTL ?= PATH=$(LOCALBIN):$(PATH) KUBECONFIG=$(ARTIFACT_DIR)/kcp.kubeconfig kubectl
152+
KIND_KUBECTL ?= kubectl --kubeconfig $(ARTIFACT_DIR)/kind.kubeconfig
153+
154+
.PHONY: kcp-synctarget
155+
kcp-synctarget: kcp-workspace $(ARTIFACT_DIR)/syncer.yaml $(YQ) ## Add the kind cluster to kcp as a target for workloads.
156+
$(KIND_KUBECTL) apply -f $(ARTIFACT_DIR)/syncer.yaml
157+
$(eval DEPLOYMENT_NAME = $(shell $(YQ) 'select(.kind=="Deployment") | .metadata.name' < $(ARTIFACT_DIR)/syncer.yaml ))
158+
$(eval DEPLOYMENT_NAMESPACE = $(shell $(YQ) 'select(.kind=="Deployment") | .metadata.namespace' < $(ARTIFACT_DIR)/syncer.yaml ))
159+
$(KIND_KUBECTL) --namespace $(DEPLOYMENT_NAMESPACE) rollout status deployment/$(DEPLOYMENT_NAME)
160+
@if [[ ! -s $(ARTIFACT_DIR)/syncer.log ]]; then ( $(KIND_KUBECTL) --namespace $(DEPLOYMENT_NAMESPACE) logs deployment/$(DEPLOYMENT_NAME) -f >$(ARTIFACT_DIR)/syncer.log 2>&1 & ); fi
161+
$(KCP_KUBECTL) wait --for=condition=Ready synctarget/controller-runtime
162+
163+
$(ARTIFACT_DIR)/syncer.yaml: ## Generate the manifests necessary to register the kind cluster with kcp.
164+
$(KCP_KUBECTL) kcp workload sync controller-runtime --resources services --syncer-image ghcr.io/kcp-dev/kcp/syncer:v$(KCP_VERSION) --output-file $(ARTIFACT_DIR)/syncer.yaml
165+
166+
.PHONY: kcp-workspace
167+
kcp-workspace: $(KUBECTL_KCP) kcp-server ## Create a workspace in kcp for the controller-manager.
168+
$(KCP_KUBECTL) kcp workspace use '~'
169+
@if ! $(KCP_KUBECTL) kcp workspace use controller-runtime-example; then $(KCP_KUBECTL) kcp workspace create controller-runtime-example --type universal --enter; fi
170+
171+
.PHONY: kcp-server
172+
kcp-server: $(KCP) $(ARTIFACT_DIR)/kcp ## Run the kcp server.
173+
@if [[ ! -s $(ARTIFACT_DIR)/kcp.log ]]; then ( $(KCP) start -v 5 --root-directory $(ARTIFACT_DIR)/kcp --kubeconfig-path $(ARTIFACT_DIR)/kcp.kubeconfig --audit-log-maxsize 1024 --audit-log-mode=batch --audit-log-batch-max-wait=1s --audit-log-batch-max-size=1000 --audit-log-batch-buffer-size=10000 --audit-log-batch-throttle-burst=15 --audit-log-batch-throttle-enable=true --audit-log-batch-throttle-qps=10 --audit-policy-file ./test/e2e/audit-policy.yaml --audit-log-path $(ARTIFACT_DIR)/audit.log >$(ARTIFACT_DIR)/kcp.log 2>&1 & ); fi
174+
@while true; do if [[ ! -s $(ARTIFACT_DIR)/kcp.kubeconfig ]]; then sleep 0.2; else break; fi; done
175+
@while true; do if ! kubectl --kubeconfig $(ARTIFACT_DIR)/kcp.kubeconfig get --raw /readyz >$(ARTIFACT_DIR)/kcp.probe.log 2>&1; then sleep 0.2; else break; fi; done
176+
177+
$(ARTIFACT_DIR)/kcp: ## Create a directory for the kcp server data.
178+
mkdir -p $(ARTIFACT_DIR)/kcp
179+
180+
.PHONY: test-e2e-cleanup
181+
test-e2e-cleanup: ## Clean up processes and directories from an end-to-end test run.
182+
kind delete cluster --name controller-runtime-example || true
183+
rm -rf $(ARTIFACT_DIR) || true
184+
pkill -sigterm kcp || true
185+
pkill -sigterm kubectl || true
68186

69187
##@ Build
70188

@@ -81,68 +199,38 @@ run: manifests generate fmt vet ## Run a controller from your host.
81199

82200
.PHONY: docker-build
83201
docker-build: build ## Build docker image with the manager.
84-
docker build -t ${IMG} .
202+
docker build -t ${REGISTRY}/${IMG} .
85203

86204
.PHONY: docker-push
87205
docker-push: ## Push docker image with the manager.
88-
docker push ${IMG}
206+
docker push ${REGISTRY}/${IMG}
89207

90208
##@ Deployment
91209

92210
ifndef ignore-not-found
93211
ignore-not-found = false
94212
endif
95213

214+
KUBECONFIG ?= $(abspath ~/.kube/config )
215+
96216
.PHONY: install
97-
install: manifests kustomize ## Install APIResourceSchemas and APIExport into kcp (using $KUBECONFIG or ~/.kube/config).
98-
$(KUSTOMIZE) build config/kcp | kubectl apply -f -
217+
install: manifests $(KUSTOMIZE) ## Install APIResourceSchemas and APIExport into kcp (using $KUBECONFIG or ~/.kube/config).
218+
$(KUSTOMIZE) build config/kcp | kubectl --kubeconfig $(KUBECONFIG) apply -f -
99219

100220
.PHONY: uninstall
101-
uninstall: manifests kustomize ## Uninstall APIResourceSchemas and APIExport from kcp (using $KUBECONFIG or ~/.kube/config). Call with ignore-not-found=true to ignore resource not found errors during deletion.
102-
$(KUSTOMIZE) build config/kcp | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
221+
uninstall: manifests $(KUSTOMIZE) ## Uninstall APIResourceSchemas and APIExport from kcp (using $KUBECONFIG or ~/.kube/config). Call with ignore-not-found=true to ignore resource not found errors during deletion.
222+
$(KUSTOMIZE) build config/kcp | kubectl --kubeconfig $(KUBECONFIG) delete --ignore-not-found=$(ignore-not-found) -f -
103223

104224
.PHONY: deploy-crd
105-
deploy-crd: manifests kustomize ## Deploy controller
106-
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
107-
$(KUSTOMIZE) build config/default-crd | kubectl apply -f - || true
225+
deploy-crd: manifests $(KUSTOMIZE) ## Deploy controller
226+
cd config/manager && $(KUSTOMIZE) edit set image controller=${REGISTRY}/${IMG}
227+
$(KUSTOMIZE) build config/default-crd | kubectl --kubeconfig $(KUBECONFIG) apply -f - || true
108228

109229
.PHONY: deploy
110-
deploy: manifests kustomize ## Deploy controller
111-
cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
112-
$(KUSTOMIZE) build config/default | kubectl apply -f -
230+
deploy: manifests $(KUSTOMIZE) ## Deploy controller
231+
cd config/manager && $(KUSTOMIZE) edit set image controller=${REGISTRY}/${IMG}
232+
$(KUSTOMIZE) build config/default | kubectl --kubeconfig $(KUBECONFIG) apply -f -
113233

114234
.PHONY: undeploy
115235
undeploy: ## Undeploy controller. Call with ignore-not-found=true to ignore resource not found errors during deletion.
116-
$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
117-
118-
##@ Build Dependencies
119-
120-
## Location to install dependencies to
121-
LOCALBIN ?= $(shell pwd)/bin
122-
$(LOCALBIN):
123-
mkdir -p $(LOCALBIN)
124-
125-
## Tool Binaries
126-
KUSTOMIZE ?= $(LOCALBIN)/kustomize
127-
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
128-
ENVTEST ?= $(LOCALBIN)/setup-envtest
129-
130-
## Tool Versions
131-
KUSTOMIZE_VERSION ?= v3.8.7
132-
CONTROLLER_TOOLS_VERSION ?= v0.8.0
133-
134-
KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
135-
.PHONY: kustomize
136-
kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
137-
$(KUSTOMIZE): $(LOCALBIN)
138-
curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN)
139-
140-
.PHONY: controller-gen
141-
controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
142-
$(CONTROLLER_GEN): $(LOCALBIN)
143-
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
144-
145-
.PHONY: envtest
146-
envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
147-
$(ENVTEST): $(LOCALBIN)
148-
GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
236+
$(KUSTOMIZE) build config/default | kubectl --kubeconfig $(KUBECONFIG) delete --ignore-not-found=$(ignore-not-found) -f -

‎config/manager/kustomization.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
1212
kind: Kustomization
1313
images:
1414
- name: controller
15-
newName: controller
16-
newTag: latest
15+
newName: localhost/controller
16+
newTag: test

‎test/e2e/apibinding.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
apiVersion: apis.kcp.dev/v1alpha1
2+
kind: APIBinding
3+
metadata:
4+
name: data.my.domain
5+
spec:
6+
reference:
7+
workspace:
8+
path: WORKSPACE
9+
exportName: data.my.domain
10+
acceptedPermissionClaims:
11+
- resource: "secrets"
12+
- resource: "configmaps"
13+
- resource: "namespaces"

‎test/e2e/audit-policy.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: audit.k8s.io/v1
2+
kind: Policy
3+
omitStages:
4+
- RequestReceived
5+
omitManagedFields: true
6+
rules:
7+
- level: None
8+
nonResourceURLs:
9+
- "/api*"
10+
- "/version"
11+
12+
- level: Metadata
13+
resources:
14+
- group: ""
15+
resources: ["secrets", "configmaps"]
16+
- group: "authorization.k8s.io"
17+
resources: ["subjectaccessreviews"]
18+
19+
- level: Metadata
20+
verbs: ["list", "watch"]
21+
22+
- level: Metadata
23+
verbs: ["get", "delete"]
24+
omitStages:
25+
- ResponseStarted
26+
27+
- level: RequestResponse
28+
verbs: ["create", "update", "patch"]
29+
omitStages:
30+
- ResponseStarted

0 commit comments

Comments
 (0)