Upload multi-cluster e2e coverage to codecov (#3984)

1. Add a new target on Makefile to build MC controller image with
   instrumented binary
2. Add steps on test-mc.sh to collect coverage files from leader and member cluster
   controllers and upload it to codecov

Signed-off-by: Lan Luo <luola@vmware.com>

Author: luolanzone

Makefile

@@ -398,6 +398,17 @@ endif
 	docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller
 	docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION)
 
+.PHONY: antrea-mc-controller-coverage
+antrea-mc-controller-coverage:
+	@echo "===> Building antrea/antrea-mc-controller-coverage Docker image <==="
+ifneq ($(NO_PULL),)
+	docker build -t antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) -f multicluster/build/images/Dockerfile.build.coverage $(DOCKER_BUILD_ARGS) .
+else
+	docker build --pull -t antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) -f multicluster/build/images/Dockerfile.build.coverage $(DOCKER_BUILD_ARGS) .
+endif
+	docker tag antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) antrea/antrea-mc-controller-coverage
+	docker tag antrea/antrea-mc-controller-coverage:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller-coverage
+
 .PHONY: flow-visibility-clickhouse-monitor
 flow-visibility-clickhouse-monitor:
 	@echo "===> Building antrea/flow-visibility-clickhouse-monitor Docker image <==="

ci/jenkins/test-mc.sh

@@ -34,6 +34,8 @@ EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east"
 WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west"
 ENABLE_MC_GATEWAY=false
 IS_CONTAINERD=false
+CODECOV_TOKEN=""
+COVERAGE=false
 
 multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
 membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
@@ -42,15 +44,17 @@ membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG)
 CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=48h"
 
 _usage="Usage: $0 [--kubeconfigs-path <KubeconfigSavePath>] [--workdir <HomePath>]
-                  [--testcase <e2e>] [--mc-gateway]
+                  [--testcase <e2e>] [--mc-gateway] [--codecov-token] [--coverage]
 
 Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set.
 
         --kubeconfigs-path            Path of cluster set kubeconfigs.
         --workdir                     Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR.
         --testcase                    Antrea multi-cluster e2e test cases on a Linux cluster set.
         --registry                    The docker registry to use instead of dockerhub.
-        --mc-gateway                  Enable Multicluster Gateway."
+        --mc-gateway                  Enable Multicluster Gateway.
+        --codecov-token               Token used to upload coverage report(s) to Codecov.
+        --coverage                    Run e2e with coverage."
 
 function print_usage {
     echoerr "$_usage"
@@ -82,6 +86,14 @@ case $key in
     ENABLE_MC_GATEWAY=true
     shift
     ;;
+    --codecov-token)
+    CODECOV_TOKEN="$2"
+    shift 2
+    ;;
+    --coverage)
+    COVERAGE=true
+    shift
+    ;;
     -h|--help)
     print_usage
     exit 0
@@ -161,10 +173,9 @@ function wait_for_antrea_multicluster_pods_ready {
 function wait_for_multicluster_controller_ready {
     echo "====== Deploying Antrea Multicluster Leader Cluster with ${LEADER_CLUSTER_CONFIG} ======"
     kubectl create ns antrea-multicluster  "${LEADER_CLUSTER_CONFIG}" || true
-    kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}"
     kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-leader-global.yml "${LEADER_CLUSTER_CONFIG}"
+    kubectl apply -f ./multicluster/test/yamls/leader-manifest.yml "${LEADER_CLUSTER_CONFIG}"
     kubectl rollout status deployment/antrea-mc-controller -n antrea-multicluster "${LEADER_CLUSTER_CONFIG}" || true
-    kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}"
     kubectl create -f ./multicluster/test/yamls/leader-access-token-secret.yml "${LEADER_CLUSTER_CONFIG}" || true
     kubectl get secret -n antrea-multicluster leader-access-token "${LEADER_CLUSTER_CONFIG}" -o yaml > ./multicluster/test/yamls/leader-access-token.yml
 
@@ -180,7 +191,7 @@ function wait_for_multicluster_controller_ready {
     for config in "${membercluster_kubeconfigs[@]}";
     do
         echo "====== Deploying Antrea Multicluster Member Cluster with ${config} ======"
-        kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member.yml ${config}
+        kubectl apply -f ./multicluster/test/yamls/member-manifest.yml ${config}
         kubectl rollout status deployment/antrea-mc-controller -n kube-system ${config}
         kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml ${config}
     done
@@ -191,6 +202,34 @@ function wait_for_multicluster_controller_ready {
     kubectl apply -f ./multicluster/test/yamls/clusterset.yml "${LEADER_CLUSTER_CONFIG}"
 }
 
+# We run the function in a subshell with "set -e" to ensure that it exits in
+# case of error (e.g. integrity check), no matter the context in which the
+# function is called.
+function run_codecov { (set -e
+    flag=$1
+    file=$2
+    dir=$3
+
+    rm -f trustedkeys.gpg codecov
+    # This is supposed to be a one-time step, but there should be no harm in
+    # getting the key every time. It does not come from the codecov.io
+    # website. Anyway, this is needed when the VM is re-created for every test.
+    curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import
+    curl -Os https://uploader.codecov.io/latest/linux/codecov
+    curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
+    curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
+
+    # Check that the sha256 matches the signature
+    gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
+    # Then check the integrity of the codecov binary
+    shasum -a 256 -c codecov.SHA256SUM
+
+    chmod +x codecov
+    ./codecov -c -t ${CODECOV_TOKEN} -F ${flag} -f ${file} -s ${dir} -C ${GIT_COMMIT} -r antrea-io/antrea
+
+    rm -f trustedkeys.gpg codecov
+)}
+
 function deliver_antrea_multicluster {
     echo "====== Building Antrea for the Following Commit ======"
     export GO111MODULE=on
@@ -231,10 +270,17 @@ function deliver_multicluster_controller {
     export GOROOT=/usr/local/go
     export PATH=${GOROOT}/bin:$PATH
 
-    export NO_PULL=1;make antrea-mc-controller
-
-    docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar
-    ./multicluster/hack/generate-manifest.sh -l antrea-multicluster > ./multicluster/test/yamls/manifest.yml
+    if $COVERAGE;then
+      export NO_PULL=1;make antrea-mc-controller-coverage
+      docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller-coverage:latest -o "${WORKDIR}"/antrea-mcs.tar
+      ./multicluster/hack/generate-manifest.sh -l antrea-multicluster -c > ./multicluster/test/yamls/leader-manifest.yml
+      ./multicluster/hack/generate-manifest.sh -m -c > ./multicluster/test/yamls/member-manifest.yml
+    else
+      export NO_PULL=1;make antrea-mc-controller
+      docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar
+      ./multicluster/hack/generate-manifest.sh -l antrea-multicluster > ./multicluster/test/yamls/leader-manifest.yml
+      ./multicluster/hack/generate-manifest.sh -m > ./multicluster/test/yamls/member-manifest.yml
+    fi
 
     for kubeconfig in "${multicluster_kubeconfigs[@]}"
     do
@@ -264,7 +310,7 @@ function deliver_multicluster_controller {
 }
 
 function run_multicluster_e2e {
-   echo "====== Running Multicluster e2e Tests ======"
+    echo "====== Running Multicluster e2e Tests ======"
     export GO111MODULE=on
     export GOPATH=${WORKDIR}/go
     export GOROOT=/usr/local/go
@@ -311,11 +357,12 @@ EOF
     done
 
     set +e
-    mkdir -p `pwd`/antrea-multicluster-test-logs
+    CURRENT_DIR=`pwd`
+    mkdir -p ${CURRENT_DIR}/antrea-multicluster-test-logs
     if ${ENABLE_MC_GATEWAY};then
-      go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir  `pwd`/antrea-multicluster-test-logs --mc-gateway
+      go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs --mc-gateway
     else
-      go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir  `pwd`/antrea-multicluster-test-logs
+      go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs
     fi
 
     if [[ "$?" != "0" ]]; then
@@ -324,6 +371,22 @@ EOF
     set -e
 }
 
+function collect_coverage {
+    COVERAGE_DIR=$1
+    timestamp=$(date +%Y%m%d%H%M%S)
+    echo "====== Collect Multicluster e2e Tests Coverage Files ======"
+    for kubeconfig in "${multicluster_kubeconfigs[@]}"; do
+      namespace="kube-system"
+      if [[ ${kubeconfig} =~ "leader" ]];then
+        namespace="antrea-multicluster"
+      fi
+      mc_controller_pod_name="$(kubectl get pods --selector=app=antrea,component=antrea-mc-controller -n ${namespace} --no-headers=true ${kubeconfig} | awk '{ print $1 }')"
+      controller_pid="$(kubectl exec -i $mc_controller_pod_name -n ${namespace} ${kubeconfig} -- pgrep antrea)"
+      kubectl exec -i $mc_controller_pod_name -n ${namespace} ${kubeconfig} -- kill -SIGINT $controller_pid
+      kubectl cp ${namespace}/$mc_controller_pod_name:antrea-mc-controller.cov.out ${COVERAGE_DIR}/$mc_controller_pod_name-$timestamp ${kubeconfig}
+    done
+}
+
 trap clean_multicluster EXIT
 clean_tmp
 clean_images
@@ -341,6 +404,16 @@ if [[ ${TESTCASE} =~ "e2e" ]]; then
     deliver_antrea_multicluster
     deliver_multicluster_controller
     run_multicluster_e2e
+    if $COVERAGE;then
+      CURRENT_DIR=`pwd`
+      rm -rf mc-e2e-coverage
+      mkdir -p mc-e2e-coverage
+      collect_coverage ${CURRENT_DIR}/mc-e2e-coverage
+      # Backup coverage files for later analysis
+      set +e;find ${DEFAULT_WORKDIR}/mc-e2e-coverage -maxdepth 1 -mtime +1 -type f | xargs -n 1 rm;set -e; # Clean up backup files older than one day.
+      cp -r mc-e2e-coverage ${DEFAULT_WORKDIR}
+      run_codecov "e2e-tests" "*antrea-mc*" "${CURRENT_DIR}/mc-e2e-coverage"
+    fi
 fi
 
 if [[ ${TEST_FAILURE} == true ]]; then

ci/jenkins/test-vmc.sh

@@ -42,7 +42,7 @@ CONTROL_PLANE_NODE_ROLE="master"
 
 _usage="Usage: $0 [--cluster-name <VMCClusterNameToUse>] [--kubeconfig <KubeconfigSavePath>] [--workdir <HomePath>]
                   [--log-mode <SonobuoyResultLogLevel>] [--testcase <e2e|conformance|all-features-conformance|whole-conformance|networkpolicy>]
-                  [--garbage-collection] [--setup-only] [--cleanup-only] [--coverage] [--test-only] [--registry]
+                  [--garbage-collection] [--setup-only] [--cleanup-only] [--coverage] [--test-only] [--codecov-token] [--registry]
 
 Setup a VMC cluster to run K8s e2e community tests (E2e, Conformance, all features Conformance, whole Conformance & Network Policy).
 
@@ -301,7 +301,7 @@ function copy_image {
   ${SSH_WITH_ANTREA_CI_KEY} -n capv@${IP} "sudo crictl images | grep '<none>' | awk '{print \$3}' | xargs -r crictl rmi"
 }
 
-# We run the function in a subshell with "set -e" to ensure that it exists in
+# We run the function in a subshell with "set -e" to ensure that it exits in
 # case of error (e.g. integrity check), no matter the context in which the
 # function is called.
 function run_codecov { (set -e

multicluster/Makefile

@@ -85,6 +85,10 @@ endif
 build: generate fmt vet ## Build manager binary.
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/antrea-mc-controller antrea.io/antrea/multicluster/cmd/...
 
+.PHONY: antrea-mc-instr-binary
+antrea-mc-instr-binary:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go test -tags testbincover -covermode count -coverpkg=antrea.io/antrea/multicluster/... -c -o bin/antrea-mc-controller-coverage antrea.io/antrea/multicluster/cmd/...
+
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 

multicluster/build/images/Dockerfile.build.coverage

@@ -0,0 +1,21 @@
+ARG GO_VERSION
+FROM golang:${GO_VERSION} as antrea-build
+
+WORKDIR /antrea
+
+COPY go.mod /antrea/go.mod
+
+RUN go mod download
+
+COPY . /antrea
+
+RUN cd multicluster && make antrea-mc-instr-binary
+
+FROM ubuntu:20.04
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="The Docker image to deploy the Antrea Multicluster controller with code coverage measurement enabled (used for testing)."
+
+USER root
+
+COPY --from=antrea-build /antrea/multicluster/bin/antrea-mc-controller-coverage /

multicluster/cmd/multicluster-controller/bincover_run_main_test.go

@@ -0,0 +1,28 @@
+// Copyright 2022 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build testbincover
+// +build testbincover
+
+package main
+
+import (
+	"testing"
+
+	"github.com/confluentinc/bincover"
+)
+
+func TestBincoverRunMain(t *testing.T) {
+	bincover.RunTest(main)
+}

multicluster/config/overlays/leader-ns/coverage/manager_command_patch_coverage.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+        - command: ["/bin/sh"]
+          args: ["-c", "/antrea-mc-controller-coverage -test.run=TestBincoverRunMain -test.coverprofile=antrea-mc-controller.cov.out leader; while true; do sleep 5 & wait $!; done"]
+          name: antrea-mc-controller
+          image: projects.registry.vmware.com/antrea/antrea-mc-controller-coverage:latest

multicluster/config/overlays/member/coverage/manager_command_patch_coverage.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+        - command: ["/bin/sh"]
+          args: ["-c", "/antrea-mc-controller-coverage -test.run=TestBincoverRunMain -test.coverprofile=antrea-mc-controller.cov.out member; while true; do sleep 5 & wait $!; done"]
+          name: antrea-mc-controller
+          image: projects.registry.vmware.com/antrea/antrea-mc-controller-coverage:latest

multicluster/hack/generate-manifest.sh

@@ -27,6 +27,7 @@ Generate a YAML manifest for Antrea MultiCluster using Kustomize and print it to
         --leader  | -l                      Generate a per-namespace manifest for a Cluster as leader in a ClusterSet.
                                             All resources will be in the given namespace
         --member  | -m                      Generate a manifest for a Cluster as member in a ClusterSet
+        --coverage| -c                      Generate a manifest which supports measuring code coverage
         --help    | -h                      Print this message and exit
 
 Environment variables IMG_NAME and IMG_TAG must be set when release mode is enabled.
@@ -43,6 +44,7 @@ function print_help {
 OVERLAY=member
 NAMESPACE=antrea-multicluster
 MODE=""
+COVERAGE=false
 
 while [[ $# -gt 0 ]]
 do
@@ -52,6 +54,10 @@ case $key in
     MODE="release"
     shift
     ;;
+    --coverage|-c)
+    COVERAGE=true
+    shift
+    ;;
     --leader|-l)
     OVERLAY=leader-ns
     NAMESPACE="$2"
@@ -113,7 +119,7 @@ then
     cp $KUSTOMIZATION_DIR/overlays/leader-ns/prefix_transformer.yaml .
     sed -ie "s/antrea-multicluster/$NAMESPACE/g" prefix_transformer.yaml
 
-cat << EOF > kustomization.yaml
+    cat << EOF > kustomization.yaml
 namespace: $NAMESPACE
 
 bases:
@@ -122,8 +128,13 @@ bases:
 transformers:
   - prefix_transformer.yaml
 EOF
+
+    if $COVERAGE; then
+        cp ../../overlays/$OVERLAY/coverage/manager_command_patch_coverage.yaml .
+        $KUSTOMIZE edit add patch --path ./manager_command_patch_coverage.yaml
+    fi
 else
-cat << EOF > kustomization.yaml
+    cat << EOF > kustomization.yaml
 bases:
   - ../overlays/$OVERLAY
 EOF
@@ -134,5 +145,11 @@ if [ "$MODE" == "release" ]; then
 else
     $KUSTOMIZE edit set image antrea/antrea-mc-controller=projects.registry.vmware.com/antrea/antrea-mc-controller:latest
 fi
+
+if [ "$OVERLAY" == "member" ] && $COVERAGE; then
+    cp ../overlays/$OVERLAY/coverage/manager_command_patch_coverage.yaml .
+    $KUSTOMIZE edit add patch --path ./manager_command_patch_coverage.yaml
+fi
+
 $KUSTOMIZE build
 rm -rf $TMP_DIR