Skip to content

Commit d8f3572

Browse files
antoninbasantoninbas
committed
Add support for containerd to antrea-eks-node-init.yml
Containerd is available as an alternative to docker since EKS v1.21. Starting with EKS v1.23, containerd will also be the default container runtime. Currently antrea-eks-node-init.yml assumes that docker is always the container runtime, which means that Antrea cannot be run on EKS clusters using containerd. We fix this by adding support for containerd. We also make a couple of improvements to the startup script in antrea-eks-node-init.yml, to ensuire that it can be run again if the contents of the script change, which is useful for testing and may also be useful to roll out patches. Finally, we update our EKS CI to test with containerd instead of docker, since containerd will be the default soon. Fixes #3471 Signed-off-by: Antonin Bas <abas@vmware.com>
1 parent 069890c commit d8f3572

2 files changed

Lines changed: 74 additions & 15 deletions

File tree

‎build/yamls/antrea-eks-node-init.yml‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ spec:
2626
name: host-aws-node-run-dir
2727
containers:
2828
- name: node-init
29-
image: gcr.io/google-containers/startup-script:v1
29+
image: gcr.io/google-containers/startup-script:v2
3030
imagePullPolicy: IfNotPresent
3131
securityContext:
3232
privileged: true
@@ -42,7 +42,11 @@ spec:
4242
set -o pipefail
4343
set -o nounset
4444
45-
if [ -f /opt/cni/antrea-node-init-status ]; then
45+
# The STARTUP_SCRIPT environment variable (which is set to the contents of this
46+
# script) will be available when the script is executed :)
47+
checkpoint_path="/opt/cni/antrea-node-init-status-$(md5sum <<<"${STARTUP_SCRIPT}" | cut -c-32)"
48+
49+
if [ -f "$checkpoint_path" ]; then
4650
echo "Antrea node init already done. Exiting"
4751
exit
4852
fi
@@ -63,6 +67,11 @@ spec:
6367
sleep 2s
6468
done
6569
70+
echo "Detecting container runtime (docker / containerd) based on whether /var/run/docker.sock exists"
71+
container_runtime="docker"
72+
test -e /var/run/docker.sock || container_runtime="containerd"
73+
echo "Container runtime: $container_runtime"
74+
6675
# Wait for kubelet to register the file update. Default sync time is 5sec
6776
# https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/dockershim/network/cni/cni.go#L50
6877
sleep 5s
@@ -78,10 +87,19 @@ spec:
7887
echo "\n"
7988
for container_id in $(cat /var/run/aws-node/ipam.json | jq -r '.allocations | .[] | .containerID'); do
8089
echo "Restarting container with ID: ${container_id}"
81-
docker kill "${container_id}" || true
90+
if [[ "$container_runtime" == "docker" ]]; then
91+
docker kill "${container_id}" || true
92+
else
93+
ctr -n=k8s.io tasks kill "${container_id}" || true
94+
fi
8295
done
8396
84-
# Save the node init status, to avoid container restart in case of node-init pod restart or worker node reboot
85-
touch /opt/cni/antrea-node-init-status
97+
# Save the Node init status, to avoid container restart in case of node-init pod
98+
# restart or worker Node reboot,
99+
# Note that gcr.io/google-containers/startup-script:v2 also includes a similar
100+
# mechanism but it doesn't prevent the script from being run again when the Node
101+
# restarts, since the checkpoint path is located in the /tmp folder.
102+
# See https://github.com/kubernetes-retired/contrib/blob/master/startup-script/manage-startup-script.sh.
103+
touch "$checkpoint_path"
86104
87105
echo "Node initialization completed"

‎ci/test-conformance-eks.sh‎

Lines changed: 51 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ function echoerr {
2020
>&2 echo "$@"
2121
}
2222

23+
CLUSTER=""
2324
REGION="us-east-2"
2425
K8S_VERSION="1.21"
2526
AWS_NODE_TYPE="t3.medium"
@@ -32,6 +33,7 @@ KUBECONFIG_PATH="$HOME/jenkins/out/eks"
3233
MODE="report"
3334
TEST_SCRIPT_RC=0
3435
KUBE_CONFORMANCE_IMAGE_VERSION=auto
36+
INSTALL_EKSCTL=true
3537

3638
_usage="Usage: $0 [--cluster-name <EKSClusterNameToUse>] [--kubeconfig <KubeconfigSavePath>] [--k8s-version <ClusterVersion>]\
3739
[--aws-access-key <AccessKey>] [--aws-secret-key <SecretKey>] [--aws-region <Region>] [--ssh-key <SSHKey] \
@@ -48,7 +50,8 @@ Setup a EKS cluster to run K8s e2e community tests (Conformance & Network Policy
4850
--ssh-key The path of key to be used for ssh access to worker nodes.
4951
--log-mode Use the flag to set either 'report', 'detail', or 'dump' level data for sonobouy results.
5052
--setup-only Only perform setting up the cluster and run test.
51-
--cleanup-only Only perform cleaning up the cluster."
53+
--cleanup-only Only perform cleaning up the cluster.
54+
--skip-eksctl-install Do not install the latest eksctl version. Eksctl must be installed already."
5255

5356
function print_usage {
5457
echoerr "$_usage"
@@ -109,6 +112,10 @@ case $key in
109112
RUN_ALL=false
110113
shift
111114
;;
115+
--skip-eksctl-install)
116+
INSTALL_EKSCTL=false
117+
shift
118+
;;
112119
-h|--help)
113120
print_usage
114121
exit 0
@@ -120,6 +127,39 @@ case $key in
120127
esac
121128
done
122129

130+
if [[ "$CLUSTER" == "" ]]; then
131+
echoerr "--cluster-name is required"
132+
exit 1
133+
fi
134+
135+
function generate_eksctl_config() {
136+
AMI_ID=$(aws ssm get-parameter \
137+
--name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id \
138+
--query "Parameter.Value" --output text)
139+
140+
cat > eksctl-containerd.yaml <<EOF
141+
---
142+
apiVersion: eksctl.io/v1alpha5
143+
kind: ClusterConfig
144+
metadata:
145+
name: ${CLUSTER}
146+
region: ${REGION}
147+
version: "${K8S_VERSION}"
148+
managedNodeGroups:
149+
- name: containerd
150+
instanceType: ${AWS_NODE_TYPE}
151+
desiredCapacity: 2
152+
ami: ${AMI_ID}
153+
ssh:
154+
allow: true
155+
publicKeyPath: ${SSH_KEY_PATH}
156+
overrideBootstrapCommand: |
157+
#!/bin/bash
158+
/etc/eks/bootstrap.sh ${CLUSTER} --container-runtime containerd
159+
EOF
160+
echo "eksctl-containerd.yaml"
161+
}
162+
123163
function setup_eks() {
124164

125165
echo "=== This cluster to be created is named: ${CLUSTER} ==="
@@ -139,20 +179,21 @@ ${AWS_SECRET_KEY}
139179
${REGION}
140180
JSON
141181
EOF
142-
echo "=== Installing latest version of eksctl ==="
143-
curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
144-
sudo mv /tmp/eksctl /usr/local/bin
182+
if [[ "$INSTALL_EKSCTL" == true ]]; then
183+
echo "=== Installing latest version of eksctl ==="
184+
curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
185+
sudo mv /tmp/eksctl /usr/local/bin
186+
fi
145187
set -e
146188
printf "\n"
189+
echo "=== Using the following eksctl ==="
190+
which eksctl
147191
echo "=== Using the following kubectl ==="
148192
which kubectl
149193

150194
echo '=== Creating a cluster in EKS ==='
151-
eksctl create cluster \
152-
--name ${CLUSTER} --region ${REGION} --version=${K8S_VERSION} \
153-
--nodegroup-name workers --node-type ${AWS_NODE_TYPE} --nodes 2 \
154-
--ssh-access --ssh-public-key ${SSH_KEY_PATH} \
155-
--managed
195+
config="$(generate_eksctl_config)"
196+
eksctl create cluster -f $config
156197
if [[ $? -ne 0 ]]; then
157198
echo "=== Failed to deploy EKS cluster! ==="
158199
exit 1
@@ -203,7 +244,7 @@ function deliver_antrea_to_eks() {
203244

204245
kubectl get nodes -o wide --no-headers=true | awk '{print $7}' | while read IP; do
205246
scp -o StrictHostKeyChecking=no -i ${SSH_PRIVATE_KEY_PATH} ${antrea_image}.tar ec2-user@${IP}:~
206-
ssh -o StrictHostKeyChecking=no -i ${SSH_PRIVATE_KEY_PATH} -n ec2-user@${IP} "sudo docker load -i ~/${antrea_image}.tar ; sudo docker tag ${DOCKER_IMG_NAME}:${DOCKER_IMG_VERSION} ${DOCKER_IMG_NAME}:latest"
247+
ssh -o StrictHostKeyChecking=no -i ${SSH_PRIVATE_KEY_PATH} -n ec2-user@${IP} "sudo ctr -n=k8s.io images import ~/${antrea_image}.tar ; sudo ctr -n=k8s.io images tag ${DOCKER_IMG_NAME}:${DOCKER_IMG_VERSION} ${DOCKER_IMG_NAME}:latest --force"
207248
done
208249
rm ${antrea_image}.tar
209250

0 commit comments

Comments
 (0)