Ubuntu 2204 Instance config

config/aws-instance.yaml

@@ -7,3 +7,7 @@ images:
     ssm_path: /aws/service/eks/optimized-ami/1.25/amazon-linux-2/recommended/image_id
     instance_type: m6a.large
     user_data_file: userdata.sh
+  ubuntu-2204:
+    ssm_path: /aws/service/canonical/ubuntu/server/focal/stable/current/amd64/hvm/ebs-gp2/ami-id
+    instance_type: m6a.large
+    user_data_file: ubuntu2204.yaml

config/configure.sh

@@ -0,0 +1,255 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -o xtrace
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "$(python -V 2>&1)" =~ "Python 2" ]]; then
+  # found python2, just use that
+  PYTHON="python"
+elif [[ -f "/usr/bin/python2.7" ]]; then
+  # System python not defaulted to python 2 but using 2.7 during migration
+  PYTHON="/usr/bin/python2.7"
+else
+  # No python2 either by default, let's see if we can find python3
+  PYTHON="python3"
+  if ! command -v ${PYTHON} >/dev/null 2>&1; then
+    echo "ERROR Python not found. Aborting."
+    exit 2
+  fi
+fi
+echo "Version : " $(${PYTHON} -V 2>&1)
+
+# CONTAINERD_HOME is the directory for containerd.
+CONTAINERD_HOME="/home/containerd"
+cd "${CONTAINERD_HOME}"
+# KUBE_HOME is the directory for kubernetes.
+KUBE_HOME="/home/kubernetes"
+
+# TODO(upodroid) remove this function after you have resolved the pull-refs below
+# fetch_metadata fetches metadata from GCE metadata server.
+# Var set:
+# 1. Metadata key: key of the metadata.
+fetch_metadata() {
+  local -r key=$1
+  local -r attributes="http://metadata.google.internal/computeMetadata/v1/instance/attributes"
+  if curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" "${attributes}/" | \
+    grep -q "^${key}$"; then
+    curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" \
+      "${attributes}/${key}"
+  fi
+}
+
+# Install yq to parse some yaml
+curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_amd64.tar.gz |\
+  tar xz && mv yq_linux_amd64 /usr/local/bin/yq
+
+# fetch_env fetches environment variables from GCE metadata server
+# and generate a env file under ${CONTAINERD_HOME}. It assumes that
+# the environment variables in metadata are in yaml format.
+fetch_env() {
+  local -r env_file_name=$1
+  (
+    umask 077;
+    local -r tmp_env_file="/tmp/${env_file_name}.yaml"
+    tmp_env_content=$(curl -sSL "$(cat /tmp/bootstrap/extra-fetches.yaml | yq "."${env_file_name}" // \"\"")")
+    if [ -z "${tmp_env_content}" ]; then
+      echo "No environment variable is specified in ${env_file_name}"
+      return
+    fi
+    echo "${tmp_env_content}" > "${tmp_env_file}"
+    # Convert the yaml format file into a shell-style file.
+    eval $(${PYTHON} -c '''
+import pipes,sys,yaml
+# check version of python and call methods appropriate for that version
+if sys.version_info[0] < 3:
+    items = yaml.load(sys.stdin).iteritems()
+else:
+    items = yaml.load(sys.stdin, Loader=yaml.BaseLoader).items()
+for k,v in items:
+  print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
+''' < "${tmp_env_file}" > "${CONTAINERD_HOME}/${env_file_name}")
+    rm -f "${tmp_env_file}"
+  )
+}
+
+# is_preloaded checks whether a package has been preloaded in the image.
+is_preloaded() {
+  local -r tar=$1
+  local -r sha1=$2
+  grep -qs "${tar},${sha1}" "${KUBE_HOME}/preload_info"
+}
+
+# CONTAINERD_ENV_METADATA is the metadata key for containerd envs.
+CONTAINERD_ENV_METADATA="containerd-env"
+fetch_env ${CONTAINERD_ENV_METADATA}
+if [ -f "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}" ]; then
+  source "${CONTAINERD_HOME}/${CONTAINERD_ENV_METADATA}"
+fi
+
+
+# CONTAINERD_PKG_PREFIX is the prefix of the cri-containerd tarball name.
+# By default use the release tarball with cni built in.
+pkg_prefix=${CONTAINERD_PKG_PREFIX:-"cri-containerd-cni"}
+# Behave differently for test and production.
+if [ "${CONTAINERD_TEST:-"false"}"  != "true" ]; then
+    # CONTAINERD_DEPLOY_PATH is the gcs path where cri-containerd tarball is stored.
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-release"}
+  # CONTAINERD_VERSION is the cri-containerd version to use.
+  version=${CONTAINERD_VERSION:-""}
+else
+  deploy_path=${CONTAINERD_DEPLOY_PATH:-"cri-containerd-staging"}
+
+  # TODO(upodroid) Revisit pull_refs
+  # PULL_REFS_METADATA is the metadata key of PULL_REFS from prow.
+  # PULL_REFS_METADATA="PULL_REFS"
+  # pull_refs=$(fetch_metadata "${PULL_REFS_METADATA}")
+  # if [ ! -z "${pull_refs}" ]; then
+  #   deploy_dir=$(echo "${pull_refs}" | sha1sum | awk '{print $1}')
+  #   deploy_path="${deploy_path}/containerd/${deploy_dir}"
+  # fi
+
+  # TODO(random-liu): Put version into the metadata instead of
+  # deciding it in cloud init. This may cause issue to reboot test.
+  if [ $(uname -m) == "aarch64" ]; then
+    version=$(curl -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+      -H "Accept: application/vnd.github.v3+json" \
+      "https://api.github.com/repos/containerd/containerd/releases/latest" \
+      | jq -r .tag_name \
+      | sed "s:v::g")
+  else
+    version=$(set +x; curl -X GET "${HEADERS[@]}" -f --ipv4 --retry 6 --retry-delay 3 --silent --show-error \
+      https://storage.googleapis.com/${deploy_path}/latest)
+  fi
+fi
+
+TARBALL_GCS_NAME="${pkg_prefix}-${version}.linux-amd64.tar.gz"
+# TARBALL_GCS_PATH is the path to download cri-containerd tarball for node e2e.
+TARBALL_GCS_PATH="https://storage.googleapis.com/${deploy_path}/${TARBALL_GCS_NAME}"
+# TARBALL is the name of the tarball after being downloaded.
+TARBALL="containerd.tar.gz"
+# CONTAINERD_TAR_SHA1 is the sha1sum of containerd tarball.
+tar_sha1="${CONTAINERD_TAR_SHA1:-""}"
+
+if [ -z "${version}" ]; then
+  # Try using preloaded containerd if version is not specified.
+  tarball_gcs_pattern="${pkg_prefix}-.*.linux-amd64.tar.gz"
+  if is_preloaded "${tarball_gcs_pattern}" "${tar_sha1}"; then
+    echo "CONTAINERD_VERSION is not set, use preloaded containerd"
+  else
+    echo "CONTAINERD_VERSION is not set, and containerd is not preloaded"
+    exit 1
+  fi
+else
+  if [ $(uname -m) == "aarch64" ]; then
+    curl -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 --retry-delay 10 \
+	"https://github.com/containerd/containerd/releases/download/v${version}/cri-containerd-${version}-linux-arm64.tar.gz"
+    tar xvf "${TARBALL}"
+    rm -f "${TARBALL}"
+  elif is_preloaded "${TARBALL_GCS_NAME}" "${tar_sha1}"; then
+    echo "${TARBALL_GCS_NAME} is preloaded"
+  else
+    # Download and untar the release tar ball.
+    $(set +x; curl -X GET "${HEADERS[@]}" -f --ipv4 -Lo "${TARBALL}" --connect-timeout 20 --max-time 300 --retry 6 \
+      --retry-delay 10 "${TARBALL_GCS_PATH}")
+    tar xvf "${TARBALL}"
+    rm -f "${TARBALL}"
+  fi
+fi
+
+# Remove crictl shipped with containerd, use crictl installed
+# by kube-up.sh.
+rm -f "${CONTAINERD_HOME}/usr/local/bin/crictl"
+rm -f "${CONTAINERD_HOME}/etc/crictl.yaml"
+
+# Generate containerd config
+config_path="${CONTAINERD_CONFIG_PATH:-"/etc/containerd/config.toml"}"
+mkdir -p $(dirname ${config_path})
+cni_bin_dir="${CONTAINERD_HOME}/opt/cni/bin"
+cni_template_path="${CONTAINERD_HOME}/opt/containerd/cluster/gce/cni.template"
+if [ "${KUBERNETES_MASTER:-}" != "true" ]; then
+  if [ "${NETWORK_POLICY_PROVIDER:-"none"}" != "none" ] || [ "${ENABLE_NETD:-}" == "true" ]; then
+    # Use Kubernetes cni daemonset on node if network policy provider is specified
+    # or netd is enabled.
+    cni_bin_dir="${KUBE_HOME}/bin"
+    cni_template_path=""
+  fi
+fi
+# Use systemd cgroup if specified in env
+systemdCgroup="${CONTAINERD_SYSTEMD_CGROUP:-"false"}"
+log_level="${CONTAINERD_LOG_LEVEL:-"info"}"
+max_container_log_line="${CONTAINERD_MAX_CONTAINER_LOG_LINE:-16384}"
+cat > ${config_path} <<EOF
+version = 2
+# Kubernetes requires the cri plugin.
+required_plugins = ["io.containerd.grpc.v1.cri"]
+# Kubernetes doesn't use containerd restart manager.
+disabled_plugins = ["io.containerd.internal.v1.restart"]
+
+[debug]
+  level = "${log_level}"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "0"
+  max_container_log_line_size = ${max_container_log_line}
+[plugins."io.containerd.grpc.v1.cri".cni]
+  bin_dir = "${cni_bin_dir}"
+  conf_dir = "/etc/cni/net.d"
+  conf_template = "${cni_template_path}"
+[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+  endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"]
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  default_runtime_name = "${CONTAINERD_DEFAULT_RUNTIME:-"runc"}"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+  BinaryName = "${CONTAINERD_HOME}/usr/local/sbin/runc"
+  SystemdCgroup = ${systemdCgroup}
+EOF
+chmod 644 "${config_path}"
+
+# containerd_extra_runtime_handler is the extra runtime handler to install.
+containerd_extra_runtime_handler=${CONTAINERD_EXTRA_RUNTIME_HANDLER:-""}
+if [[ -n "${containerd_extra_runtime_handler}" ]]; then
+  cat >> ${config_path} <<EOF
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${containerd_extra_runtime_handler}]
+  runtime_type = "${CONTAINERD_EXTRA_RUNTIME_TYPE:-io.containerd.runc.v1}"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.${containerd_extra_runtime_handler}.options]
+${CONTAINERD_EXTRA_RUNTIME_OPTIONS:-}
+EOF
+fi
+
+echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \
+  /etc/profile.d/containerd_env.sh
+
+# Run extra init script for test.
+if [ "${CONTAINERD_TEST:-"false"}"  == "true" ]; then
+  # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded.
+  EXTRA_INIT_SCRIPT="containerd-extra-init.sh"
+  extra_init="$(cat /tmp/bootstrap/extra-fetches.yaml | yq '.extra_init // ""')"
+  # Return if containerd-extra-init-sh is not set.
+  if [ -z "${extra_init}" ]; then
+    exit 0
+  fi
+  extra_init=$(curl -sSL "$(cat /tmp/bootstrap/extra-fetches.yaml | yq .extra_init)")
+  echo "${extra_init}" > "${EXTRA_INIT_SCRIPT}"
+  chmod 544 "${EXTRA_INIT_SCRIPT}"
+  ./${EXTRA_INIT_SCRIPT}
+fi

config/ubuntu2204.yaml

@@ -0,0 +1,67 @@
+#cloud-config
+# This bootstraps a public ubuntu 2204 image from scratch.
+write_files:
+  - path: /tmp/bootstrap/extra-fetches.yaml
+    content: |
+      # valid keys are containerd-env, and extra_init
+      containerd-env: https://raw.githubusercontent.com/kubernetes/test-infra/master/jobs/e2e_node/containerd/containerd-main/env
+  - path: /etc/systemd/system/containerd-installation.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=Download and install containerd binaries and configurations.
+      After=network-online.target
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStartPre=/bin/mkdir -p /home/containerd
+      ExecStartPre=/bin/mount --bind /home/containerd /home/containerd
+      ExecStartPre=/bin/mount -o remount,exec /home/containerd
+      ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -o /home/containerd/configure.sh https://raw.githubusercontent.com/kubernetes-sigs/provider-aws-test-infra/main/config/configure.sh
+      ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh
+      ExecStart=/home/containerd/configure.sh
+      [Install]
+      WantedBy=containerd.target
+  - path: /etc/systemd/system/containerd.service
+    permissions: 0644
+    owner: root
+    content: |
+      # installed by cloud-init
+      [Unit]
+      Description=containerd container runtime
+      Documentation=https://containerd.io
+      After=containerd-installation.service
+      [Service]
+      Restart=always
+      RestartSec=5
+      Delegate=yes
+      KillMode=process
+      OOMScoreAdjust=-999
+      LimitNOFILE=1048576
+      # Having non-zero Limit*s causes performance problems due to accounting overhead
+      # in the kernel. We recommend using cgroups to do container-local accounting.
+      LimitNPROC=infinity
+      LimitCORE=infinity
+      TasksMax=infinity
+      ExecStartPre=/sbin/modprobe overlay
+      ExecStart=/home/containerd/usr/local/bin/containerd
+      [Install]
+      WantedBy=containerd.target
+  - path: /etc/systemd/system/containerd.target
+    permissions: 0644
+    owner: root
+    content: |
+      [Unit]
+      Description=Containerd
+      [Install]
+      WantedBy=multi-user.target
+runcmd:
+  # Stop the existing containerd service if there is one. (for Docker 18.09+)
+  - systemctl is-active containerd && systemctl stop containerd
+  - systemctl daemon-reload
+  - systemctl enable containerd-installation.service
+  - systemctl enable containerd.service
+  - systemctl enable containerd.target
+  - systemctl start containerd.target