[installer-tests] Add tests for k8s versions and ubuntu version

Author: nandajavarma

.werft/installer-tests.ts

@@ -33,12 +33,15 @@ interface TestConfig {
     CLOUD: string;
 }
 
+const k8s_version: string = randK8sVersion(testConfig)
+const os_version: string = randOsVersion() // applicable only for k3s
+
 // Each of the TEST_CONFIGURATIONS define an integration test end-to-end
 // It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
 const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
     STANDARD_GKE_TEST: {
         CLOUD: "gcp",
-        DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
+        DESCRIPTION: `Deploy Gitpod on GKE(version ${k8s_version})`,
         PHASES: [
             "STANDARD_GKE_CLUSTER",
             "CERT_MANAGER",
@@ -52,8 +55,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
     STANDARD_K3S_TEST: {
         CLOUD: "gcp", // the cloud provider is still GCP
         DESCRIPTION:
-            "Deploy Gitpod on a K3s cluster, created on a GCP instance," +
-            " with managed DNS and run integrations tests",
+            `Deploy Gitpod on a K3s cluster(version ${k8s_version}), on a GCP instance with ubuntu ${os_version}`,
         PHASES: [
             "STANDARD_K3S_CLUSTER_ON_GCP",
             "CERT_MANAGER",
@@ -65,7 +67,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
     },
     STANDARD_AKS_TEST: {
         CLOUD: "azure",
-        DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
+        DESCRIPTION: `Deploy Gitpod on AKS(version ${k8s_version})`,
         PHASES: [
             "STANDARD_AKS_CLUSTER",
             "CERT_MANAGER",
@@ -79,7 +81,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
     },
     STANDARD_EKS_TEST: {
         CLOUD: "aws",
-        DESCRIPTION: "Create an EKS cluster",
+        DESCRIPTION: `Create an EKS cluster(version ${k8s_version})`,
         PHASES: [
             "STANDARD_EKS_CLUSTER",
             "CERT_MANAGER",
@@ -110,23 +112,23 @@ const cloud: string = config.CLOUD;
 const INFRA_PHASES: { [name: string]: InfraConfig } = {
     STANDARD_GKE_CLUSTER: {
         phase: "create-std-gke-cluster",
-        makeTarget: "gke-standard-cluster",
-        description: "Creating a GKE cluster with 1 nodepool each for workspace and server",
+        makeTarget: `gke-standard-cluster`,
+        description: `Creating a GCP GKE cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
     },
     STANDARD_K3S_CLUSTER_ON_GCP: {
         phase: "create-std-k3s-cluster",
-        makeTarget: "k3s-standard-cluster",
-        description: "Creating a k3s cluster on GCP with 1 node",
+        makeTarget: `k3s-standard-cluster os_version=${os_version}`,
+        description: `Creating a k3s(version: ${k8s_version}) cluster on GCP with 1 node`,
     },
     STANDARD_AKS_CLUSTER: {
         phase: "create-std-aks-cluster",
-        makeTarget: "aks-standard-cluster",
-        description: "Creating an aks cluster(azure)",
+        makeTarget: `aks-standard-cluster`,
+        description: `Creating an Azure AKS cluster(version: ${k8s_version})`,
     },
     STANDARD_EKS_CLUSTER: {
         phase: "create-std-eks-cluster",
-        makeTarget: "eks-standard-cluster",
-        description: "Creating a EKS cluster with 1 nodepool each for workspace and server",
+        makeTarget: `eks-standard-cluster`,
+        description: `Creating a AWS EKS cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
     },
     CERT_MANAGER: {
         phase: "setup-cert-manager",
@@ -140,7 +142,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
     },
     GENERATE_KOTS_CONFIG: {
         phase: "generate-kots-config",
-        makeTarget: `generate-kots-config storage=${randomize()} registry=${randomize()} db=${randomize()}`,
+        makeTarget: `generate-kots-config storage=${randDeps()} registry=${randDeps()} db=${randDeps()}`,
         description: `Generate KOTS Config file`,
     },
     CLUSTER_ISSUER: {
@@ -335,7 +337,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
     werft.log(phase, `Calling ${makeTarget}`);
 
     // exporting cloud env var is important for the make targets
-    const response = exec(`export cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
+    const response = exec(`export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
         slice: phase,
         dontCheckRc: true,
     });
@@ -347,24 +349,61 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
             werft.fail(phase, "Operation failed");
             return response.code;
         }
-        werft.log(phase, `Phase failed`);
+        werft.log(phase, `'${description}' failed`);
     } else {
-        werft.log(phase, `Phase succeeded`);
+        werft.log(phase, `'${description}' succeeded`);
         werft.done(phase);
     }
 
     return response.code;
 }
 
-function randomize(): string {
-    // in the follow-up PR we will add `${platform}-${resource}` as an option here to
-    // test against resource dependencies(storage, db, registry) for each cloud platform
+function randomize(options: string[]): string {
+    return options[Math.floor(Math.random() * options.length)];
+}
+
+function randDeps(): string {
     var depOptions: string[] = ["incluster", "external"]
+
     if(deps && depOptions.includes(deps)) {
         return deps
     }
 
-    return depOptions[Math.floor(Math.random() * depOptions.length)];
+    return randomize(depOptions)
+}
+
+function randK8sVersion(config: string): string {
+    var options: string[] = []
+    switch(config) {
+        case "STANDARD_GKE_TEST": {
+            options = ["1.21", "1.22", "1.23"]
+            break;
+        }
+        case "STANDARD_AKS_TEST": {
+            options = ["1.21", "1.22", "1.23"]
+            break;
+        }
+        case "STANDARD_EKS_TEST": {
+            options = ["1.20", "1.21", "1.22"]
+            break;
+        }
+        case "STANDARD_K3S_TEST": {
+            options = ["v1.22.12+k3s1", "v1.23.9+k3s1", "v1.24.3+k3s1"]
+            break;
+        }
+    }
+    // in the follow-up PR we will add `${platform}-${resource}` as an option here to
+    // test against resource dependencies(storage, db, registry) for each cloud platform
+
+    return randomize(options)
+}
+
+function randOsVersion(): string {
+    // in the follow-up PR we will add `${platform}-${resource}` as an option here to
+    // test against resource dependencies(storage, db, registry) for each cloud platform
+    var options: string[] = ["2204", "2004", "1804"]
+
+    return randomize(options)
 }
 
 function cleanup() {

.werft/jobs/build/self-hosted-upgrade-tests.ts

@@ -46,7 +46,7 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern
 
         werft.phase(upgradeConfig.phase, upgradeConfig.description);
 
-        annotation = `${annotation} -a cluster=${phase}`
+        annotation = `${annotation} -a cluster=${phase} -a updateGitHubStatus=gitpod-io/gitpod`
 
         const testFile: string = ".werft/self-hosted-installer-tests.yaml";
 

install/infra/terraform/aks/kubernetes.tf

@@ -1,8 +1,3 @@
-data "azurerm_kubernetes_service_versions" "k8s" {
-  location        = azurerm_resource_group.gitpod.location
-  include_preview = false
-}
-
 resource "azurerm_role_assignment" "k8s" {
   count = var.dns_enabled ? 1 : 0
 
@@ -20,37 +15,37 @@ resource "azurerm_role_assignment" "k8s_reader" {
 }
 
 resource "azurerm_kubernetes_cluster" "k8s" {
-  name                = format(local.name_format, local.location, "primary")
-  location            = azurerm_resource_group.gitpod.location
-  resource_group_name = azurerm_resource_group.gitpod.name
-  dns_prefix          = "gitpod"
-  tags                = {}
-  api_server_authorized_ip_ranges     = []
-
-  kubernetes_version               = data.azurerm_kubernetes_service_versions.k8s.latest_version
+  name                            = format(local.name_format, local.location, "primary")
+  location                        = azurerm_resource_group.gitpod.location
+  resource_group_name             = azurerm_resource_group.gitpod.name
+  dns_prefix                      = "gitpod"
+  tags                            = {}
+  api_server_authorized_ip_ranges = []
+
+  kubernetes_version               = var.cluster_version
   http_application_routing_enabled = false
 
   default_node_pool {
     name    = local.nodes.0.name
     vm_size = local.machine
 
 
-    node_taints                  = []
-    tags                         = {}
-    zones                        = []
+    node_taints = []
+    tags        = {}
+    zones       = []
 
     enable_auto_scaling  = true
     min_count            = 1
     max_count            = 10
-    orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+    orchestrator_version = var.cluster_version
     node_labels          = local.nodes.0.labels
 
     type           = "VirtualMachineScaleSets"
     vnet_subnet_id = azurerm_subnet.network.id
   }
 
   identity {
-    type = "SystemAssigned"
+    type         = "SystemAssigned"
     identity_ids = []
   }
 
@@ -74,7 +69,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" {
   enable_auto_scaling  = true
   min_count            = 1
   max_count            = 10
-  orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
+  orchestrator_version = var.cluster_version
   node_labels          = local.nodes[count.index + 1].labels
   vnet_subnet_id       = azurerm_subnet.network.id
 }

install/infra/terraform/aks/variables.tf

@@ -1,8 +1,12 @@
 // Common variables
 variable "kubeconfig" {
-    default = "./kubeconfig"
+  default = "./kubeconfig"
+}
 
+variable "cluster_version" {
+  description = "kubernetes version of to create the cluster with"
 }
+
 variable "dns_enabled" {}
 variable "domain_name" {}
 variable "enable_airgapped" {}
@@ -14,6 +18,5 @@ variable "workspace_name" {
 
 // Azure-specific variables
 variable "location" {
-    default = "northeurope"
-
+  default = "northeurope"
 }

install/infra/terraform/eks/kubernetes.tf

@@ -2,30 +2,30 @@ module "vpc" {
   source  = "terraform-aws-modules/vpc/aws"
   version = "3.12.0"
 
-  name               = "vpc-${var.cluster_name}"
-  cidr               = var.vpc_cidr
-  azs                = var.vpc_availability_zones
-  private_subnets    = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
-  public_subnets     = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
-  enable_nat_gateway = true
+  name                 = "vpc-${var.cluster_name}"
+  cidr                 = var.vpc_cidr
+  azs                  = var.vpc_availability_zones
+  private_subnets      = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
+  public_subnets       = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
+  enable_nat_gateway   = true
   enable_dns_hostnames = true
 }
 
 resource "aws_security_group" "nodes" {
-  name = "nodes-sg-${var.cluster_name}"
+  name   = "nodes-sg-${var.cluster_name}"
   vpc_id = module.vpc.vpc_id
 
   ingress {
-    from_port = 0
-    to_port = 0
-    protocol = "-1"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
 
   egress {
-    from_port = 0
-    to_port = 0
-    protocol = "-1"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
     cidr_blocks = ["0.0.0.0/0"]
   }
 }
@@ -34,10 +34,10 @@ module "eks" {
   source  = "terraform-aws-modules/eks/aws"
   version = "18.8.1"
 
-  cluster_name                    = var.cluster_name
-  cluster_version                 = "1.22"
+  cluster_name    = var.cluster_name
+  cluster_version = var.cluster_version
 
-  cluster_endpoint_public_access  = true
+  cluster_endpoint_public_access = true
 
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.public_subnets
@@ -58,7 +58,7 @@ module "eks" {
     iam_role_attach_cni_policy = true
     ami_id                     = var.image_id
     enable_bootstrap_user_data = true
-    vpc_security_group_ids = [aws_security_group.nodes.id]
+    vpc_security_group_ids     = [aws_security_group.nodes.id]
   }
 
   eks_managed_node_groups = {
@@ -144,7 +144,7 @@ module "vpc_cni_irsa" {
 }
 
 resource "null_resource" "kubeconfig" {
-  depends_on = [ module.eks ]
+  depends_on = [module.eks]
   provisioner "local-exec" {
     command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}"
   }

install/infra/terraform/eks/variables.tf

@@ -2,38 +2,45 @@ variable "cluster_name" {
   type        = string
   description = "EKS cluster name."
 }
+
+variable "cluster_version" {
+  type        = string
+  description = "Kubernetes version to create the cluster with"
+  default     = "1.22"
+}
+
 variable "kubeconfig" {
   type        = string
   description = "Path to the kubeconfig file"
   default     = "kubeconfig"
 }
 
 variable "image_id" {
-  type = string
+  type        = string
   description = "AMI Image ID specific to the region"
   // latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/
   default = "ami-0793b4124359a6ad7"
 }
 
 variable "service_machine_type" {
-  type = string
+  type        = string
   description = "Machine type for service workload node pool"
-  default = "m6i.xlarge"
+  default     = "m6i.xlarge"
 }
 
 variable "workspace_machine_type" {
-  type = string
+  type        = string
   description = "Machine type for workspace workload node pool"
-  default = "m6i.2xlarge"
+  default     = "m6i.2xlarge"
 }
 
 variable "region" {
-  type = string
+  type    = string
   default = "eu-west-1"
 }
 
 variable "vpc_availability_zones" {
-  type = list(string)
+  type    = list(string)
   default = ["eu-west-1c", "eu-west-1b"]
 }