Skip to content

Commit 42eae5f

Browse files
nandajavarmamustard-mh
nandajavarma
authored and
mustard-mh
committed
[installer-tests] Add tests for k8s versions and ubuntu version
1 parent fbf2e6f commit 42eae5f

File tree

12 files changed

+224
-126
lines changed

12 files changed

+224
-126
lines changed

‎.werft/installer-tests.ts

Lines changed: 60 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,15 @@ interface TestConfig {
3333
CLOUD: string;
3434
}
3535

36+
const k8s_version: string = randK8sVersion(testConfig)
37+
const os_version: string = randOsVersion() // applicable only for k3s
38+
3639
// Each of the TEST_CONFIGURATIONS define an integration test end-to-end
3740
// It should be a combination of multiple INFRA_PHASES, order of PHASES slice is important
3841
const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
3942
STANDARD_GKE_TEST: {
4043
CLOUD: "gcp",
41-
DESCRIPTION: "Deploy Gitpod on GKE, with managed DNS, and run integration tests",
44+
DESCRIPTION: `Deploy Gitpod on GKE(version ${k8s_version})`,
4245
PHASES: [
4346
"STANDARD_GKE_CLUSTER",
4447
"CERT_MANAGER",
@@ -52,8 +55,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
5255
STANDARD_K3S_TEST: {
5356
CLOUD: "gcp", // the cloud provider is still GCP
5457
DESCRIPTION:
55-
"Deploy Gitpod on a K3s cluster, created on a GCP instance," +
56-
" with managed DNS and run integrations tests",
58+
`Deploy Gitpod on a K3s cluster(version ${k8s_version}), on a GCP instance with ubuntu ${os_version}`,
5759
PHASES: [
5860
"STANDARD_K3S_CLUSTER_ON_GCP",
5961
"CERT_MANAGER",
@@ -65,7 +67,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
6567
},
6668
STANDARD_AKS_TEST: {
6769
CLOUD: "azure",
68-
DESCRIPTION: "Deploy Gitpod on AKS, with managed DNS, and run integration tests",
70+
DESCRIPTION: `Deploy Gitpod on AKS(version ${k8s_version})`,
6971
PHASES: [
7072
"STANDARD_AKS_CLUSTER",
7173
"CERT_MANAGER",
@@ -79,7 +81,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
7981
},
8082
STANDARD_EKS_TEST: {
8183
CLOUD: "aws",
82-
DESCRIPTION: "Create an EKS cluster",
84+
DESCRIPTION: `Create an EKS cluster(version ${k8s_version})`,
8385
PHASES: [
8486
"STANDARD_EKS_CLUSTER",
8587
"CERT_MANAGER",
@@ -110,23 +112,23 @@ const cloud: string = config.CLOUD;
110112
const INFRA_PHASES: { [name: string]: InfraConfig } = {
111113
STANDARD_GKE_CLUSTER: {
112114
phase: "create-std-gke-cluster",
113-
makeTarget: "gke-standard-cluster",
114-
description: "Creating a GKE cluster with 1 nodepool each for workspace and server",
115+
makeTarget: `gke-standard-cluster`,
116+
description: `Creating a GCP GKE cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
115117
},
116118
STANDARD_K3S_CLUSTER_ON_GCP: {
117119
phase: "create-std-k3s-cluster",
118-
makeTarget: "k3s-standard-cluster",
119-
description: "Creating a k3s cluster on GCP with 1 node",
120+
makeTarget: `k3s-standard-cluster os_version=${os_version}`,
121+
description: `Creating a k3s(version: ${k8s_version}) cluster on GCP with 1 node`,
120122
},
121123
STANDARD_AKS_CLUSTER: {
122124
phase: "create-std-aks-cluster",
123-
makeTarget: "aks-standard-cluster",
124-
description: "Creating an aks cluster(azure)",
125+
makeTarget: `aks-standard-cluster`,
126+
description: `Creating an Azure AKS cluster(version: ${k8s_version})`,
125127
},
126128
STANDARD_EKS_CLUSTER: {
127129
phase: "create-std-eks-cluster",
128-
makeTarget: "eks-standard-cluster",
129-
description: "Creating a EKS cluster with 1 nodepool each for workspace and server",
130+
makeTarget: `eks-standard-cluster`,
131+
description: `Creating a AWS EKS cluster(version: ${k8s_version}) with 1 nodepool each for workspace and server`,
130132
},
131133
CERT_MANAGER: {
132134
phase: "setup-cert-manager",
@@ -140,7 +142,7 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
140142
},
141143
GENERATE_KOTS_CONFIG: {
142144
phase: "generate-kots-config",
143-
makeTarget: `generate-kots-config storage=${randomize()} registry=${randomize()} db=${randomize()}`,
145+
makeTarget: `generate-kots-config storage=${randDeps()} registry=${randDeps()} db=${randDeps()}`,
144146
description: `Generate KOTS Config file`,
145147
},
146148
CLUSTER_ISSUER: {
@@ -335,7 +337,7 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
335337
werft.log(phase, `Calling ${makeTarget}`);
336338

337339
// exporting cloud env var is important for the make targets
338-
const response = exec(`export cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
340+
const response = exec(`export TF_VAR_cluster_version=${k8s_version} cloud=${cloud} && make -C ${makefilePath} ${makeTarget}`, {
339341
slice: phase,
340342
dontCheckRc: true,
341343
});
@@ -347,24 +349,61 @@ function callMakeTargets(phase: string, description: string, makeTarget: string,
347349
werft.fail(phase, "Operation failed");
348350
return response.code;
349351
}
350-
werft.log(phase, `Phase failed`);
352+
werft.log(phase, `'${description}' failed`);
351353
} else {
352-
werft.log(phase, `Phase succeeded`);
354+
werft.log(phase, `'${description}' succeeded`);
353355
werft.done(phase);
354356
}
355357

356358
return response.code;
357359
}
358360

359-
function randomize(): string {
360-
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
361-
// test against resource dependencies(storage, db, registry) for each cloud platform
361+
function randomize(options: string[]): string {
362+
return options[Math.floor(Math.random() * options.length)];
363+
}
364+
365+
function randDeps(): string {
362366
var depOptions: string[] = ["incluster", "external"]
367+
363368
if(deps && depOptions.includes(deps)) {
364369
return deps
365370
}
366371

367-
return depOptions[Math.floor(Math.random() * depOptions.length)];
372+
return randomize(depOptions)
373+
}
374+
375+
function randK8sVersion(config: string): string {
376+
var options: string[] = []
377+
switch(config) {
378+
case "STANDARD_GKE_TEST": {
379+
options = ["1.21", "1.22", "1.23"]
380+
break;
381+
}
382+
case "STANDARD_AKS_TEST": {
383+
options = ["1.21", "1.22", "1.23"]
384+
break;
385+
}
386+
case "STANDARD_EKS_TEST": {
387+
options = ["1.20", "1.21", "1.22"]
388+
break;
389+
}
390+
case "STANDARD_K3S_TEST": {
391+
options = ["v1.22.12+k3s1", "v1.23.9+k3s1", "v1.24.3+k3s1"]
392+
break;
393+
}
394+
}
395+
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
396+
// test against resource dependencies(storage, db, registry) for each cloud platform
397+
398+
return randomize(options)
399+
}
400+
401+
function randOsVersion(): string {
402+
// in the follow-up PR we will add `${platform}-${resource}` as an option here to
403+
// test against resource dependencies(storage, db, registry) for each cloud platform
404+
var options: string[] = ["2204", "2004", "1804"]
405+
406+
return randomize(options)
368407
}
369408

370409
function cleanup() {

‎.werft/jobs/build/self-hosted-upgrade-tests.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ export async function triggerUpgradeTests(werft: Werft, config: JobConfig, usern
4646

4747
werft.phase(upgradeConfig.phase, upgradeConfig.description);
4848

49-
annotation = `${annotation} -a cluster=${phase}`
49+
annotation = `${annotation} -a cluster=${phase} -a updateGitHubStatus=gitpod-io/gitpod`
5050

5151
const testFile: string = ".werft/self-hosted-installer-tests.yaml";
5252

‎install/infra/terraform/aks/kubernetes.tf

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
1-
data "azurerm_kubernetes_service_versions" "k8s" {
2-
location = azurerm_resource_group.gitpod.location
3-
include_preview = false
4-
}
5-
61
resource "azurerm_role_assignment" "k8s" {
72
count = var.dns_enabled ? 1 : 0
83

@@ -20,37 +15,37 @@ resource "azurerm_role_assignment" "k8s_reader" {
2015
}
2116

2217
resource "azurerm_kubernetes_cluster" "k8s" {
23-
name = format(local.name_format, local.location, "primary")
24-
location = azurerm_resource_group.gitpod.location
25-
resource_group_name = azurerm_resource_group.gitpod.name
26-
dns_prefix = "gitpod"
27-
tags = {}
28-
api_server_authorized_ip_ranges = []
29-
30-
kubernetes_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
18+
name = format(local.name_format, local.location, "primary")
19+
location = azurerm_resource_group.gitpod.location
20+
resource_group_name = azurerm_resource_group.gitpod.name
21+
dns_prefix = "gitpod"
22+
tags = {}
23+
api_server_authorized_ip_ranges = []
24+
25+
kubernetes_version = var.cluster_version
3126
http_application_routing_enabled = false
3227

3328
default_node_pool {
3429
name = local.nodes.0.name
3530
vm_size = local.machine
3631

3732

38-
node_taints = []
39-
tags = {}
40-
zones = []
33+
node_taints = []
34+
tags = {}
35+
zones = []
4136

4237
enable_auto_scaling = true
4338
min_count = 1
4439
max_count = 10
45-
orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
40+
orchestrator_version = var.cluster_version
4641
node_labels = local.nodes.0.labels
4742

4843
type = "VirtualMachineScaleSets"
4944
vnet_subnet_id = azurerm_subnet.network.id
5045
}
5146

5247
identity {
53-
type = "SystemAssigned"
48+
type = "SystemAssigned"
5449
identity_ids = []
5550
}
5651

@@ -74,7 +69,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "pools" {
7469
enable_auto_scaling = true
7570
min_count = 1
7671
max_count = 10
77-
orchestrator_version = data.azurerm_kubernetes_service_versions.k8s.latest_version
72+
orchestrator_version = var.cluster_version
7873
node_labels = local.nodes[count.index + 1].labels
7974
vnet_subnet_id = azurerm_subnet.network.id
8075
}

‎install/infra/terraform/aks/variables.tf

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
// Common variables
22
variable "kubeconfig" {
3-
default = "./kubeconfig"
3+
default = "./kubeconfig"
4+
}
45

6+
variable "cluster_version" {
7+
description = "kubernetes version of to create the cluster with"
58
}
9+
610
variable "dns_enabled" {}
711
variable "domain_name" {}
812
variable "enable_airgapped" {}
@@ -14,6 +18,5 @@ variable "workspace_name" {
1418

1519
// Azure-specific variables
1620
variable "location" {
17-
default = "northeurope"
18-
21+
default = "northeurope"
1922
}

‎install/infra/terraform/eks/kubernetes.tf

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,30 @@ module "vpc" {
22
source = "terraform-aws-modules/vpc/aws"
33
version = "3.12.0"
44

5-
name = "vpc-${var.cluster_name}"
6-
cidr = var.vpc_cidr
7-
azs = var.vpc_availability_zones
8-
private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
9-
public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
10-
enable_nat_gateway = true
5+
name = "vpc-${var.cluster_name}"
6+
cidr = var.vpc_cidr
7+
azs = var.vpc_availability_zones
8+
private_subnets = [var.private_primary_subnet_cidr, var.private_secondary_subnet_cidr]
9+
public_subnets = [var.public_primary_subnet_cidr, var.public_secondary_subnet_cidr, var.public_db_subnet_cidr_1, var.public_db_subnet_cidr_2]
10+
enable_nat_gateway = true
1111
enable_dns_hostnames = true
1212
}
1313

1414
resource "aws_security_group" "nodes" {
15-
name = "nodes-sg-${var.cluster_name}"
15+
name = "nodes-sg-${var.cluster_name}"
1616
vpc_id = module.vpc.vpc_id
1717

1818
ingress {
19-
from_port = 0
20-
to_port = 0
21-
protocol = "-1"
19+
from_port = 0
20+
to_port = 0
21+
protocol = "-1"
2222
cidr_blocks = ["0.0.0.0/0"]
2323
}
2424

2525
egress {
26-
from_port = 0
27-
to_port = 0
28-
protocol = "-1"
26+
from_port = 0
27+
to_port = 0
28+
protocol = "-1"
2929
cidr_blocks = ["0.0.0.0/0"]
3030
}
3131
}
@@ -34,10 +34,10 @@ module "eks" {
3434
source = "terraform-aws-modules/eks/aws"
3535
version = "18.8.1"
3636

37-
cluster_name = var.cluster_name
38-
cluster_version = "1.22"
37+
cluster_name = var.cluster_name
38+
cluster_version = var.cluster_version
3939

40-
cluster_endpoint_public_access = true
40+
cluster_endpoint_public_access = true
4141

4242
vpc_id = module.vpc.vpc_id
4343
subnet_ids = module.vpc.public_subnets
@@ -58,7 +58,7 @@ module "eks" {
5858
iam_role_attach_cni_policy = true
5959
ami_id = var.image_id
6060
enable_bootstrap_user_data = true
61-
vpc_security_group_ids = [aws_security_group.nodes.id]
61+
vpc_security_group_ids = [aws_security_group.nodes.id]
6262
}
6363

6464
eks_managed_node_groups = {
@@ -144,7 +144,7 @@ module "vpc_cni_irsa" {
144144
}
145145

146146
resource "null_resource" "kubeconfig" {
147-
depends_on = [ module.eks ]
147+
depends_on = [module.eks]
148148
provisioner "local-exec" {
149149
command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}"
150150
}

‎install/infra/terraform/eks/variables.tf

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,38 +2,45 @@ variable "cluster_name" {
22
type = string
33
description = "EKS cluster name."
44
}
5+
6+
variable "cluster_version" {
7+
type = string
8+
description = "Kubernetes version to create the cluster with"
9+
default = "1.22"
10+
}
11+
512
variable "kubeconfig" {
613
type = string
714
description = "Path to the kubeconfig file"
815
default = "kubeconfig"
916
}
1017

1118
variable "image_id" {
12-
type = string
19+
type = string
1320
description = "AMI Image ID specific to the region"
1421
// latest ubuntu image for 1.22 k8s for eu-west-1 region, refer https://cloud-images.ubuntu.com/docs/aws/eks/
1522
default = "ami-0793b4124359a6ad7"
1623
}
1724

1825
variable "service_machine_type" {
19-
type = string
26+
type = string
2027
description = "Machine type for service workload node pool"
21-
default = "m6i.xlarge"
28+
default = "m6i.xlarge"
2229
}
2330

2431
variable "workspace_machine_type" {
25-
type = string
32+
type = string
2633
description = "Machine type for workspace workload node pool"
27-
default = "m6i.2xlarge"
34+
default = "m6i.2xlarge"
2835
}
2936

3037
variable "region" {
31-
type = string
38+
type = string
3239
default = "eu-west-1"
3340
}
3441

3542
variable "vpc_availability_zones" {
36-
type = list(string)
43+
type = list(string)
3744
default = ["eu-west-1c", "eu-west-1b"]
3845
}
3946

0 commit comments

Comments
 (0)