connect the s390x cluster to argocd

infra/gcp/terraform/k8s-infra-prow/buckets.tf

@@ -16,7 +16,7 @@ limitations under the License.
 
 module "gcb_bucket" {
   source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
-  version = "~> 5"
+  version = "~> 11.0"
 
   name       = "k8s-infra-prow-gcb"
   project_id = module.project.project_id

infra/gcp/terraform/k8s-infra-prow/gke.tf

@@ -17,7 +17,7 @@ limitations under the License.
 // WARNING, MAKE SURE YOU DON"T DESTROY THESE CLUSTERS ACCIDENTALLY
 module "prow" {
   source                       = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster"
-  version                      = "~> 30.2"
+  version                      = "~> 37.1"
   project_id                   = module.project.project_id
   name                         = "prow"
   region                       = "us-central1"
@@ -73,7 +73,7 @@ module "prow" {
 
 module "utility_cluster" {
   source                       = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster"
-  version                      = "~> 30.2"
+  version                      = "~> 37.1"
   project_id                   = module.project.project_id
   name                         = "utility"
   region                       = "us-central1"

infra/gcp/terraform/k8s-infra-prow/iam.tf

@@ -173,3 +173,31 @@ resource "google_iam_workload_identity_pool_provider" "ppc64le" {
     jwks_json         = data.http.ppc64le_jwks.response_body
   }
 }
+
+data "http" "s390x_issuer" {
+  url      = "https://d7b2a019-eu-de.lb.appdomain.cloud:6443/.well-known/openid-configuration"
+  insecure = true
+}
+
+data "http" "s390x_jwks" {
+  url      = "https://d7b2a019-eu-de.lb.appdomain.cloud:6443/openid/v1/jwks"
+  insecure = true
+}
+
+resource "google_iam_workload_identity_pool_provider" "s390x" {
+  workload_identity_pool_id          = google_iam_workload_identity_pool.ibm_clusters.workload_identity_pool_id
+  project                            = module.project.project_id
+  workload_identity_pool_provider_id = "s390x"
+
+  attribute_mapping = {
+    "google.subject"                 = "\"ns/\" + assertion['kubernetes.io']['namespace'] + \"/sa/\" + assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.namespace"            = "assertion['kubernetes.io']['namespace']"
+    "attribute.service_account_name" = "assertion['kubernetes.io']['serviceaccount']['name']"
+    "attribute.pod"                  = "assertion['kubernetes.io']['pod']['name']"
+  }
+  oidc {
+    allowed_audiences = ["sts.googleapis.com"]
+    issuer_uri        = jsondecode(data.http.s390x_issuer.response_body)["issuer"]
+    jwks_json         = data.http.s390x_jwks.response_body
+  }
+}

infra/gcp/terraform/k8s-infra-prow/main.tf

@@ -16,7 +16,7 @@ limitations under the License.
 
 module "project" {
   source  = "terraform-google-modules/project-factory/google"
-  version = "~> 14.5"
+  version = "~> 18.0"
 
   name            = "k8s-infra-prow"
   project_id      = "k8s-infra-prow"

infra/gcp/terraform/k8s-infra-prow/provider.tf

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 terraform {
-  required_version = "1.6.5"
+  required_version = "1.10.5"
 
   backend "gcs" {
     bucket = "k8s-infra-tf-prow-clusters"
@@ -25,11 +25,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "~> 5.45.2"
+      version = "~> 6.45.0"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = "~> 5.45.2"
+      version = "~> 6.45.0"
     }
   }
 }

infra/gcp/terraform/k8s-infra-prow/vpc.tf

@@ -16,7 +16,7 @@ limitations under the License.
 
 module "vpc" {
   source  = "terraform-google-modules/network/google"
-  version = "~> 9.3"
+  version = "~> 11.1"
 
   project_id      = module.project.project_id
   network_name    = "prow"

kubernetes/gke-utility/argocd/clusters.yaml

@@ -96,6 +96,32 @@ spec:
     kind: ClusterSecretStore
     name: k8s-infra-prow
 ---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: ibm-s390x
+spec:
+  target:
+    template:
+      engineVersion: v2
+      data:
+        name: ibm-s90x
+        server: https://d7b2a019-eu-de.lb.appdomain.cloud:6443
+        config: "{{ .config }}"
+      metadata:
+        labels:
+          clusterType: prow
+          environment: prod
+          prowNamespace: test-pods
+          cloud: ibm
+  data:
+    - remoteRef:
+        key: ibm-s390x-argo-secret
+      secretKey: config
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: k8s-infra-prow
+---
 apiVersion: v1
 kind: Secret
 metadata:

kubernetes/ibm-s390x/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- mkumatag
+- Prajyot-Parab
+- Rajalakshmi-Girish
+
+reviewers:
+- mkumatag
+- Prajyot-Parab
+- Rajalakshmi-Girish
+
+labels:
+- sig/k8s-infra
+- area/infra
+- area/infra/ibmcloud

kubernetes/ibm-s390x/helm/external-secrets.yaml

@@ -0,0 +1,133 @@
+extraObjects:
+  - apiVersion: external-secrets.io/v1beta1
+    kind: ClusterSecretStore
+    metadata:
+      name: k8s-infra-prow-build
+    spec:
+      provider:
+        gcpsm:
+          projectID: k8s-infra-prow-build
+  # - apiVersion: external-secrets.io/v1beta1
+  #   kind: ClusterSecretStore
+  #   metadata:
+  #     name: secretstore-ibm-k8s
+  #   spec:
+  #     provider:
+  #       ibm:
+  #         serviceUrl: "https://3297fd32-6322-45e2-af3f-00b1a5af3565.us-south.secrets-manager.appdomain.cloud"
+  #         auth:
+  #           secretRef:
+  #             secretApiKeySecretRef:
+  #               name: ibm-sm-apikey
+  #               key: API_KEY
+  #               namespace: external-secrets
+  # - apiVersion: external-secrets.io/v1beta1
+  #   kind: ExternalSecret
+  #   metadata:
+  #     name: ibm-sm-apikey
+  #   spec:
+  #     data:
+  #       - remoteRef:
+  #           key: ibm-sm-apikey
+  #         secretKey: API_KEY
+  #     secretStoreRef:
+  #       kind: ClusterSecretStore
+  #       name: k8s-infra-prow-build
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: google-adc
+    data:
+      adc.json: |
+        {
+          "universe_domain": "googleapis.com",
+          "type": "external_account",
+          "audience": "//iam.googleapis.com/projects/16065310909/locations/global/workloadIdentityPools/ibm-clusters/providers/s390x",
+          "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+          "token_url": "https://sts.googleapis.com/v1/token",
+          "credential_source": {
+            "file": "/var/run/secrets/google-iam-token/serviceaccount/token",
+            "format": {
+              "type": "text"
+            }
+          }
+        }
+  # - apiVersion: external-secrets.io/v1beta1
+  #   kind: ExternalSecret
+  #   metadata:
+  #     name: secret-rotator-api-key
+  #   spec:
+  #     refreshInterval: 60m
+  #     secretStoreRef:
+  #       name: secretstore-ibm-k8s
+  #       kind: ClusterSecretStore
+  #     target:
+  #       name: secret-rotator-api-key
+  #       creationPolicy: Owner
+  #     data:
+  #       - secretKey: api-key
+  #         remoteRef:
+  #           key: iam_credentials/2067d245-e61c-11b2-2c5a-b2be281ea4b8
+  # - apiVersion: batch/v1
+  #   kind: CronJob
+  #   metadata:
+  #     name: ibmcloud-secret-rotator
+  #     labels:
+  #       app: ibmcloud-secret-rotator
+  #   spec:
+  #     schedule: "0 */2 * * *"
+  #     jobTemplate:
+  #       spec:
+  #         template:
+  #           spec:
+  #             containers:
+  #             - name: rotator-container
+  #               image: public.ecr.aws/docker/library/golang:1.24
+  #               imagePullPolicy: Always
+  #               command:
+  #               - /bin/bash
+  #               args:
+  #               - -c
+  #               - |
+  #                 set -o errexit
+  #                 set -o nounset
+  #                 set -o pipefail
+
+  #                 go install sigs.k8s.io/provider-ibmcloud-test-infra/secret-manager@71ef4d8
+  #                 secret-manager rotate --instance-id 3297fd32-6322-45e2-af3f-00b1a5af3565 --labels rotate:true --confirm
+  #               env:
+  #               - name: IBMCLOUD_ENV_FILE
+  #                 value: "/home/.ibmcloud/api-key"
+  #               volumeMounts:
+  #               - name: credentials
+  #                 mountPath: /home/.ibmcloud
+  #             restartPolicy: OnFailure
+  #             volumes:
+  #             - name: credentials
+  #               secret:
+  #                 secretName: secret-rotator-api-key
+
+extraVolumes:
+  - name: google-iam-token
+    projected:
+      defaultMode: 420
+      sources:
+        - serviceAccountToken:
+            audience: sts.googleapis.com
+            expirationSeconds: 86400
+            path: token
+  - name: google-adc
+    configMap:
+      name: google-adc
+
+extraEnv:
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    value: /etc/google/adc.json
+
+extraVolumeMounts:
+  - mountPath: /var/run/secrets/google-iam-token/serviceaccount
+    name: google-iam-token
+    readOnly: true
+  - mountPath: /etc/google
+    name: google-adc
+    readOnly: true

kubernetes/ibm-s390x/helm/kyverno.yaml

@@ -0,0 +1,10 @@
+---
+webhooksCleanup:
+  image:
+    repository: registry.k8s.io/kubectl
+    tag: v1.32.3
+
+policyReportsCleanup:
+  image:
+    repository: registry.k8s.io/kubectl
+    tag: v1.32.3

kubernetes/ibm-s390x/prow/boskos-janitor.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: boskos-ibmcloud-janitor
+  labels:
+    app: boskos-ibmcloud-janitor
+spec:
+  replicas: 2 # 2 distributed janitor instances
+  selector:
+    matchLabels:
+      app: boskos-ibmcloud-janitor
+  template:
+    metadata:
+      labels:
+        app: boskos-ibmcloud-janitor
+    spec:
+      terminationGracePeriodSeconds: 300
+      containers:
+        - name: boskos-ibmcloud-janitor
+          image: gcr.io/k8s-staging-boskos/ibmcloud-janitor-boskos:v20250612-e9e5322
+          args:
+            - --boskos-url=http://boskos.test-pods.svc.cluster.local.
+            - --resource-type=powervs
+            - --ignore-api-key=true
+            - --account-id=efa47ec6fd45473a9e1fd6b7b8363f5c
+          env:
+          - name: IBMCLOUD_ENV_FILE # TODO: explore on how to read key from the file instead of env var
+            value: "/home/.ibmcloud/api-key"
+          volumeMounts:
+            - name: credentials
+              mountPath: /home/.ibmcloud
+      volumes:
+        - name: credentials
+          secret:
+            secretName: boskos-janitor-api-key

kubernetes/ibm-s390x/prow/boskos-reaper.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: boskos-reaper
+  labels:
+    app: boskos-reaper
+spec:
+  selector:
+    matchLabels:
+      app: boskos-reaper
+  replicas: 1 # one canonical source of resources
+  template:
+    metadata:
+      labels:
+        app: boskos-reaper
+    spec:
+      terminationGracePeriodSeconds: 30
+      containers:
+        - name: boskos-reaper
+          image: gcr.io/k8s-staging-boskos/reaper:v20250612-e9e5322
+          args:
+            - --boskos-url=http://boskos.test-pods.svc.cluster.local.
+            - --resource-type=powervs

kubernetes/ibm-s390x/prow/boskos-resources-configmap.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+data:
+  config: |
+    resources:
+    - names:
+      - k8s-s390x-test-vpc
+      state: dirty
+      type: vpc-service
+kind: ConfigMap
+metadata:
+  name: resources