[preview] fix wait, diff-apply, regression-check slack

Author: vulkoingim

.github/workflows/build.yml

@@ -19,7 +19,6 @@ jobs:
       # github.head_ref is set by a pull_request event - contains the name of the source branch of the PR
       # github.ref_name is set if the event is NOT a pull_request - it contains only the branch name.
       group: ${{ github.head_ref || github.ref_name }}-configuration
-      cancel-in-progress: true
     outputs:
       is_main_branch: ${{ (github.head_ref || github.ref) == 'refs/heads/main' }}
       version: ${{ steps.branches.outputs.sanitized-branch-name }}-gha.${{github.run_number}}
@@ -140,7 +139,6 @@ jobs:
       group: ${{ github.head_ref || github.ref_name }}-build-gitpod
       # For the main branch we always want the build job to run to completion
       # For other branches we only care about the "latest" version, so it is fine to cancel in-progress builds
-      cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
     services:
       mysql:
         image: mysql:5.7

.github/workflows/preview-env-check-regressions.yml

@@ -122,6 +122,11 @@ jobs:
               with:
                   paths: "test/tests/**/TEST.xml"
               if: always()
+            - id: auth
+              uses: google-github-actions/auth@v1
+              with:
+                  token_format: access_token
+                  credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
             - name: Get Secrets from GCP
               id: 'secrets'
               uses: 'google-github-actions/get-secretmanager-secrets@v1'

dev/preview/workflow/lib/k8s-util.sh

@@ -86,3 +86,16 @@ function readWerftSecret {
     get secret "${name}" -o jsonpath="{.data.${key}}" \
   | base64 -d
 }
+
+function diff-apply {
+  local context=$1
+  shift
+  local yaml=$1
+  yaml=$(realpath "${yaml}")
+
+  if kubectl --context "${context}" diff -f "${yaml}" > /dev/null; then
+    echo "Skipping ${yaml}, as it produced no diff"
+  else
+    kubectl --context "${context}" apply --server-side -f "${yaml}"
+  fi
+}

dev/preview/workflow/preview/deploy-gitpod.sh

@@ -134,35 +134,16 @@ EOF
 }
 
 function installRookCeph {
-  kubectl \
-    --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \
-    --context "${PREVIEW_K3S_KUBE_CONTEXT}" \
-    apply -f "$ROOT/.werft/vm/manifests/rook-ceph/crds.yaml" --server-side --force-conflicts
+  diff-apply "${PREVIEW_K3S_KUBE_CONTEXT}" "$ROOT/.werft/vm/manifests/rook-ceph/crds.yaml"
 
   kubectl \
     --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \
     --context "${PREVIEW_K3S_KUBE_CONTEXT}" \
     wait --for condition=established --timeout=120s crd/cephclusters.ceph.rook.io
 
-  kubectl \
-    --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \
-    --context "${PREVIEW_K3S_KUBE_CONTEXT}" \
-    apply -f "$ROOT/.werft/vm/manifests/rook-ceph/common.yaml" -f "$ROOT/.werft/vm/manifests/rook-ceph/operator.yaml"
-
-  kubectl \
-    --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \
-    --context "${PREVIEW_K3S_KUBE_CONTEXT}" \
-    apply -f "$ROOT/.werft/vm/manifests/rook-ceph/cluster-test.yaml"
-
-  kubectl \
-    --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \
-    --context "${PREVIEW_K3S_KUBE_CONTEXT}" \
-    apply -f "$ROOT/.werft/vm/manifests/rook-ceph/storageclass-test.yaml"
-
-  kubectl \
-    --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \
-    --context "${PREVIEW_K3S_KUBE_CONTEXT}" \
-    apply -f "$ROOT/.werft/vm/manifests/rook-ceph/snapshotclass.yaml"
+  for file in common operator cluster-test storageclass-test snapshotclass;do
+      diff-apply "${PREVIEW_K3S_KUBE_CONTEXT}" "$ROOT/.werft/vm/manifests/rook-ceph/$file.yaml"
+  done
 }
 
 # Install Fluent-Bit sending logs to GCP
@@ -375,7 +356,7 @@ then
   yq d -i admin-login-secret.yaml metadata.creationTimestamp
   yq d -i admin-login-secret.yaml metadata.uid
   yq d -i admin-login-secret.yaml metadata.resourceVersion
-  kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" apply -f admin-login-secret.yaml
+  diff-apply "${PREVIEW_K3S_KUBE_CONTEXT}" admin-login-secret.yaml
   rm -f admin-login-secret.yaml
 
   yq w -i "${INSTALLER_CONFIG_PATH}" adminLoginSecret.kind "secret"
@@ -390,7 +371,7 @@ yq w -i stripe-api-keys.secret.yaml metadata.namespace "default"
 yq d -i stripe-api-keys.secret.yaml metadata.creationTimestamp
 yq d -i stripe-api-keys.secret.yaml metadata.uid
 yq d -i stripe-api-keys.secret.yaml metadata.resourceVersion
-kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" apply -f stripe-api-keys.secret.yaml
+diff-apply "${PREVIEW_K3S_KUBE_CONTEXT}" stripe-api-keys.secret.yaml
 rm -f stripe-api-keys.secret.yaml
 
 #
@@ -400,8 +381,9 @@ kubectl --kubeconfig "${DEV_KUBE_PATH}" --context "${DEV_KUBE_CONTEXT}" --namesp
 | yq w - metadata.namespace ${PREVIEW_NAMESPACE} \
 | yq d - metadata.uid \
 | yq d - metadata.resourceVersion \
-| yq d - metadata.creationTimestamp \
-| kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" apply -f -
+| yq d - metadata.creationTimestamp > host-key.yaml
+diff-apply "${PREVIEW_K3S_KUBE_CONTEXT}" host-key.yaml
+rm -f host-key.yaml
 
 yq w -i "${INSTALLER_CONFIG_PATH}" sshGatewayHostKey.kind "secret"
 yq w -i "${INSTALLER_CONFIG_PATH}" sshGatewayHostKey.name "host-key"
@@ -515,7 +497,7 @@ yq w -i spicedb-secret.yaml metadata.namespace "default"
 yq d -i spicedb-secret.yaml metadata.creationTimestamp
 yq d -i spicedb-secret.yaml metadata.uid
 yq d -i spicedb-secret.yaml metadata.resourceVersion
-kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" apply -f spicedb-secret.yaml
+diff-apply "${PREVIEW_K3S_KUBE_CONTEXT}" spicedb-secret.yaml
 rm -f spicedb-secret.yaml
 
 #
@@ -617,13 +599,26 @@ rm -f /tmp/public-api
 log_info "Applying manifests (installing)"
 
 kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" delete -n "${PREVIEW_NAMESPACE}" job migrations || true
-kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" apply -f "${INSTALLER_RENDER_PATH}"
+# export the function so we can use it in xargs
+export -f diff-apply
+mkdir temp-installer || true
+pushd temp-installer
+# this will split the big yaml produced by the installer, so we can diff individual parts of it and run them in parallel
+yq4 -s '.kind + "_" + .metadata.name' "../${INSTALLER_RENDER_PATH}"
+rm .yml || true # this one is a leftover from the split
+# shellcheck disable=SC2038
+find . | xargs -n 1 -I {} -P 5 bash -c "diff-apply ${PREVIEW_K3S_KUBE_CONTEXT} {}"
+log_info "Applied all"
+popd
+rm -rf temp-installer
 rm -f "${INSTALLER_RENDER_PATH}"
 
 # =========================
-# Wait for pods to be ready
+# Wait for objects to be ready
 # =========================
-waitUntilAllPodsAreReady "${PREVIEW_K3S_KUBE_PATH}" "${PREVIEW_K3S_KUBE_CONTEXT}" "$PREVIEW_NAMESPACE"
+for item in deployment.apps/blobserve deployment.apps/content-service deployment.apps/dashboard deployment.apps/ide-metrics deployment.apps/ide-proxy deployment.apps/ide-service deployment.apps/image-builder-mk3 deployment.apps/minio deployment.apps/node-labeler deployment.apps/payment-endpoint deployment.apps/proxy deployment.apps/public-api-server deployment.apps/redis deployment.apps/server deployment.apps/spicedb deployment.apps/usage deployment.apps/ws-manager deployment.apps/ws-manager-bridge deployment.apps/ws-proxy statefulset.apps/messagebus statefulset.apps/mysql statefulset.apps/openvsx-proxy daemonset.apps/agent-smith daemonset.apps/fluent-bit daemonset.apps/registry-facade daemonset.apps/ws-daemon; do
+  kubectl rollout status "${item}"
+done
 
 # =====================
 # Add agent smith token