Merge pull request #4535 from pmtk/healthchecks-fail-fast

[release-4.18] USHIFT-5379: Early quit for microshift greenboot scripts

Author: openshift-merge-bot[bot]

packaging/greenboot/functions.sh

@@ -6,6 +6,8 @@
 SCRIPT_NAME=$(basename "$0")
 SCRIPT_PID=$$
 
+MICROSHIFT_GREENBOOT_FAIL_MARKER=/run/microshift-greenboot-healthcheck-failed
+
 OCCONFIG_OPT="--kubeconfig /var/lib/microshift/resources/kubeadmin/kubeconfig"
 OCGET_OPT="--no-headers"
 OCGET_CMD="oc get ${OCCONFIG_OPT}"
@@ -126,6 +128,21 @@ function get_wait_timeout() {
     echo "${wait_timeout}"
 }
 
+# Exit if previous MicroShift healthcheck scripts failed.
+function exit_if_fail_marker_exists() {
+    if [ -f "${MICROSHIFT_GREENBOOT_FAIL_MARKER}" ]; then
+        >&2 echo "'${MICROSHIFT_GREENBOOT_FAIL_MARKER}' file exists, exiting with an error"
+        exit 1
+    fi
+}
+
+# Create fail marker and exit
+function create_fail_marker_and_exit() {
+    >&2 echo "Creating '${MICROSHIFT_GREENBOOT_FAIL_MARKER}' file and exiting with an error."
+    touch "${MICROSHIFT_GREENBOOT_FAIL_MARKER}"
+    exit 1
+}
+
 # Run a command with a second delay until it returns a zero exit status
 #
 # arg1: Time in seconds to wait for a command to succeed

packaging/greenboot/microshift-running-check-gateway-api.sh

@@ -40,6 +40,8 @@ if [ "$(id -u)" -ne 0 ] ; then
     exit 1
 fi
 
+exit_if_fail_marker_exists
+
 echo "STARTED"
 
 # Print the boot variable status
@@ -62,5 +64,5 @@ LOG_POD_EVENTS=true
 echo "Waiting ${WAIT_TIMEOUT_SECS}s for '${CHECK_DEPLOY_NS}' Deployments to be ready"
 if ! wait_for "${WAIT_TIMEOUT_SECS}" namespace_deployment_ready ; then
     echo "Error: Timed out waiting for '${CHECK_DEPLOY_NS}' Deployments to be ready"
-    exit 1
+    create_fail_marker_and_exit
 fi

packaging/greenboot/microshift-running-check-multus.sh

@@ -40,6 +40,8 @@ if [ "$(id -u)" -ne 0 ] ; then
     exit 1
 fi
 
+exit_if_fail_marker_exists
+
 echo "STARTED"
 
 # Print the boot variable status
@@ -62,5 +64,5 @@ LOG_POD_EVENTS=true
 echo "Waiting ${WAIT_TIMEOUT_SECS}s for '${CHECK_DAEMONSET_NS}' DaemonSets to be ready"
 if ! wait_for "${WAIT_TIMEOUT_SECS}" namespace_daemonset_ready ; then
     echo "Error: Timed out waiting for '${CHECK_DAEMONSET_NS}' DaemonSets to be ready"
-    exit 1
+    create_fail_marker_and_exit
 fi

packaging/greenboot/microshift-running-check-olm.sh

@@ -40,6 +40,8 @@ if [ "$(id -u)" -ne 0 ] ; then
     exit 1
 fi
 
+exit_if_fail_marker_exists
+
 echo "STARTED"
 
 # Print the boot variable status
@@ -62,5 +64,5 @@ LOG_POD_EVENTS=true
 echo "Waiting ${WAIT_TIMEOUT_SECS}s for '${CHECK_DEPLOY_NS}' deployments to be ready"
 if ! wait_for "${WAIT_TIMEOUT_SECS}" namespace_deployment_ready ; then
     echo "Error: Timed out waiting for '${CHECK_DEPLOY_NS}' deployments to be ready"
-    exit 1
+    create_fail_marker_and_exit
 fi

packaging/greenboot/microshift-running-check.sh

@@ -128,7 +128,7 @@ LOG_FAILURE_FILES+=("/var/lib/microshift-backups/prerun_failed.log")
 echo "Waiting ${WAIT_TIMEOUT_SECS}s for MicroShift service to be active and not failed"
 if ! wait_for "${WAIT_TIMEOUT_SECS}" microshift_service_active ; then
     echo "Error: Timed out waiting for MicroShift service to be active"
-    exit 1
+    create_fail_marker_and_exit
 fi
 
 # Wait for MicroShift API health endpoints to be OK
@@ -138,7 +138,7 @@ if ! wait_for "${WAIT_TIMEOUT_SECS}" microshift_health_endpoints_ok ; then
     log_failure_cmd "health-livez"  "${OCGET_CMD} --raw=/livez?verbose"
 
     echo "Error: Timed out waiting for MicroShift API health endpoints to be OK"
-    exit 1
+    create_fail_marker_and_exit
 fi
 
 if lvmsShouldBeDeployed; then
@@ -165,7 +165,7 @@ LOG_POD_EVENTS=true
 echo "Waiting ${WAIT_TIMEOUT_SECS}s for any pods to be running"
 if ! wait_for "${WAIT_TIMEOUT_SECS}" any_pods_running ; then
     echo "Error: Timed out waiting for any MicroShift pod to be running"
-    exit 1
+    create_fail_marker_and_exit
 fi
 
 # Wait for MicroShift core pod images to be downloaded
@@ -175,7 +175,7 @@ for i in "${!PODS_NS_LIST[@]}"; do
     echo "Waiting ${WAIT_TIMEOUT_SECS}s for pod image(s) from the '${CHECK_PODS_NS}' namespace to be downloaded"
     if ! wait_for "${WAIT_TIMEOUT_SECS}" namespace_images_downloaded; then
         echo "Error: Timed out waiting for pod image(s) from the '${CHECK_PODS_NS}' namespace to be downloaded"
-        exit 1
+        create_fail_marker_and_exit
     fi
 done
 
@@ -187,7 +187,7 @@ for i in "${!PODS_NS_LIST[@]}"; do
     echo "Waiting ${WAIT_TIMEOUT_SECS}s for ${CHECK_PODS_CT} pod(s) from the '${CHECK_PODS_NS}' namespace to be in 'Ready' state"
     if ! wait_for "${WAIT_TIMEOUT_SECS}" namespace_pods_ready; then
         echo "Error: Timed out waiting for ${CHECK_PODS_CT} pod(s) in the '${CHECK_PODS_NS}' namespace to be in 'Ready' state"
-        exit 1
+        create_fail_marker_and_exit
     fi
 done
 
@@ -216,5 +216,5 @@ done
 
 # Exit with an error code if the pod restart check failed
 if ${check_failed} ; then
-    exit 1
+    create_fail_marker_and_exit
 fi