[oadp-1.1] OADP-774 must-gather: add timeout to velero logs/describe, var typos, remove duplicate logs, add make run (#821)

* Typo shell variable

* mustgather `make run` target

Signed-off-by: Tiger Kaovilai <[email protected]>

* do not gather restic logs twice

Signed-off-by: Tiger Kaovilai <[email protected]>

* force a timeout of velero command execution

* Use value for timeout from $timeout

Signed-off-by: Tiger Kaovilai <[email protected]>
Co-authored-by: Tiger Kaovilai <[email protected]>

Author: openshift-cherrypick-robot

must-gather/Makefile

@@ -8,6 +8,13 @@ PROMETHEUS_DUMP_PATH ?= $(shell find ./must-gather.local* -name prom_data.tar.gz
 
 build: docker-build docker-push
 
+run: IMAGE_REGISTRY:=ttl.sh
+run: IMAGE_NAME:=oadp/must-gather-$(shell git rev-parse --short HEAD)-$(shell echo $$RANDOM)
+run: IMAGE_TAG:=1h
+run:
+	IMAGE_REGISTRY=$(IMAGE_REGISTRY) IMAGE_NAME=$(IMAGE_NAME) IMAGE_TAG=$(IMAGE_TAG) make build && \
+	oc adm must-gather --image ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
+
 docker-build:
 	docker build -t ${IMAGE_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG} .
 

must-gather/collection-scripts/logs/gather_logs_backup

@@ -15,11 +15,11 @@ skip_tls=$8
 mkdir -p "{object_collection_path}"
 echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup describe ${backup}'"
 if [ "$timeout" = "0s" ]; then
-    oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
+    oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
 else
-    oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
+    oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout ${timeout} /velero describe backup ${backup} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/backup-describe-${backup}.txt" &
 fi
 echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero backup logs ${backup}'"
-oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" &
+oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero backup logs ${backup} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/backup-${backup}.log" &
 
 wait

must-gather/collection-scripts/logs/gather_logs_pvb

@@ -13,12 +13,5 @@ object_collection_path=$6
 node=$(oc get podvolumebackup $pvb --namespace $ns -o jsonpath='{.spec.node}')
 mkdir -p ${object_collection_path}
 oc describe podvolumebackup ${pvb} --namespace ${ns} &> "${object_collection_path}/pvb-describe-${pvb}.txt" &
-for pod in $(oc get pods -o wide --field-selector spec.nodeName=${node} --selector name=restic --no-headers --namespace $ns | awk '{print $1}'); do
-    echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting Pod logs..."
-    oc logs --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/current.log" &
-    echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting previous Pod logs..."
-    oc logs --previous --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/previous.log" & 
-    pwait $max_parallelism
-done
-
+# logs covered by restic pod logs in gather_logs_pods
 wait

must-gather/collection-scripts/logs/gather_logs_pvr

@@ -9,12 +9,8 @@ max_parallelism=$4
 pvr=$5
 object_collection_path=$6
 
-# Gather PVR describe and logs
+# Gather PVR describe
 mkdir -p ${object_collection_path}
 oc describe podvolumerestores.velero.io ${pvr} --namespace ${ns} &> "${object_collection_path}/pvr-describe-${pvr}.txt"
-echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting Pod logs..."
-oc logs --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/current.log" &
-echo "[cluster=${cluster}][ns=${ns}][pod=${pod}] Collecting previous Pod logs..."
-oc logs --previous --all-containers --namespace ${ns} ${pod} --since ${logs_since} &> "${object_collection_path}/previous.log" & 
-
+# logs covered by restic pod logs in gather_logs_pods
 wait

must-gather/collection-scripts/logs/gather_logs_restore

@@ -12,14 +12,14 @@ timeout=$7
 skip_tls=$8
 
 # Gather restore describe and logs
-mkdir -p "{object_collection_path}"
+mkdir -p "${object_collection_path}"
 echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore describe ${restore}'"
 if [ "$timeout" = "0s" ]; then
-    oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
+    oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
 else
-    oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
+    oc -n ${ns} exec --request-timeout=${timeout} $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout ${timeout} /velero describe restore ${restore} --insecure-skip-tls-verify=${skip_tls} --details" &> "${object_collection_path}/restore-describe-${restore}.txt" &
 fi
 echo "[cluster=${cluster}][ns=${ns}] Gathering 'velero restore logs ${restore}'"
-oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "/velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" &
+oc -n ${ns} exec $(oc -n ${ns} get po -l component=velero -o custom-columns=name:.metadata.name --no-headers) -- /bin/bash -c "timeout 30s /velero restore logs ${restore} --insecure-skip-tls-verify=${skip_tls} --timeout=30s" &> "${object_collection_path}/restore-${restore}.log" &
 
 wait