From e4509de16b8722beef687b618d9501665e1c72e0 Mon Sep 17 00:00:00 2001
From: "evan.rush" <rush_evan@bah.com>
Date: Mon, 27 Sep 2021 21:42:45 +0000
Subject: [PATCH] CI HR ArtifactFailed retry
---
tests/deploy/02_wait_for_helmreleases.sh | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/tests/deploy/02_wait_for_helmreleases.sh b/tests/deploy/02_wait_for_helmreleases.sh
index 9e6fe34d49..1c16c0a0df 100755
--- a/tests/deploy/02_wait_for_helmreleases.sh
+++ b/tests/deploy/02_wait_for_helmreleases.sh
@@ -52,7 +52,22 @@ function check_if_hr_exist() {
function wait_all_hr() {
timeElapsed=0
while true; do
- if [[ "$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].reason}')" =~ Failed ]]; then
+ hrstatus=$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].reason}')
+ hrready=$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].status}')
+ # HR ArtifactFailed, retry
+ artifactfailedcounter=0
+ while [[ $artifactfailedcounter -lt 3 ]]; do
+ if [[ ! "$hrstatus" =~ ArtifactFailed ]]; then
+ break
+ else
+ artifactfailedcounter=$(($artifactfailedcounter+1))
+ echo "Helm Artifact Failed, waiting 5 seconds."
+ sleep 5
+ hrstatus=$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].reason}')
+ fi
+ done
+ # HR *Failed, exit
+ if [[ "$hrstatus" =~ Failed ]]; then
state=$(kubectl get hr -A -o go-template='{{range $items,$contents := .items}}{{printf "HR %s" $contents.metadata.name}}{{printf " status is %s\n" (index $contents.status.conditions 0).reason}}{{end}}')
failed=$(echo "${state}" | grep "Failed")
echo "Found failed Helm Release(s). Exiting now."
@@ -63,8 +78,8 @@ function wait_all_hr() {
done
exit 1
fi
- if [[ "$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].status}')" != *Unknown* ]]; then
- if [[ "$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].status}')" != *False* ]]; then
+ if [[ "$hrready" != *Unknown* ]]; then
+ if [[ "$hrready" != *False* ]]; then
echo "All HR's deployed"
break
fi
--
GitLab