02_wait_for_helmreleases.sh

#!/usr/bin/env bash

set -e

## This is an array to instantiate the order of wait conditions
ORDERED_HELMRELEASES="gatekeeper istio-operator istio monitoring eck-operator ek fluent-bit twistlock cluster-auditor authservice argocd gitlab haproxy-sso gitlab-runner minio-operator minio anchore sonarqube mattermost-operator mattermost keycloak nexus-repository-manager"

## This is the actual deployed helmrelease objects in the cluster
DEPLOYED_HELMRELEASES=$(kubectl get hr --no-headers -n bigbang | awk '{ print $1}')

printf "Identified the following deployed helmreleases:\n%s" "${DEPLOYED_HELMRELEASES}"

## Function to test an array contains an element
## Args:
## $1: array to search
## $2: element to search for
function array_contains() {
    local array="$1[@]"
    local seeking=$2
    local in=1
    for element in ${!array}; do
        if [[ $element == "$seeking" ]]; then
            in=0
            break
        fi
    done
    return $in
}

## Function to wait on helmrelease
## Args:
## $1: package name
function wait_on() {
  echo "Waiting on package $1"
  kubectl wait --for=condition=Ready --timeout 600s helmrelease -n bigbang $1;
}

## Function to wait on all statefulsets
function wait_sts() {
   timeElapsed=0
   while true; do
      sts=$(kubectl get sts -A -o jsonpath='{.items[*].status.replicas}' | xargs)
      totalSum=$(echo $sts | awk '{for (i=1; i<=NF; i++) c+=$i} {print c}')
      readySts=$(kubectl get sts -A -o jsonpath='{.items[*].status.readyReplicas}' | xargs)
      readySum=$(echo $readySts | awk '{for (i=1; i<=NF; i++) c+=$i} {print c}')
      if [[ $totalSum -eq $readySum ]]; then
         break
      fi
      sleep 5
      timeElapsed=$(($timeElapsed+5))
      if [[ $timeElapsed -ge 600 ]]; then
         echo "Timed out while waiting for stateful sets to be ready."
         exit 1
      fi
   done
}

## Function to wait on all daemonsets
function wait_daemonset(){
   timeElapsed=0
   while true; do
      dmnset=$(kubectl get daemonset -A -o jsonpath='{.items[*].status.desiredNumberScheduled}' | xargs)
      totalSum=$(echo $dmnset | awk '{for (i=1; i<=NF; i++) c+=$i} {print c}')
      readyDmnset=$(kubectl get daemonset -A -o jsonpath='{.items[*].status.numberReady}' | xargs)
      readySum=$(echo $readyDmnset | awk '{for (i=1; i<=NF; i++) c+=$i} {print c}')
      if [[ $totalSum -eq $readySum ]]; then
         break
      fi
      sleep 5
      timeElapsed=$(($timeElapsed+5))
      if [[ $timeElapsed -ge 600 ]]; then
         echo "Timed out while waiting for daemon sets to be ready."
         exit 1
      fi
   done
}


for package in $ORDERED_HELMRELEASES;
do
  if array_contains DEPLOYED_HELMRELEASES "$package";
  then wait_on "$package"
  else echo "Expected package: $package, but not found in release. Update the array in this script if this package is no longer needed"
  fi
done

kubectl get helmreleases,kustomizations,gitrepositories -A

for package in $DEPLOYED_HELMRELEASES;
do
  if array_contains ORDERED_HELMRELEASES "$package";
  then echo ""
  else 
    echo "Found package: $package, but not found in this script array. Update the array in this script if this package is always needed"
    wait_on "$package"
  fi
done

# Double check everything got waited on...
kubectl wait --for=condition=Ready --timeout 600s helmrelease -n bigbang --all

echo "Waiting on Secrets Kustomization"
kubectl wait --for=condition=Ready --timeout 300s kustomizations.kustomize.toolkit.fluxcd.io -n bigbang secrets

# In case some helm releases are marked as ready before all objects are live...
echo "Waiting on all jobs, deployments, statefulsets, and daemonsets"
kubectl wait --for=condition=available --timeout 600s -A deployment --all > /dev/null
wait_sts
wait_daemonset
if kubectl get job -A -o jsonpath='{.items[].metadata.name}' &> /dev/null; then
  kubectl wait --for=condition=complete --timeout 300s -A job --all > /dev/null
fi