CI Emoji

.gitlab-ci.yml

@@ -32,16 +32,16 @@ include:
 #  - Smoke tests are executed for all pipelines except scheduled nightly runs.
 #  -
 stages:
-  - smoke tests
-  - network up
-  - cluster up
-  - bigbang up
-  - test
-  - bigbang down
-  - cluster down
-  - network down
-  - package
-  - release
+  - 🔥 smoke tests
+  - 🔌 network up
+  - ⚓ cluster up
+  - 🌌 bigbang up
+  - 🤞 test
+  - 💣 bigbang down
+  - 💣 cluster down
+  - 💣 network down
+  - 📦 package
+  - 🚀 release
 
 variables:
   RELEASE_BUCKET: umbrella-bigbang-releases
@@ -110,20 +110,20 @@ pre vars:
       ./${test_script} && export EXIT_CODE=$? || export EXIT_CODE=$?
       if [[ ${EXIT_CODE} -ne 0 ]]; then
         if [[ ${EXIT_CODE} -ne 123 ]]; then
-          echo -e "\e[31m${test_script} failed, see log output above and cluster debug.\e[0m"
+          echo -e "\e[31m❌ ${test_script} failed, see log output above and cluster debug.\e[0m"
           exit ${EXIT_CODE}
         fi
         # 123 error codes are allowed to continue
-        echo -e "\e[31m${test_script} failed but was allowed to continue, see log output above and cluster debug.\e[0m"
+        echo -e "\e[31m⚠️ ${test_script} failed but was allowed to continue, see log output above and cluster debug.\e[0m"
         EXIT_FLAG=1
       fi
       echo -e "\e[0Ksection_end:`date +%s`:${test_script##*/}\r\e[0K"
     done
     if [[ -n "$EXIT_FLAG" ]]; then
-      echo -e "\e[31mWARNING: One or more BB tests failed but were allowed to continue. See output of scripts above for details.\e[0m"
+      echo -e "\e[31m⚠️ WARNING: One or more BB tests failed but were allowed to continue. See output of scripts above for details.\e[0m"
     fi
 clean install:
-  stage: smoke tests
+  stage: 🔥 smoke tests
   extends:
     - .k3d-ci
   variables:
@@ -162,7 +162,7 @@ clean install:
       - runner_system_failure
 
 upgrade:
-  stage: smoke tests
+  stage: 🔥 smoke tests
   dependencies:
     - pre vars
   extends:
@@ -180,13 +180,13 @@ upgrade:
   variables:
     CLUSTER_NAME: "upgrade-${CI_COMMIT_SHORT_SHA}"
   script:
-    - echo "Install Big Bang from ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
+    - echo "🌌 Install Big Bang from ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
     - echo -e "\e[0Ksection_start:`date +%s`:git_master[collapsed=true]\r\e[0K\e[33;1mGit Fetch Master\e[37m"  
     - git fetch && git checkout ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}
     - echo -e "\e[0Ksection_end:`date +%s`:git_master\r\e[0K"
     - *deploy_bigbang
     - *test_bigbang
-    - echo "Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
+    - echo "🌌 Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
     - echo -e "\e[0Ksection_start:`date +%s`:git_upgrade[collapsed=true]\r\e[0K\e[33;1mGit Upgrade\e[37m"    
     - git reset --hard && git clean -fd
     - git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
@@ -217,7 +217,7 @@ upgrade:
 
 # Abstract for job manually triggering infrastructure builds
 .infra fork:
-  stage: network up
+  stage: 🔌 network up
   rules:
     # Run on scheduled jobs OR when `test-ci` label is assigned
     - if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
@@ -264,7 +264,7 @@ aws/network down:
     - .bigbang-gitlab-runner-tags
     - .infra cleanup
     - .network down
-  stage: network down
+  stage: 💣 network down
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
     action: stop
@@ -283,7 +283,7 @@ aws/network down:
 
 # Create RKE2 cluster on AWS
 aws/rke2/cluster up:
-  stage: cluster up
+  stage: ⚓ cluster up
   extends:
     - .bigbang-gitlab-runner-tags
     - .infra create
@@ -303,7 +303,7 @@ aws/rke2/cluster up:
 
 # Install BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang up:
-  stage: bigbang up
+  stage: 🌌 bigbang up
   extends:
     - .bigbang-gitlab-runner-tags
     - .infra create
@@ -329,7 +329,7 @@ aws/rke2/bigbang up:
 
 # Run tests on BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang test:
-  stage: test
+  stage: 🤞 test
   extends:
     - .bigbang-gitlab-runner-tags
     - .infra create
@@ -371,7 +371,7 @@ aws/rke2/bigbang test:
 
 # Uninstall BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang down:
-  stage: bigbang down
+  stage: 💣 bigbang down
   extends:
     - .bigbang-gitlab-runner-tags
     - .infra cleanup
@@ -398,7 +398,7 @@ aws/rke2/bigbang down:
 
 # Destroy RKE2 cluster on AWS
 aws/rke2/cluster down:
-  stage: cluster down
+  stage: 💣 cluster down
   extends:
     - .bigbang-gitlab-runner-tags
     - .infra cleanup
@@ -423,7 +423,7 @@ aws/rke2/cluster down:
 #
 
 package:
-  stage: package
+  stage: 📦 package
   image: registry.dso.mil/platform-one/big-bang/bigbang/synker:0.0.3
   extends:
     - .bigbang-gitlab-runner-tags
@@ -483,7 +483,7 @@ package:
       - runner_system_failure
 
 release:
-  stage: release
+  stage: 🚀 release
   image: registry.gitlab.com/gitlab-org/release-cli:latest
   extends:
     - .bigbang-gitlab-runner-tags

scripts/hosts.sh

 #!/bin/bash
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
 
 ## Adds all the vs hostnames and LB IP to /etc/hosts
 ## Get the LB Hostname

scripts/install_flux.sh

 #!/usr/bin/env bash
 
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
 
 #
 # global defaults

scripts/package/gits.sh

 #!/bin/bash
-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x
 
 mkdir -p repos/
 

tests/ci/k3d/deploy_k3d.sh

 #!/usr/bin/env bash
 
-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x
 docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
 # if keycloak label or all packages label add deploy k3d without loadbalancer so metallb can be used
 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "keycloak" ||  $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then

tests/ci/k3d/metallb/install_metallb.sh

 #!/usr/bin/env bash
 
-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x
 
 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "keycloak" ||  $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then
   kubectl create -f tests/ci/k3d/metallb/metallb.yaml

tests/deploy/00_deploy_flux.sh

 #!/usr/bin/env bash
 
-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x
 
 # install flux with the dedicated helper script
 ./scripts/install_flux.sh \

tests/deploy/01_deploy_bigbang.sh

 #!/usr/bin/env bash
 
-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x
 
 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then
-  echo "all-packages label enabled, or on default branch or tag, enabling all addons"
+  echo "🌌 all-packages label enabled, or on default branch or tag, enabling all addons"
   yq e ".addons.*.enabled = "true"" $CI_VALUES_FILE > tmpfile && mv tmpfile $CI_VALUES_FILE
 else
   IFS=","
@@ -30,7 +31,7 @@ if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] && [[ "$CI_COMMIT_BRANCH" == "maste
 fi
 
 # deploy BigBang using dev sized scaling
-echo "Installing BigBang with the following configurations:"
+echo "🚀 Installing BigBang with the following configurations:"
 cat $CI_VALUES_FILE
 
 helm upgrade -i bigbang chart -n bigbang --create-namespace \
@@ -44,16 +45,16 @@ helm upgrade -i bigbang chart -n bigbang --create-namespace \
 
 # apply secrets kustomization pointing to current branch or master if an upgrade job
 if [[ $(git branch --show-current) == "${CI_DEFAULT_BRANCH}" ]]; then
-  echo "Deploying secrets from the ${CI_DEFAULT_BRANCH} branch"
+  echo "🚀 Deploying secrets from the ${CI_DEFAULT_BRANCH} branch"
   kubectl apply -f tests/ci/shared-secrets.yaml
 elif [[ $(git branch --show-current) == "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]]; then
-  echo "Deploying secrets from the ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} branch"
+  echo "🚀 Deploying secrets from the ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} branch"
   cat tests/ci/shared-secrets.yaml | sed 's|master|'"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"'|g' | kubectl apply -f -
 elif [ -z "$CI_COMMIT_TAG" ]; then
-  echo "Deploying secrets from the ${CI_COMMIT_REF_NAME} branch"
+  echo "🚀 Deploying secrets from the ${CI_COMMIT_REF_NAME} branch"
   cat tests/ci/shared-secrets.yaml | sed 's|master|'"$CI_COMMIT_REF_NAME"'|g' | kubectl apply -f -
 else
-  echo "Deploying secrets from the ${CI_COMMIT_REF_NAME} tag"
+  echo "🚀 Deploying secrets from the ${CI_COMMIT_REF_NAME} tag"
   # NOTE: $CI_COMMIT_REF_NAME = $CI_COMMIT_TAG when running on a tagged build
   cat tests/ci/shared-secrets.yaml | sed 's|branch: master|tag: '"$CI_COMMIT_REF_NAME"'|g' | kubectl apply -f -
 fi

tests/deploy/02_wait_for_helmreleases.sh

 #!/usr/bin/env bash
 
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
 
 ## Array of core HRs
 CORE_HELMRELEASES=("gatekeeper" "istio-operator" "istio" "monitoring" "eck-operator" "ek" "fluent-bit" "twistlock" "cluster-auditor" "jaeger" "kiali")
@@ -37,12 +37,12 @@ function array_contains() {
 ## Function to check/wait on HR existence
 function check_if_hr_exist() {
     timeElapsed=0
-    echo "Waiting for $1 HR to exist"
+    echo "⏳ Waiting for $1 HR to exist"
     until kubectl get hr -n bigbang $1 &> /dev/null; do
       sleep 5
       timeElapsed=$(($timeElapsed+5))
       if [[ $timeElapsed -ge 60 ]]; then
-         echo "Timed out while waiting for $1 HR to exist"
+         echo "❌ Timed out while waiting for $1 HR to exist"
          exit 1
       fi
     done
@@ -61,7 +61,7 @@ function wait_all_hr() {
               break
             else
               artifactfailedcounter=$(($artifactfailedcounter+1))
-              echo "Helm Artifact Failed, waiting 5 seconds."
+              echo "⏳ Helm Artifact Failed, waiting 5 seconds."
               sleep 5
               hrstatus=$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].reason}')
             fi
@@ -70,8 +70,8 @@ function wait_all_hr() {
         if [[ "$hrstatus" =~ Failed ]]; then
             state=$(kubectl get hr -A -o go-template='{{range $items,$contents := .items}}{{printf "HR %s" $contents.metadata.name}}{{printf " status is %s\n" (index $contents.status.conditions 0).reason}}{{end}}')
             failed=$(echo "${state}" | grep "Failed")
-            echo "Found failed Helm Release(s). Exiting now."
-            echo "${failed}"
+            echo "❌ Found failed Helm Release(s). Exiting now."
+            echo "❌ ${failed}"
             failed_hrs=$(echo "{$failed}" | awk  '{print $2}')
             for hr in $failed_hrs; do
                 kubectl describe hr -n bigbang $hr
@@ -80,14 +80,14 @@ function wait_all_hr() {
         fi
         if [[ "$hrready" != *Unknown* ]]; then
             if [[ "$hrready" != *False* ]]; then
-                echo "All HR's deployed"
+                echo "✅ All HR's deployed"
                 break
             fi
         fi
         sleep 5
         timeElapsed=$(($timeElapsed+5))
         if [[ $timeElapsed -ge 1800 ]]; then
-            echo "Timed out while waiting for hr's to be ready."
+            echo "❌ Timed out while waiting for hr's to be ready."
             exit 1
         fi
     done
@@ -107,7 +107,7 @@ function wait_sts() {
       sleep 5
       timeElapsed=$(($timeElapsed+5))
       if [[ $timeElapsed -ge 600 ]]; then
-         echo "Timed out while waiting for stateful sets to be ready."
+         echo "❌ Timed out while waiting for stateful sets to be ready."
          exit 1
       fi
    done
@@ -127,7 +127,7 @@ function wait_daemonset(){
       sleep 5
       timeElapsed=$(($timeElapsed+5))
       if [[ $timeElapsed -ge 600 ]]; then
-         echo "Timed out while waiting for daemon sets to be ready."
+         echo "❌ Timed out while waiting for daemon sets to be ready."
          exit 1
       fi
    done
@@ -162,7 +162,7 @@ function wait_crd(){
 HELMRELEASES=(${CORE_HELMRELEASES[@]})
 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then
     HELMRELEASES+=(${ADD_ON_HELMRELEASES[@]})
-    echo "All helmreleases enabled: all-packages label enabled, or on default branch or tag."
+    echo "🌌 All helmreleases enabled: all-packages label enabled, or on default branch or tag."
 elif [[ ! -z "$CI_MERGE_REQUEST_LABELS" ]]; then
     IFS=","
     for package in $CI_MERGE_REQUEST_LABELS; do
@@ -178,10 +178,10 @@ elif [[ ! -z "$CI_MERGE_REQUEST_LABELS" ]]; then
             fi
         fi
     done
-    echo "Found enabled helmreleases: ${HELMRELEASES[@]}"
+    echo "✅ Found enabled helmreleases: ${HELMRELEASES[@]}"
 fi
 
-echo "Waiting on GitRepositories"
+echo "⏳ Waiting on GitRepositories"
 kubectl wait --for=condition=Ready --timeout 180s gitrepositories -n bigbang --all
 
 for package in "${HELMRELEASES[@]}";
@@ -189,18 +189,18 @@ do
     check_if_hr_exist "$package"
 done
 
-echo "Waiting on helm releases..."
+echo "⏳ Waiting on helm releases..."
 wait_all_hr
-echo "Waiting for custom resources..."
+echo "⏳ Waiting for custom resources..."
 wait_crd
 
 kubectl get helmreleases,kustomizations,gitrepositories -A
 
-echo "Waiting on Secrets Kustomization"
+echo "⏳ Waiting on Secrets Kustomization"
 kubectl wait --for=condition=Ready --timeout 300s kustomizations.kustomize.toolkit.fluxcd.io -n bigbang secrets
 
 # In case some helm releases are marked as ready before all objects are live...
-echo "Waiting on all jobs, deployments, statefulsets, and daemonsets"
+echo "⏳ Waiting on all jobs, deployments, statefulsets, and daemonsets"
 kubectl wait --for=condition=available --timeout 600s -A deployment --all > /dev/null
 wait_sts
 wait_daemonset

tests/tests/01_virtualservices.sh

@@ -2,7 +2,7 @@
 
 # exit on error
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
 
 # Populate /etc/hosts
 ip=$(kubectl -n istio-system get service public-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

tests/tests/02_non_ironbank.sh

@@ -2,7 +2,7 @@
 
 # exit on error
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
 
 # Quick check for non iron bank images
 echo "Showing images not from ironbank:"