Compare revisions

Tunde Oladipupo · Ernest Chuang · Micah Nagel · Micah Nagel · Ryan Garcia · Ryan Garcia
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,16 +32,16 @@ include:
 #  - Smoke tests are executed for all pipelines except scheduled nightly runs.
 #  -
 stages:
-  - smoke tests
-  - network up
-  - cluster up
-  - bigbang up
-  - test
-  - bigbang down
-  - cluster down
-  - network down
-  - package
-  - release
+  - 🔥 smoke tests
+  - 🔌 network up
+  - ⚓ cluster up
+  - 🌌 bigbang up
+  - 🤞 test
+  - 💣 bigbang down
+  - 💣 cluster down
+  - 💣 network down
+  - 📦 package
+  - 🚀 release

 variables:
  RELEASE_BUCKET: umbrella-bigbang-releases
@@ -110,20 +110,20 @@ pre vars:
      ./${test_script} && export EXIT_CODE=$? || export EXIT_CODE=$?
      if [[ ${EXIT_CODE} -ne 0 ]]; then
        if [[ ${EXIT_CODE} -ne 123 ]]; then
-          echo -e "\e[31m${test_script} failed, see log output above and cluster debug.\e[0m"
+          echo -e "\e[31m❌ ${test_script} failed, see log output above and cluster debug.\e[0m"
          exit ${EXIT_CODE}
        fi
        # 123 error codes are allowed to continue
-        echo -e "\e[31m${test_script} failed but was allowed to continue, see log output above and cluster debug.\e[0m"
+        echo -e "\e[31m⚠️ ${test_script} failed but was allowed to continue, see log output above and cluster debug.\e[0m"
        EXIT_FLAG=1
      fi
      echo -e "\e[0Ksection_end:`date +%s`:${test_script##*/}\r\e[0K"
    done
    if [[ -n "$EXIT_FLAG" ]]; then
-      echo -e "\e[31mWARNING: One or more BB tests failed but were allowed to continue. See output of scripts above for details.\e[0m"
+      echo -e "\e[31m⚠️ WARNING: One or more BB tests failed but were allowed to continue. See output of scripts above for details.\e[0m"
    fi
 clean install:
-  stage: smoke tests
+  stage: 🔥 smoke tests
  extends:
    - .k3d-ci
  variables:
@@ -162,7 +162,7 @@ clean install:
      - runner_system_failure

 upgrade:
-  stage: smoke tests
+  stage: 🔥 smoke tests
  dependencies:
    - pre vars
  extends:
@@ -180,13 +180,13 @@ upgrade:
  variables:
    CLUSTER_NAME: "upgrade-${CI_COMMIT_SHORT_SHA}"
  script:
-    - echo "Install Big Bang from ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
+    - echo "🌌 Install Big Bang from ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
    - echo -e "\e[0Ksection_start:`date +%s`:git_master[collapsed=true]\r\e[0K\e[33;1mGit Fetch Master\e[37m"  
    - git fetch && git checkout ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}
    - echo -e "\e[0Ksection_end:`date +%s`:git_master\r\e[0K"
    - *deploy_bigbang
    - *test_bigbang
-    - echo "Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
+    - echo "🌌 Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
    - echo -e "\e[0Ksection_start:`date +%s`:git_upgrade[collapsed=true]\r\e[0K\e[33;1mGit Upgrade\e[37m"    
    - git reset --hard && git clean -fd
    - git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
@@ -217,7 +217,7 @@ upgrade:

 # Abstract for job manually triggering infrastructure builds
 .infra fork:
-  stage: network up
+  stage: 🔌 network up
  rules:
    # Run on scheduled jobs OR when `test-ci` label is assigned
    - if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
@@ -264,7 +264,7 @@ aws/network down:
    - .bigbang-gitlab-runner-tags
    - .infra cleanup
    - .network down
-  stage: network down
+  stage: 💣 network down
  environment:
    name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
    action: stop
@@ -283,7 +283,7 @@ aws/network down:

 # Create RKE2 cluster on AWS
 aws/rke2/cluster up:
-  stage: cluster up
+  stage: ⚓ cluster up
  extends:
    - .bigbang-gitlab-runner-tags
    - .infra create
@@ -303,7 +303,7 @@ aws/rke2/cluster up:

 # Install BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang up:
-  stage: bigbang up
+  stage: 🌌 bigbang up
  extends:
    - .bigbang-gitlab-runner-tags
    - .infra create
@@ -329,7 +329,7 @@ aws/rke2/bigbang up:

 # Run tests on BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang test:
-  stage: test
+  stage: 🤞 test
  extends:
    - .bigbang-gitlab-runner-tags
    - .infra create
@@ -371,7 +371,7 @@ aws/rke2/bigbang test:

 # Uninstall BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang down:
-  stage: bigbang down
+  stage: 💣 bigbang down
  extends:
    - .bigbang-gitlab-runner-tags
    - .infra cleanup
@@ -398,7 +398,7 @@ aws/rke2/bigbang down:

 # Destroy RKE2 cluster on AWS
 aws/rke2/cluster down:
-  stage: cluster down
+  stage: 💣 cluster down
  extends:
    - .bigbang-gitlab-runner-tags
    - .infra cleanup
@@ -423,7 +423,7 @@ aws/rke2/cluster down:
 #

 package:
-  stage: package
+  stage: 📦 package
  image: registry.dso.mil/platform-one/big-bang/bigbang/synker:0.0.3
  extends:
    - .bigbang-gitlab-runner-tags
@@ -483,7 +483,7 @@ package:
      - runner_system_failure

 release:
-  stage: release
+  stage: 🚀 release
  image: registry.gitlab.com/gitlab-org/release-cli:latest
  extends:
    - .bigbang-gitlab-runner-tags

--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -206,7 +206,7 @@ jaeger:
  git:
    repo: https://repo1.dso.mil/platform-one/big-bang/apps/core/jaeger.git
    path: "./chart"
-    tag: "2.23.0-bb.5"
+    tag: "2.26.0-bb.0"

  # -- Flux reconciliation overrides specifically for the Jaeger Package
  flux:
@@ -677,7 +677,7 @@ addons:
    git:
      repo: https://repo1.dso.mil/platform-one/big-bang/apps/developer-tools/gitlab-runner.git
      path: "./chart"
-      tag: "0.33.1-bb.2"
+      tag: "0.33.1-bb.3"

    # -- Flux reconciliation overrides specifically for the Gitlab Runner Package
    flux: {}
@@ -753,7 +753,7 @@ addons:
    git:
      repo: https://repo1.dso.mil/platform-one/big-bang/apps/developer-tools/sonarqube.git
      path: "./chart"
-      tag: "9.6.3-bb.8"
+      tag: "9.6.3-bb.9"

    # -- Flux reconciliation overrides specifically for the Sonarqube Package
    flux: {}

--- a/scripts/hosts.sh
+++ b/scripts/hosts.sh
 #!/bin/bash
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR

 ## Adds all the vs hostnames and LB IP to /etc/hosts
 ## Get the LB Hostname

--- a/scripts/install_flux.sh
+++ b/scripts/install_flux.sh
 #!/usr/bin/env bash

 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR

 #
 # global defaults
 #
-
+FLUX_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
+FLUX_KUSTOMIZATION="${FLUX_SCRIPT_DIR}/../base/flux"
 REGISTRY_URL=registry1.dso.mil
-FLUX_KUSTOMIZATION=base/flux
 FLUX_SECRET=private-registry
 WAIT_TIMEOUT=300


--- a/scripts/package/gits.sh
+++ b/scripts/package/gits.sh
 #!/bin/bash
-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x

 mkdir -p repos/


--- a/tests/ci/k3d/deploy_k3d.sh
+++ b/tests/ci/k3d/deploy_k3d.sh
 #!/usr/bin/env bash

-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x
 docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
 # if keycloak label or all packages label add deploy k3d without loadbalancer so metallb can be used
 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "keycloak" ||  $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then

--- a/tests/ci/k3d/metallb/install_metallb.sh
+++ b/tests/ci/k3d/metallb/install_metallb.sh
 #!/usr/bin/env bash

-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x

 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "keycloak" ||  $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then
  kubectl create -f tests/ci/k3d/metallb/metallb.yaml

--- a/tests/ci/k3d/values.yaml
+++ b/tests/ci/k3d/values.yaml
@@ -383,9 +383,16 @@ addons:
          persistence:
            size: 256Mi
          resources:
+            ## values raised to help pass CI after default values for gitaly are fixed then can revert to original request.
+            #requests:
+            #  cpu: 50m
+            #limits: {}
            requests:
-              cpu: 50m
-            limits: {}
+              cpu: 400m
+              memory: 600Mi
+            limits:
+              cpu: 400m
+              memory: 600Mi
        shared-secrets:
          resources:
            requests:
@@ -469,22 +476,23 @@ addons:
      runners:
        protected: false
      bbtests:
-        # TODO: This test runs fine locally with the same values, but fails in CI
-        enabled: false
+        enabled: true
        cypress:
          artifacts: true
+          envs:
+            cypress_baseUrl: "https://gitlab.bigbang.dev"
+            cypress_gitlab_first_name: "testrunner"
+            cypress_gitlab_last_name: "userrunner"
+            cypress_gitlab_email: "gitlab@bigbang.dev"
+            cypress_gitlab_username: "gitlabrunner_user"
+            cypress_gitlab_password: "gitlabrunner_pass"
+            cypress_gitlab_project: "runner-hello-world"
          secretEnvs:
            - name: cypress_adminpassword
              valueFrom:
                secretKeyRef:
                  name: gitlab-gitlab-initial-root-password
                  key: password
-          envs:
-            cypress_baseUrl: "https://gitlab.bigbang.dev"
-            cypress_gitlab_email: "gitlab@bigbang.dev"
-            cypress_gitlab_user: "gitlab_user"
-            cypress_gitlab_password: "gitlab_pass"
-            cypress_gitlab_project: "hello-world"

  anchore:
    enabled: false

--- a/tests/deploy/00_deploy_flux.sh
+++ b/tests/deploy/00_deploy_flux.sh
 #!/usr/bin/env bash

-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x

 # install flux with the dedicated helper script
 ./scripts/install_flux.sh \

--- a/tests/deploy/01_deploy_bigbang.sh
+++ b/tests/deploy/01_deploy_bigbang.sh
 #!/usr/bin/env bash

-set -ex
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -e
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+set -x

 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then
-  echo "all-packages label enabled, or on default branch or tag, enabling all addons"
+  echo "🌌 all-packages label enabled, or on default branch or tag, enabling all addons"
  yq e ".addons.*.enabled = "true"" $CI_VALUES_FILE > tmpfile && mv tmpfile $CI_VALUES_FILE
 else
  IFS=","
@@ -30,7 +31,7 @@ if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] && [[ "$CI_COMMIT_BRANCH" == "maste
 fi

 # deploy BigBang using dev sized scaling
-echo "Installing BigBang with the following configurations:"
+echo "🚀 Installing BigBang with the following configurations:"
 cat $CI_VALUES_FILE

 helm upgrade -i bigbang chart -n bigbang --create-namespace \
@@ -44,16 +45,16 @@ helm upgrade -i bigbang chart -n bigbang --create-namespace \

 # apply secrets kustomization pointing to current branch or master if an upgrade job
 if [[ $(git branch --show-current) == "${CI_DEFAULT_BRANCH}" ]]; then
-  echo "Deploying secrets from the ${CI_DEFAULT_BRANCH} branch"
+  echo "🚀 Deploying secrets from the ${CI_DEFAULT_BRANCH} branch"
  kubectl apply -f tests/ci/shared-secrets.yaml
 elif [[ $(git branch --show-current) == "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]]; then
-  echo "Deploying secrets from the ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} branch"
+  echo "🚀 Deploying secrets from the ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME} branch"
  cat tests/ci/shared-secrets.yaml | sed 's|master|'"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"'|g' | kubectl apply -f -
 elif [ -z "$CI_COMMIT_TAG" ]; then
-  echo "Deploying secrets from the ${CI_COMMIT_REF_NAME} branch"
+  echo "🚀 Deploying secrets from the ${CI_COMMIT_REF_NAME} branch"
  cat tests/ci/shared-secrets.yaml | sed 's|master|'"$CI_COMMIT_REF_NAME"'|g' | kubectl apply -f -
 else
-  echo "Deploying secrets from the ${CI_COMMIT_REF_NAME} tag"
+  echo "🚀 Deploying secrets from the ${CI_COMMIT_REF_NAME} tag"
  # NOTE: $CI_COMMIT_REF_NAME = $CI_COMMIT_TAG when running on a tagged build
  cat tests/ci/shared-secrets.yaml | sed 's|branch: master|tag: '"$CI_COMMIT_REF_NAME"'|g' | kubectl apply -f -
 fi
--- a/tests/deploy/02_wait_for_helmreleases.sh
+++ b/tests/deploy/02_wait_for_helmreleases.sh
 #!/usr/bin/env bash

 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR

 ## Array of core HRs
 CORE_HELMRELEASES=("gatekeeper" "istio-operator" "istio" "monitoring" "eck-operator" "ek" "fluent-bit" "twistlock" "cluster-auditor" "jaeger" "kiali")
@@ -37,12 +37,12 @@ function array_contains() {
 ## Function to check/wait on HR existence
 function check_if_hr_exist() {
    timeElapsed=0
-    echo "Waiting for $1 HR to exist"
+    echo "⏳ Waiting for $1 HR to exist"
    until kubectl get hr -n bigbang $1 &> /dev/null; do
      sleep 5
      timeElapsed=$(($timeElapsed+5))
      if [[ $timeElapsed -ge 60 ]]; then
-         echo "Timed out while waiting for $1 HR to exist"
+         echo "❌ Timed out while waiting for $1 HR to exist"
         exit 1
      fi
    done
@@ -61,7 +61,7 @@ function wait_all_hr() {
              break
            else
              artifactfailedcounter=$(($artifactfailedcounter+1))
-              echo "Helm Artifact Failed, waiting 5 seconds."
+              echo "⏳ Helm Artifact Failed, waiting 5 seconds."
              sleep 5
              hrstatus=$(kubectl get hr -n bigbang -o jsonpath='{.items[*].status.conditions[0].reason}')
            fi
@@ -70,8 +70,8 @@ function wait_all_hr() {
        if [[ "$hrstatus" =~ Failed ]]; then
            state=$(kubectl get hr -A -o go-template='{{range $items,$contents := .items}}{{printf "HR %s" $contents.metadata.name}}{{printf " status is %s\n" (index $contents.status.conditions 0).reason}}{{end}}')
            failed=$(echo "${state}" | grep "Failed")
-            echo "Found failed Helm Release(s). Exiting now."
-            echo "${failed}"
+            echo "❌ Found failed Helm Release(s). Exiting now."
+            echo "❌ ${failed}"
            failed_hrs=$(echo "{$failed}" | awk  '{print $2}')
            for hr in $failed_hrs; do
                kubectl describe hr -n bigbang $hr
@@ -80,14 +80,14 @@ function wait_all_hr() {
        fi
        if [[ "$hrready" != *Unknown* ]]; then
            if [[ "$hrready" != *False* ]]; then
-                echo "All HR's deployed"
+                echo "✅ All HR's deployed"
                break
            fi
        fi
        sleep 5
        timeElapsed=$(($timeElapsed+5))
        if [[ $timeElapsed -ge 1800 ]]; then
-            echo "Timed out while waiting for hr's to be ready."
+            echo "❌ Timed out while waiting for hr's to be ready."
            exit 1
        fi
    done
@@ -107,7 +107,7 @@ function wait_sts() {
      sleep 5
      timeElapsed=$(($timeElapsed+5))
      if [[ $timeElapsed -ge 600 ]]; then
-         echo "Timed out while waiting for stateful sets to be ready."
+         echo "❌ Timed out while waiting for stateful sets to be ready."
         exit 1
      fi
   done
@@ -127,7 +127,7 @@ function wait_daemonset(){
      sleep 5
      timeElapsed=$(($timeElapsed+5))
      if [[ $timeElapsed -ge 600 ]]; then
-         echo "Timed out while waiting for daemon sets to be ready."
+         echo "❌ Timed out while waiting for daemon sets to be ready."
         exit 1
      fi
   done
@@ -135,8 +135,8 @@ function wait_daemonset(){

 # Check for and run the wait_project function within <repo>/tests/wait.sh to wait for custom resources
 function wait_crd(){
-  yq e '. | keys | .[] | ... comments=""' "${VALUES_FILE}" | while IFS= read -r package; do
-    if [[ "$(yq e ".${package}.enabled" "${VALUES_FILE}")" == "true" ]]; then
+  yq e '(.,.addons) | .[] | ... comments="" | (path | join("."))' "${CI_VALUES_FILE}" | while IFS= read -r package; do
+    if [[ "$(yq e ".${package}.enabled" "${CI_VALUES_FILE}")" == "true" ]]; then
      gitrepo=$(yq e ".${package}.git.repo" "${VALUES_FILE}")
      version=$(yq e ".${package}.git.tag" "${VALUES_FILE}")
      if [[ -z "$version" || "$version" == "null" ]]; then
@@ -162,7 +162,7 @@ function wait_crd(){
 HELMRELEASES=(${CORE_HELMRELEASES[@]})
 if [[ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]] || [[ ! -z "$CI_COMMIT_TAG" ]] || [[ $CI_MERGE_REQUEST_LABELS =~ "all-packages" ]]; then
    HELMRELEASES+=(${ADD_ON_HELMRELEASES[@]})
-    echo "All helmreleases enabled: all-packages label enabled, or on default branch or tag."
+    echo "🌌 All helmreleases enabled: all-packages label enabled, or on default branch or tag."
 elif [[ ! -z "$CI_MERGE_REQUEST_LABELS" ]]; then
    IFS=","
    for package in $CI_MERGE_REQUEST_LABELS; do
@@ -178,10 +178,10 @@ elif [[ ! -z "$CI_MERGE_REQUEST_LABELS" ]]; then
            fi
        fi
    done
-    echo "Found enabled helmreleases: ${HELMRELEASES[@]}"
+    echo "✅ Found enabled helmreleases: ${HELMRELEASES[@]}"
 fi

-echo "Waiting on GitRepositories"
+echo "⏳ Waiting on GitRepositories"
 kubectl wait --for=condition=Ready --timeout 180s gitrepositories -n bigbang --all

 for package in "${HELMRELEASES[@]}";
@@ -189,18 +189,18 @@ do
    check_if_hr_exist "$package"
 done

-echo "Waiting on helm releases..."
+echo "⏳ Waiting on helm releases..."
 wait_all_hr
-echo "Waiting for custom resources..."
+echo "⏳ Waiting for custom resources..."
 wait_crd

 kubectl get helmreleases,kustomizations,gitrepositories -A

-echo "Waiting on Secrets Kustomization"
+echo "⏳ Waiting on Secrets Kustomization"
 kubectl wait --for=condition=Ready --timeout 300s kustomizations.kustomize.toolkit.fluxcd.io -n bigbang secrets

 # In case some helm releases are marked as ready before all objects are live...
-echo "Waiting on all jobs, deployments, statefulsets, and daemonsets"
+echo "⏳ Waiting on all jobs, deployments, statefulsets, and daemonsets"
 kubectl wait --for=condition=available --timeout 600s -A deployment --all > /dev/null
 wait_sts
 wait_daemonset

--- a/tests/tests/01_virtualservices.sh
+++ b/tests/tests/01_virtualservices.sh
@@ -2,7 +2,7 @@

 # exit on error
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR

 # Populate /etc/hosts
 ip=$(kubectl -n istio-system get service public-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

--- a/tests/tests/02_non_ironbank.sh
+++ b/tests/tests/02_non_ironbank.sh
@@ -2,7 +2,7 @@

 # exit on error
 set -e
-trap 'echo exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR
+trap 'echo ❌ exit at ${0}:${LINENO}, command was: ${BASH_COMMAND} 1>&2' ERR

 # Quick check for non iron bank images
 echo "Showing images not from ironbank:"
No results found