diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 936db8bcc16e7bb6add2a9988071a14708e85f71..79077b4b0bff91b5915bb5035c7aa4d60adb4fa9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,9 +95,14 @@ pre vars:
- |
set -e
for test_script in $(find ./tests/tests -type f -name '*.sh' | sort); do
- chmod +x ${test_script}
echo -e "\e[0Ksection_start:`date +%s`:${test_script##*/}[collapsed=true]\r\e[0K\e[33;1m${test_script##*/}\e[37m"
- ./${test_script}
+ chmod +x ${test_script}
+ echo "Executing ${test_script}..."
+ ./${test_script} && export EXIT_CODE=$? || export EXIT_CODE=$?
+ if [[ ${EXIT_CODE} -ne 0 ]]; then
+ echo "${test_script} failed, see log output above and cluster debug."
+ exit ${EXIT_CODE}
+ fi
echo -e "\e[0Ksection_end:`date +%s`:${test_script##*/}\r\e[0K"
done
@@ -124,10 +129,11 @@ clean install:
artifacts:
paths:
- images.txt
- - "cypress-tests/*/tests/cypress/screenshots"
- - "cypress-tests/*/tests/cypress/videos"
- expire_in: 7 days
+ - "test-artifacts/"
+ expire_in: 3 days
when: always
+ allow_failure:
+ exit_codes: 123
upgrade:
stage: smoke tests
@@ -163,10 +169,11 @@ upgrade:
- *test_bigbang
artifacts:
paths:
- - "cypress-tests/*/tests/cypress/screenshots"
- - "cypress-tests/*/tests/cypress/videos"
- expire_in: 7 days
+ - "test-artifacts/"
+ expire_in: 3 days
when: always
+ allow_failure:
+ exit_codes: 123
#-----------------------------------------------------------------------------------------------------------------------
# Rules for execution of AWS based K3S cluster deployment: Infrastructure jobs
@@ -309,6 +316,8 @@ aws/rke2/bigbang test:
- *test_bigbang
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
+ allow_failure:
+ exit_codes: 123
retry:
max: 2
when:
diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl
index 1eadf30aa44d7d0442e20c7b81b38ba137a6d946..454428a9324d0a0893305ec13cc4c656bfe8ffeb 100644
--- a/chart/templates/_helpers.tpl
+++ b/chart/templates/_helpers.tpl
@@ -76,6 +76,7 @@ Build common set of file extensions to include/exclude
/**/*.md
/**/*.txt
/**/*.sh
+ !/chart/tests/scripts/*.sh
{{- end -}}
{{/*
diff --git a/chart/templates/gitlab-runner/values.yaml b/chart/templates/gitlab-runner/values.yaml
index 7f392f00060c0e19ccb81c5129b22d836450a6f4..9e51f6488ba67b5428e6100c536abeed66369845 100644
--- a/chart/templates/gitlab-runner/values.yaml
+++ b/chart/templates/gitlab-runner/values.yaml
@@ -6,6 +6,9 @@
imagePullSecrets:
- name: private-registry
+networkPolicies:
+ enabled: {{ .Values.networkPolicies.enabled }}
+
monitoring:
enabled: {{ .Values.monitoring.enabled }}
{{- end -}}
diff --git a/chart/values.yaml b/chart/values.yaml
index f603612c8b52da70a54cae786b51d5db06894406..7ec45ef7ff2c03c774468a02a95d66043dbdf295 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -197,7 +197,7 @@ jaeger:
git:
repo: https://repo1.dso.mil/platform-one/big-bang/apps/core/jaeger.git
path: "./chart"
- tag: "2.23.0-bb.1"
+ tag: "2.23.0-bb.2"
# -- Flux reconciliation overrides specifically for the Jaeger Package
flux:
@@ -1031,7 +1031,7 @@ addons:
git:
repo: https://repo1.dso.mil/platform-one/big-bang/apps/cluster-utilities/velero.git
path: "./chart"
- tag: "2.23.6-bb.0"
+ tag: "2.23.6-bb.1"
# -- Flux reconciliation overrides specifically for the Velero Package
flux: {}
diff --git a/tests/ci/k3d/values.yaml b/tests/ci/k3d/values.yaml
index d4eb3cf589e0c0090a0173d1023d8518bd6db659..27b97fb8dcf5b413e3213f445433f418f2c338b3 100644
--- a/tests/ci/k3d/values.yaml
+++ b/tests/ci/k3d/values.yaml
@@ -36,11 +36,45 @@ logging:
max: 1g
kibana:
count: 1
+ bbtests:
+ # TODO: Connection refused on the script test currently
+ # https://repo1.dso.mil/platform-one/big-bang/apps/core/elasticsearch-kibana/-/issues/39
+ enabled: false
+ cypress:
+ artifacts: true
+ envs:
+ cypress_kibana_url: "https://kibana.bigbang.dev"
+ secretEnvs:
+ - name: cypress_elastic_password
+ valueFrom:
+ secretKeyRef:
+ name: "logging-ek-es-elastic-user"
+ key: elastic
+ scripts:
+ image: registry1.dso.mil/ironbank/stedolan/jq:1.6
+ envs:
+ elasticsearch_host: "https://{{ .Release.Name }}-es-http.{{ .Release.Namespace }}.svc.cluster.local:9200"
+ desired_version: "{{ .Values.elasticsearch.version }}"
+ secretEnvs:
+ - name: ELASTIC_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: "logging-ek-es-elastic-user"
+ key: elastic
fluentbit:
values:
securityContext:
privileged: true
+ bbtests:
+ # TODO: Connection refused on the test currently
+ # https://repo1.dso.mil/platform-one/big-bang/apps/core/fluentbit/-/issues/18
+ scripts:
+ # Image commented out to disable the test since the BB Test Lib version being used doesn't have the enabled flag
+ # image: registry1.dso.mil/ironbank/stedolan/jq:1.6
+ envs:
+ fluent_host: "http://{{ include \"fluent-bit.fullname\" . }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.port }}"
+ desired_version: "{{ .Values.image.tag }}"
istio:
enabled: true
@@ -50,6 +84,30 @@ istio:
auth:
strategy: "anonymous"
+jaeger:
+ enabled: true
+ values:
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: "https://tracing.bigbang.dev"
+
+kiali:
+ enabled: true
+ values:
+ cr:
+ spec:
+ auth:
+ strategy: "anonymous"
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: 'https://kiali.bigbang.dev'
+
clusterAuditor:
enabled: true
values:
@@ -81,6 +139,17 @@ monitoring:
cpu: 100m
memory: 30Mi
limits: {}
+ grafana:
+ testFramework:
+ enabled: false
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_prometheus_url: 'https://prometheus.bigbang.dev'
+ cypress_grafana_url: 'https://grafana.bigbang.dev'
+ cypress_alertmanager_url: 'https://alertmanager.bigbang.dev'
gatekeeper:
enabled: true
@@ -102,6 +171,23 @@ gatekeeper:
excludedNamespaces:
# Allows load balancer containers to map ports for k3d
- istio-system
+ bbtests:
+ # TODO: Test will need to be refactored at BB level to properly run since we can't turn everything to deny
+ # https://repo1.dso.mil/platform-one/big-bang/apps/core/policy/-/issues/133
+ enabled: false
+ scripts:
+ image: registry1.dso.mil/ironbank/opensource/kubernetes-1.21/kubectl:v1.21.1
+ additionalVolumeMounts:
+ - name: "{{ .Chart.Name }}-test-config"
+ mountPath: /yaml
+ - name: "{{ .Chart.Name }}-kube-cache"
+ mountPath: /.kube/cache
+ additionalVolumes:
+ - name: "{{ .Chart.Name }}-test-config"
+ configMap:
+ name: "{{ .Chart.Name }}-test-config"
+ - name: "{{ .Chart.Name }}-kube-cache"
+ emptyDir: {}
twistlock:
enabled: true
@@ -109,8 +195,19 @@ twistlock:
console:
persistence:
size: 256Mi
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_baseUrl: "https://twistlock.bigbang.dev"
+ scripts:
+ image: registry1.dso.mil/ironbank/stedolan/jq:1.6
+ envs:
+ twistlock_host: "https://twistlock.bigbang.dev"
+ desired_version: "{{ .Values.console.image.tag }}"
-# Explicitly enable all addons for CI
+# Addons are toggled based on labels in CI
addons:
argocd:
enabled: false
@@ -152,6 +249,19 @@ addons:
cpu: 50m
memory: 128Mi
limits: {}
+ configs:
+ secret:
+ argocdServerAdminPassword: '$2a$10$rUDZDckdDZ2TEwk9PDs3QuqjkL58qR1IHE1Kj4MwDx.7/m5dytZJm'
+ bbtests:
+ # TODO: Disabled pending resolution of some "timing?" issues
+ # https://repo1.dso.mil/platform-one/big-bang/apps/core/argocd/-/issues/17
+ enabled: false
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: "https://argocd.bigbang.dev"
+ cypress_user: "admin"
+ cypress_password: "Password123"
authservice:
enabled: false
@@ -192,6 +302,8 @@ addons:
webservice:
minReplicas: 1
maxReplicas: 1
+ helmTests:
+ enabled: false
sidekiq:
minReplicas: 1
maxReplicas: 1
@@ -248,6 +360,34 @@ addons:
slave:
persistence:
size: 256Mi
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_baseUrl: https://gitlab.bigbang.dev
+ cypress_gitlab_first_name: "test"
+ cypress_gitlab_last_name: "user"
+ cypress_gitlab_username: "testuser"
+ cypress_gitlab_password: "12345678"
+ cypress_gitlab_email: "testuser@example.com"
+ cypress_gitlab_project: "my-awesome-project"
+ secretEnvs:
+ - name: cypress_adminpassword
+ valueFrom:
+ secretKeyRef:
+ name: gitlab-gitlab-initial-root-password
+ key: password
+ scripts:
+ image: "registry.dso.mil/platform-one/big-bang/apps/developer-tools/gitlab/bbtests:0.0.3"
+ envs:
+ GITLAB_USER: "testuser"
+ GITLAB_PASS: "12345678"
+ GITLAB_EMAIL: "testuser@example.com"
+ GITLAB_PROJECT: "my-awesome-project"
+ GITLAB_REPOSITORY: https://gitlab.bigbang.dev
+ GITLAB_ORIGIN: https://testuser:12345678@gitlab.bigbang.dev
+ GITLAB_REGISTRY: registry.bigbang.dev
gitlabRunner:
enabled: false
@@ -257,6 +397,25 @@ addons:
memory: 64Mi
cpu: 50m
limits: {}
+ runners:
+ protected: false
+ bbtests:
+ # TODO: This test runs fine locally with the same values, but fails in CI
+ enabled: false
+ cypress:
+ artifacts: true
+ secretEnvs:
+ - name: cypress_adminpassword
+ valueFrom:
+ secretKeyRef:
+ name: gitlab-gitlab-initial-root-password
+ key: password
+ envs:
+ cypress_baseUrl: "https://gitlab.bigbang.dev"
+ cypress_gitlab_email: "gitlab@bigbang.dev"
+ cypress_gitlab_user: "gitlab_user"
+ cypress_gitlab_password: "gitlab_pass"
+ cypress_gitlab_project: "hello-world"
anchore:
enabled: false
@@ -385,6 +544,19 @@ addons:
cpu: 100m
memory: 200Mi
limits: {}
+ bbtests:
+ enabled: true
+ scripts:
+ image: registry1.dso.mil/ironbank/anchore/cli/cli:0.9.1
+ envs:
+ ANCHORE_CLI_URL: "https://anchore-api.bigbang.dev/v1"
+ ANCHORE_CLI_USER: admin
+ secretEnvs:
+ - name: ANCHORE_CLI_PASS
+ valueFrom:
+ secretKeyRef:
+ name: "{{ template \"anchore-engine.fullname\" . }}-admin-pass"
+ key: ANCHORE_ADMIN_PASSWORD
sonarqube:
enabled: false
@@ -409,6 +581,19 @@ addons:
limits: {}
tests:
enabled: false
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: "https://sonarqube.bigbang.dev"
+ cypress_url_setup: "https://sonarqube.bigbang.dev/setup"
+ cypress_user: "admin"
+ cypress_password: "new_admin_password"
+ account:
+ adminPassword: new_admin_password
+ currentAdminPassword: admin
+ curlContainerImage: registry1.dso.mil/ironbank/big-bang/base:8.4
minioOperator:
enabled: false
@@ -424,6 +609,41 @@ addons:
- servers: 1
volumesPerServer: 4
size: 256Mi
+ bbtests:
+ # TODO: Seems like a timing issue with BB CI
+ # https://repo1.dso.mil/platform-one/big-bang/apps/application-utilities/minio/-/issues/7
+ enabled: false
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: 'http://minio.bigbang.dev/login'
+ secretEnvs:
+ - name: cypress_secretkey
+ valueFrom:
+ secretKeyRef:
+ name: "{{ .Values.tenants.secrets.name }}"
+ key: secretkey
+ - name: cypress_accesskey
+ valueFrom:
+ secretKeyRef:
+ name: "{{ .Values.tenants.secrets.name }}"
+ key: accesskey
+ scripts:
+ image: registry1.dso.mil/ironbank/opensource/minio/mc:RELEASE.2021-09-02T09-21-27Z
+ envs:
+ MINIO_PORT: '80'
+ MINIO_HOST: 'http://minio'
+ secretEnvs:
+ - name: SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ name: "{{ .Values.tenants.secrets.name }}"
+ key: secretkey
+ - name: ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ name: "{{ .Values.tenants.secrets.name }}"
+ key: accesskey
mattermostoperator:
enabled: false
@@ -451,6 +671,15 @@ addons:
- servers: 1
volumesPerServer: 4
size: 256Mi
+ bbtests:
+ enabled: true
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: https://chat.bigbang.dev
+ cypress_mm_email: "test@bigbang.dev"
+ cypress_mm_user: "bigbang"
+ cypress_mm_password: "Bigbang#123"
nexus:
enabled: false
@@ -467,42 +696,80 @@ addons:
requests:
cpu: 100m
memory: 1500Mi
+ bbtests:
+ # TODO: Disabled pending resolution of "timing?" issues
+ # https://repo1.dso.mil/platform-one/big-bang/apps/developer-tools/nexus/-/issues/9
+ enabled: false
+ cypress:
+ artifacts: true
+ envs:
+ cypress_nexus_url: "https://nexus.bigbang.dev"
+ cypress_nexus_user: "admin"
+ cypress_nexus_pass_new: "new_admin_password"
+ secretEnvs:
+ - name: cypress_nexus_pass
+ valueFrom:
+ secretKeyRef:
+ name: nexus-repository-manager-secret
+ key: admin.password
velero:
enabled: false
plugins:
- aws
values:
- # Set provider name and backup storage location bucket name
- configuration:
- provider: aws
- # TODO: Bucket for this
- backupStorageLocation:
- bucket: bigbang-dogfood-velero
- prefix: ci
- # config:
- # region: us-west-1
- # profile: test
- # volumeSnapshotLocation:
- # provider: aws
- # config:
- # bucket: velero
- # region: us-west-1
-
- # schedules:
- # mybackup:
- # labels:
- # myenv: foo
- # schedule: "0 0 * * *"
- # template:
- # ttl: "240h"
- # includedNamespaces:
- # - foo
-
- # Set a service account so that the CRD clean up job has proper permissions to delete CRDs
serviceAccount:
server:
name: velero
+ configuration:
+ # minio uses s3 provider
+ provider: aws
+ backupStorageLocation:
+ bucket: velero
+ config: &minio-config
+ region: velero
+ insecureSkipTLSVerify: "true"
+ s3ForcePathStyle: "true"
+ s3Url: &minio-address https://minio.bigbang.dev
+ volumeSnapshotLocation:
+ provider: aws
+ config:
+ region: velero
+ credentials:
+ useSecret: true
+ secretContents:
+ cloud: |
+ [default]
+ aws_access_key_id = minio
+ aws_secret_access_key = minio123
+ bbtests:
+ # TODO: Velero test is messy and times out running in BB CI
+ # https://repo1.dso.mil/platform-one/big-bang/apps/cluster-utilities/velero/-/issues/9
+ enabled: false
+ scripts:
+ image: registry1.dso.mil/ironbank/opensource/velero/velero:v1.6.0
+ additionalVolumes:
+ - name: transfer-kubectl
+ emptyDir: {}
+ - name: &yamlVolName yaml-configs
+ configMap:
+ name: "{{ .Chart.Name }}-backup-restore-files-config"
+ additionalVolumeMounts:
+ - name: transfer-kubectl
+ mountPath: /usr/local/bin/kubectl
+ subPath: kubectl
+ - name: *yamlVolName
+ mountPath: &yamlMountPath /yaml
+ envs:
+ MINIO_HOST: *minio-address
+ TEST_YAML_DIR: *yamlMountPath
+ MINIO_USER: minio
+ MINIO_PASS: minio123
+ secretEnvs:
+ - name: NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
keycloak:
enabled: false
@@ -514,4 +781,11 @@ addons:
requests:
cpu: 10m
memory: 16Mi
- limits: {}
\ No newline at end of file
+ limits: {}
+ bbtests:
+ # TODO: Keycloak in CI not supported yet
+ enabled: false
+ cypress:
+ artifacts: true
+ envs:
+ cypress_url: "https://keycloak.bigbang.dev"
diff --git a/tests/tests/03_cypress.sh b/tests/tests/03_cypress.sh
deleted file mode 100755
index d817e426c81d846e69d5a6363d791e758b220a1e..0000000000000000000000000000000000000000
--- a/tests/tests/03_cypress.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env bash
-
-# exit on error
-set -e
-
-#Clear out folder if present
-rm -rf cypress-tests/
-#Create folder for cypress tests
-mkdir -p cypress-tests/
-
-#Cloning core
-yq e '. | keys | .[] | ... comments=""' "tests/ci/k3d/values.yaml" | while IFS= read -r package; do
- if [[ "$(yq e ".${package}.enabled" "tests/ci/k3d/values.yaml")" == "true" ]]; then
- #Checking for branch not tag
- if [ "$(yq e ".${package}.git.tag" "chart/values.yaml")" != null ]; then
- echo "Cloning ${package} into cypress-tests"
- git -C cypress-tests/ clone -b $(yq e ".${package}.git.tag" "chart/values.yaml") $(yq e ".${package}.git.repo" "chart/values.yaml")
- else
- echo "Cloning ${package} into cypress-tests"
- git -C cypress-tests/ clone -b $(yq e ".${package}.git.branch" "chart/values.yaml") $(yq e ".${package}.git.repo" "chart/values.yaml")
- fi
- fi
-done
-
-#Cloning addons
-yq e '.addons | keys | .[] | ... comments=""' "tests/ci/k3d/values.yaml" | while IFS= read -r package; do
- if [ "$(yq e ".addons.${package}.enabled" "tests/ci/k3d/values.yaml")" == "true" ]; then
- #Checking for branch not tag
- if [ "$(yq e ".addons.${package}.git.tag" "chart/values.yaml")" != null ]; then
- echo "Cloning ${package} into cypress-tests"
- git -C cypress-tests/ clone -b $(yq e ".addons.${package}.git.tag" "chart/values.yaml") $(yq e ".addons.${package}.git.repo" "chart/values.yaml")
- else
- echo "Cloning ${package} into cypress-tests"
- git -C cypress-tests/ clone -b $(yq e ".addons.${package}.git.branch" "chart/values.yaml") $(yq e ".addons.${package}.git.repo" "chart/values.yaml")
- fi
- fi
-done
-
-#Running Cypress tests
-for dir in cypress-tests/*/
-do
- if [ -f "${dir}tests/cypress.json" ]; then
- if [ "$(yq e ".addons.keycloak.enabled" "tests/ci/k3d/values.yaml")" == "true" ]; then
- echo "Running cypress tests. Keycloak is enabled. Directory is ${dir}"
- if [ "${dir}" == "cypress-tests/elasticsearch-kibana/" ]; then
- echo "Keycloak is enabled and cypress directory is ${dir}"
- echo "Running cypress tests in ${dir}"
- CYPRESS_kibana_url=kibana.admin.bigbang.dev cypress run --project "${dir}"tests
- fi
- if [ "${dir}" == "cypress-tests/monitoring/" ]; then
- echo "Keycloak is enabled and cypress directory is ${dir}"
- echo "Running cypress tests in ${dir}"
- CYPRESS_prometheus_url=prometheus.admin.bigbang.dev CYPRESS_grafana_url=grafana.admin.bigbang.dev cypress run --project "${dir}"tests
- fi
- if [ "${dir}" == "cypress-tests/twistlock/" ]; then
- echo "Keycloak is enabled and cypress directory is ${dir}"
- echo "Running cypress tests in ${dir}"
- CYPRESS_twistlock_url=twistlock.admin.bigbang.dev cypress run --project "${dir}"tests
- fi
- else
- echo "Keycloak not enabled"
- echo "Running cypress tests in ${dir}"
- cypress run --project "${dir}"tests
- fi
- fi
-done
-
diff --git a/tests/tests/03_helm_tests.sh b/tests/tests/03_helm_tests.sh
new file mode 100755
index 0000000000000000000000000000000000000000..baecbb39de746f2708b8945fbd088f9ca0adfdba
--- /dev/null
+++ b/tests/tests/03_helm_tests.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+# exit on error
+set -e
+
+# Get original CoreDNS config
+kubectl get configmap -n kube-system coredns -o jsonpath='{.data.NodeHosts}' > newhosts
+
+# Safeguard in case configmap doesn't end with newline
+if [[ $(tail -c 1 newhosts) != "" ]]; then
+ echo "" >> newhosts
+fi
+
+# Get each VS hostname + ingress gateway IP and add to newhosts
+for vs in $(kubectl get virtualservice -A -o go-template='{{range .items}}{{.metadata.name}}{{":"}}{{.metadata.namespace}}{{" "}}{{end}}'); do
+ vs_name=$(echo ${vs} | awk -F: '{print $1}')
+ vs_namespace=$(echo ${vs} | awk -F: '{print $2}')
+ hosts=$(kubectl get virtualservice ${vs_name} -n ${vs_namespace} -o go-template='{{range .spec.hosts}}{{.}}{{" "}}{{end}}')
+ gateway=$(kubectl get virtualservice ${vs_name} -n ${vs_namespace} -o jsonpath='{.spec.gateways[0]}' | awk -F/ '{print $2}')
+ ingress_gateway=$(kubectl get gateway -n istio-system $gateway -o jsonpath='{.spec.selector.app}')
+ external_ip=$(kubectl get svc -n istio-system $ingress_gateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+ for host in $hosts; do
+ host=$(echo ${host} | xargs)
+ # Remove previous entry if on upgrade job
+ sed -i "/$host/d" newhosts
+ echo "${external_ip} ${host}" >> newhosts
+ done
+done
+
+# Patch CoreDNS and restart pod
+echo "Setting up CoreDNS for VS resolution..."
+hosts=$(cat newhosts) yq e -n '.data.NodeHosts = strenv(hosts)' > patch.yaml
+kubectl patch configmap -n kube-system coredns --patch "$(cat patch.yaml)"
+kubectl rollout restart deployment -n kube-system coredns
+kubectl rollout status deployment -n kube-system coredns --timeout=30s
+
+# Gather all HRs we should test
+installed_helmreleases=$(helm list -n bigbang -o json | jq '.[].name' | tr -d '"' | grep -v "bigbang")
+mkdir -p test-artifacts
+ERRORS=0
+
+# For each HR, if it has helm tests: run them, capture exit code, output logs, and save cypress artifacts
+for hr in $installed_helmreleases; do
+ echo "Running helm tests for ${hr}..."
+ test_result=$(helm test $hr -n bigbang) && export EXIT_CODE=$? || export EXIT_CODE=$?
+ test_result=$(echo "${test_result}" | sed '/NOTES/Q')
+ namespace=$(echo "$test_result" | yq eval '."NAMESPACE"' -)
+ test_suite=$(echo "$test_result" | yq eval '.["TEST SUITE"]' -)
+ if [ ! $test_suite == "None" ]; then
+ # Since logs are cluttery, only output when failed
+ if [[ ${EXIT_CODE} -ne 0 ]]; then
+ echo "⌠One or more tests failed for ${hr}"
+ ERRORS=$((ERRORS + 1))
+ for pod in $(echo "$test_result" | grep "TEST SUITE" | grep "test" | awk -F: '{print $2}' | xargs); do
+ # Only output failed pod logs, not all test pods
+ if [[ $(kubectl get pod -n ${namespace} ${pod} -o jsonpath='{.status.phase}' 2>/dev/null | xargs) == "Failed" ]]; then
+ echo -e "---\nLogs for ${pod}:\n---"
+ kubectl logs --tail=-1 -n ${namespace} ${pod}
+ fi
+ done
+ echo "---"
+ else
+ echo "✅ All tests sucessful for ${hr}"
+ fi
+
+ # Grab script logs to save for the artifacts (don't get cypress because its not text friendly + we have the videos/screenshots)
+ for pod in $(echo "$test_result" | grep "TEST SUITE" | grep "test" | awk -F: '{print $2}' | xargs); do
+ if [[ ! "$pod" =~ "cypress" ]]; then
+ if kubectl get pod -n ${namespace} ${pod} &>/dev/null; then
+ mkdir -p test-artifacts/${hr}/scripts
+ kubectl logs --tail=-1 -n ${namespace} ${pod} >> test-artifacts/${hr}/scripts/pod-logs.txt
+ fi
+ fi
+ done
+
+ # Always save off the artifacts if they exist
+ if kubectl get configmap -n ${namespace} cypress-screenshots &>/dev/null; then
+ mkdir -p test-artifacts/${hr}/cypress
+ kubectl get configmap -n ${namespace} cypress-screenshots -o jsonpath='{.data.cypress-screenshots\.tar\.gz\.b64}' > cypress-screenshots.tar.gz.b64
+ cat cypress-screenshots.tar.gz.b64 | base64 -d > cypress-screenshots.tar.gz
+ tar -zxf cypress-screenshots.tar.gz --strip-components=2 -C test-artifacts/${hr}/cypress
+ rm -rf cypress-screenshots.tar.gz.b64 cypress-screenshots.tar.gz
+ kubectl delete configmap -n ${namespace} cypress-screenshots &>/dev/null
+ fi
+ if kubectl get configmap -n ${namespace} cypress-videos &>/dev/null; then
+ mkdir -p test-artifacts/${hr}/cypress
+ kubectl get configmap -n ${namespace} cypress-videos -o jsonpath='{.data.cypress-videos\.tar\.gz\.b64}' > cypress-videos.tar.gz.b64
+ cat cypress-videos.tar.gz.b64 | base64 -d > cypress-videos.tar.gz
+ tar -zxf cypress-videos.tar.gz --strip-components=2 -C test-artifacts/${hr}/cypress
+ rm -rf cypress-videos.tar.gz.b64 cypress-videos.tar.gz
+ kubectl delete configmap -n ${namespace} cypress-videos &>/dev/null
+ fi
+ else
+ echo "😞 No tests found for ${hr}"
+ fi
+done
+
+echo "Finished running all helm tests."
+
+if [ $ERRORS -gt 0 ]; then
+ echo "⌠Encountered $ERRORS package(s) with errors while running tests. See output logs for failed test(s) above and artifacts in the job."
+ exit 123
+else
+ echo "✅ All helm tests run successfully."
+fi
\ No newline at end of file