From 4a91c66bb486594548577da87d98eec57c9092c1 Mon Sep 17 00:00:00 2001
From: Josh Wolf <josh@joshwolf.dev>
Date: Mon, 15 Mar 2021 14:34:41 -0600
Subject: [PATCH] ci: allow for dynamic inclusion of addons based on ci and use
k3d for incluster testing
---
.gitlab-ci.yml | 9 +-
.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml | 59 +++-
.../minio/minio-operator/gitrepository.yaml | 2 +-
.../minio-operator-helmrelease.yaml | 2 +-
.../minio/minio-operator/namespace.yaml | 2 +-
scripts/deploy/01_deploy_bigbang.sh | 16 +-
scripts/deploy/02_wait_for_helmreleases.sh | 6 +-
scripts/package/synker.yaml | 3 -
tests/ci/k3d/config.yaml | 22 ++
tests/ci/k3d/values.yaml | 298 +++++++++++++++++-
10 files changed, 387 insertions(+), 32 deletions(-)
create mode 100644 tests/ci/k3d/config.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c42da226b1..caa956144e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -101,17 +101,20 @@ pre vars:
clean install:
stage: smoke tests
extends:
- - .k3s-ci
+ - .k3d-ci
variables:
CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}"
rules:
+ - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master"'
+ when: never
- *chart_changes
script:
- *deploy_bigbang
- *test_bigbang
# Fetch list of all images ran
- - curl -sfL k3s:8081/images | jq -r '.images[].repoTags[0]' > images.txt
+ - cid=$(docker ps -aqf "name=k3d-${CI_JOB_ID}-server-0")
+ - docker exec $cid crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)' | tee images.txt
artifacts:
paths:
- images.txt
@@ -122,7 +125,7 @@ upgrade:
dependencies:
- pre vars
extends:
- - .k3s-ci
+ - .k3d-ci
rules:
# skip job when MR title starts with 'Breaking Change'
- if: '$CI_MERGE_REQUEST_TITLE =~ /^Breaking Change/'
diff --git a/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml b/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml
index f37d75867c..903a991120 100644
--- a/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml
+++ b/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml
@@ -1,5 +1,5 @@
.k8s-util:
- image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k8s-ci:v1.20.2
+ image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k8s-ci:v1.20.4-bb.2
.dind-runner:
tags:
@@ -7,20 +7,73 @@
- privileged
- dogfood
+#
+# In cluster k3s using K3D with the docker daemon as a sidecar
+#
+# This will connect to a remote docker daemon over tls tcp (defined at installation of gitlab runners) and create
+# a k3d cluster in a custom built docker bridge network.
+#
+.k3d-ci:
+ extends:
+ - .k8s-util
+ - .dind-runner
+ # services:
+ # # Added in through gitlab ci configuration, left in incase some poor soul needs to come debug this later
+ # - name: docker:20.10.5-dind
+ variables:
+ DOCKER_HOST: tcp://localhost:2376
+ DOCKER_TLS_CERTDIR: "/certs"
+ DOCKER_TLS_VERIFY: 1
+ DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+ DOCKER_DRIVER: overlay2
+ before_script:
+ - docker network create ${CI_JOB_ID} --driver=bridge -o "com.docker.network.driver.mtu"="1450"
+ - k3d cluster create ${CI_JOB_ID} --config tests/ci/k3d/config.yaml --network ${CI_JOB_ID}
+ - until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done
+ - kubectl get all -A
+ after_script:
+ - kubectl get all -A
+ - kubectl get gitrepository,helmrelease,kustomizations -A
+ - k3d cluster delete ${CI_JOB_ID}
+ - docker network rm ${CI_JOB_ID}
+
+#
+# In cluster k3s using k3s as a sidecar
+#
+# This will spin up k3s as a gitlab ci sidecar
+#
.k3s-ci:
extends:
- .k8s-util
- .dind-runner
services:
- - name: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k3s-ci:v1.20.2-k3s1-bb.0
+ - name: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k3s-ci:v1.20.4-k3s1-bb.0
alias: k3s
command:
- "server"
- "--tls-san=k3s"
- - "--disable=metrics-server"
- "--disable=traefik"
before_script:
- mkdir -p ~/.kube
- curl -sfL k3s:8081?service=k3s > ~/.kube/config
- kubectl version
- kubectl cluster-info
+ after_script:
+ - kubectl get all -A
+ - kubectl get gitrepository,helmrelease,kustomizations -A
+
+#
+# In cluster k3d using K3D with a dind container
+#
+# This will spin up a k3d cluster using a dind container as the base container
+#
+.k3d-dind-ci:
+ extends:
+ - .dind-runner
+ image:
+ name: rancher/k3d:v4.3.0-dind
+ before_script:
+ - nohup dockerd-entrypoint.sh &
+ - until docker ps 2>&1 > /dev/null; do sleep 1s; done
+ - k3d cluster create --config tests/ci/k3d/config.yaml
+ - until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done
diff --git a/chart/templates/minio/minio-operator/gitrepository.yaml b/chart/templates/minio/minio-operator/gitrepository.yaml
index 6d80c63d41..970e839dc0 100644
--- a/chart/templates/minio/minio-operator/gitrepository.yaml
+++ b/chart/templates/minio/minio-operator/gitrepository.yaml
@@ -1,4 +1,4 @@
-{{- if and (not .Values.offline) .Values.addons.minioOperator.enabled }}
+{{- if and (not .Values.offline) (or .Values.addons.minioOperator.enabled .Values.addons.minio.enabled) }}
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
diff --git a/chart/templates/minio/minio-operator/minio-operator-helmrelease.yaml b/chart/templates/minio/minio-operator/minio-operator-helmrelease.yaml
index bd7b1de6a5..4dd6c44fec 100644
--- a/chart/templates/minio/minio-operator/minio-operator-helmrelease.yaml
+++ b/chart/templates/minio/minio-operator/minio-operator-helmrelease.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.minioOperator.enabled }}
+{{- if or .Values.addons.minioOperator.enabled .Values.addons.minio.enabled }}
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
diff --git a/chart/templates/minio/minio-operator/namespace.yaml b/chart/templates/minio/minio-operator/namespace.yaml
index 88a1e7cdfe..e78c6493da 100644
--- a/chart/templates/minio/minio-operator/namespace.yaml
+++ b/chart/templates/minio/minio-operator/namespace.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.addons.minioOperator.enabled }}
+{{- if or .Values.addons.minioOperator.enabled .Values.addons.minio.enabled }}
apiVersion: v1
kind: Namespace
metadata:
diff --git a/scripts/deploy/01_deploy_bigbang.sh b/scripts/deploy/01_deploy_bigbang.sh
index 1f94df3c6a..8d07055439 100755
--- a/scripts/deploy/01_deploy_bigbang.sh
+++ b/scripts/deploy/01_deploy_bigbang.sh
@@ -2,6 +2,8 @@
set -ex
+CI_VALUES_FILE="tests/ci/k3d/values.yaml"
+
# Deploy flux and wait for it to be ready
echo "Installing Flux"
flux --version
@@ -26,12 +28,22 @@ kubectl apply -f ./scripts/deploy/flux.yaml
# wait for flux
flux check
+IFS=","
+for package in $CI_MERGE_REQUEST_LABELS; do
+ if [ "$(yq e ".addons.${package}.enabled" $CI_VALUES_FILE 2>/dev/null)" == "false" ]; then
+ echo "Identified \"$package\" from labels"
+ yq e ".addons.${package}.enabled = "true"" $CI_VALUES_FILE > tmpfile && mv tmpfile $CI_VALUES_FILE
+ fi
+done
+
# deploy BigBang using dev sized scaling
-echo "Installing BigBang"
+echo "Installing BigBang with the following configurations:"
+cat $CI_VALUES_FILE
+
helm upgrade -i bigbang chart -n bigbang --create-namespace \
--set registryCredentials[0].username='robot$bigbang' --set registryCredentials[0].password=${REGISTRY1_PASSWORD} \
--set registryCredentials[0].registry=registry1.dso.mil \
--f tests/ci/k3d/values.yaml
+-f ${CI_VALUES_FILE}
# apply secrets kustomization pointing to current branch
echo "Deploying secrets from the ${CI_COMMIT_REF_NAME} branch"
diff --git a/scripts/deploy/02_wait_for_helmreleases.sh b/scripts/deploy/02_wait_for_helmreleases.sh
index 18df499ed4..83099665a4 100755
--- a/scripts/deploy/02_wait_for_helmreleases.sh
+++ b/scripts/deploy/02_wait_for_helmreleases.sh
@@ -1,13 +1,15 @@
#!/usr/bin/env bash
-set -ex
+set -e
## This is an array to instantiate the order of wait conditions
ORDERED_HELMRELEASES="gatekeeper istio-operator istio monitoring eck-operator ek fluent-bit twistlock cluster-auditor authservice argocd gitlab haproxy-sso gitlab-runner minio-operator minio anchore sonarqube mattermost-operator mattermost"
-## This the actual deployed helmrelease objects in the cluster
+## This is the actual deployed helmrelease objects in the cluster
DEPLOYED_HELMRELEASES=$(kubectl get hr --no-headers -n bigbang | awk '{ print $1}')
+printf "Identified the following deployed helmreleases:\n%s" "${DEPLOYED_HELMRELEASES}"
+
## Function to test an array contains an element
## Args:
## $1: array to search
diff --git a/scripts/package/synker.yaml b/scripts/package/synker.yaml
index 45a6d50ce5..b25c1a450b 100644
--- a/scripts/package/synker.yaml
+++ b/scripts/package/synker.yaml
@@ -33,9 +33,6 @@ source:
- registry1.dso.mil/ironbank/anchore/enterprise/enterprise:2.4.1
- registry1.dso.mil/ironbank/anchore/enterpriseui/enterpriseui:2.4.1
- # not capturing this image from CI deployment. Adding it explicitly
- - registry1.dso.mil/ironbank/opensource/fluent/fluent-bit:1.6.6
-
# Images used by pipeline-templates
# NOTE: These are __not__ fully inclusive yet, see this issue for updates: https://repo1.dso.mil/platform-one/big-bang/bigbang/-/issues/145
- registry.il2.dso.mil/platform-one/devops/pipeline-templates/pipeline-job/sonar-scanner-dotnet31:4.10
diff --git a/tests/ci/k3d/config.yaml b/tests/ci/k3d/config.yaml
new file mode 100644
index 0000000000..1ea9880058
--- /dev/null
+++ b/tests/ci/k3d/config.yaml
@@ -0,0 +1,22 @@
+apiVersion: k3d.io/v1alpha2
+kind: Simple
+name: ci
+servers: 1
+options:
+ k3s:
+ extraServerArgs:
+ - --disable=traefik
+ k3d:
+ wait: true
+volumes:
+ - volume: /etc/machine-id:/etc/machine-id
+ nodeFilters:
+ - server[*]
+ - agent[*]
+ports:
+ - port: 80:80
+ nodeFilters:
+ - loadbalancer
+ - port: 443:443
+ nodeFilters:
+ - loadbalancer
diff --git a/tests/ci/k3d/values.yaml b/tests/ci/k3d/values.yaml
index 1be727d414..3851125124 100644
--- a/tests/ci/k3d/values.yaml
+++ b/tests/ci/k3d/values.yaml
@@ -14,17 +14,17 @@ logging:
persistence:
size: 5Gi
resources:
- limits:
- cpu: 1
- memory: 2Gi
+ requests:
+ cpu: .5
+ limits: {}
data:
count: 1
persistence:
size: 5Gi
resources:
- limits:
- cpu: 1
- memory: 2Gi
+ requests:
+ cpu: .5
+ limits: {}
istio:
enabled: true
@@ -36,14 +36,64 @@ istio:
clusterAuditor:
enabled: true
+ values:
+ resources:
+ requests:
+ cpu: 100m
+ memory: .5Gi
+ limits: {}
monitoring:
enabled: true
+ values:
+ alertmanager:
+ alertmanagerSpec:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ prometheusOperator:
+ resources:
+ requests:
+ cpu: 250m
+ memory: 400Mi
+ limits: {}
+ prometheus:
+ prometheusSpec:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ grafana:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits: {}
+ kubeStateMetrics:
+ resources:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+ limits: {}
+ nodeExporter:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 30Mi
+ limits: {}
gatekeeper:
enabled: true
values:
replicas: 1
+ resources:
+ requests:
+ cpu: 100m
+ memory: 256Mi
+ limits: {}
twistlock:
enabled: true
@@ -55,27 +105,243 @@ twistlock:
# Explicitly enable all addons for CI
addons:
argocd:
- enabled: true
+ enabled: false
+ values:
+ controller:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ dex:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ redis:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ server:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ repoServer:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+
authservice:
- enabled: true
+ enabled: false
chains:
minimal:
callback_uri: "https://minimal.bigbang.dev"
+ values:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 100Mi
+ limits: {}
+
gitlab:
- enabled: true
+ enabled: false
sso:
enabled: false
+ values:
+ global:
+ rails:
+ bootstrap:
+ enabled: false
+ gitlab-runner:
+ resources:
+ requests:
+ cpu: 10m
+ limits: {}
+ gitlab:
+ webservice:
+ minReplicas: 1
+ maxReplicas: 1
+ sidekiq:
+ minReplicas: 1
+ maxReplicas: 1
+ gitlab-shell:
+ minReplicas: 1
+ maxReplicas: 1
+ gitaly:
+ resources:
+ requests:
+ cpu: 50m
+ limits: {}
+ shared-secrets:
+ resources:
+ requests:
+ cpu: 10m
+ limits: {}
+ migrations:
+ resources:
+ requests:
+ cpu: 10m
+ limits: {}
+ task-runner:
+ resources:
+ requests:
+ cpu: 10m
+ limits: {}
+ registry:
+ hpa:
+ minReplicas: 1
+ maxReplicas: 1
+ postgresql:
+ metrics:
+ resources:
+ requests:
+ cpu: 10m
+ limits: {}
+ minio:
+ resources:
+ requests:
+ cpu: 50m
+ limits: {}
+
gitlabRunner:
- enabled: true
+ enabled: false
+ values:
+ resources:
+ requests:
+ memory: 64Mi
+ cpu: 50m
+ limits: {}
+
anchore:
- enabled: true
+ enabled: false
+ values:
+ postgresql:
+ persistence:
+ size: 20Gi
+ anchoreAnalyzer:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreApi:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreCatalog:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchorePolicyEngine:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreSimpleQueue:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreEnterpriseFeeds:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreEnterpriseRbac:
+ authResource:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ managerResources:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreEnterpriseReports:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreEnterpriseNotifications:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ anchoreEntperpiseUi:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+
sonarqube:
- enabled: true
+ enabled: false
+ values:
+ plugins:
+ install: []
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ persistence:
+ enabled: false
+ size: 5Gi
+ postgresql:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 200Mi
+ limits: {}
+ tests:
+ enabled: false
+
minioOperator:
- enabled: true
+ enabled: false
+
minio:
- enabled: true
+ enabled: false
+ values:
+ zones:
+ servers: 1
+ volumesPerServer: 1
+
mattermostoperator:
- enabled: true
+ enabled: false
+
mattermost:
- enabled: true
+ enabled: false
+ values:
+ replicaCount: 1
+ resources:
+ requests:
+ cpu: 100m
+ memory: 128Mi
+ limits: {}
+ minio:
+ zones:
+ servers: 1
--
GitLab