Merge branch '258-dynamic-inclusion-of-addons-in-ci' into 'master'

Resolve "dynamic inclusion of addons in ci"

Summary

* ci: change k3s as a sidecar to k3d using (tls enabled) dockerd as a sidecar using ci created `bridge` networks with properly tuned `mtu`
* ci: add logic to enable addons based on MR labels (name must be case exact match to the helm value block)
* infra: bump `ci-optimized` rke2 node pool to `c5d.12xlarge`, resize to `{ min: 1, desired: 1, max: 5 }`
* infra: introduce cluster-autoscaler to `ci-optimized` rke2 node pool and actually test it, new nodes join in ~2m but to be safe, set ci timeout to 5m
* infra: use instance storage for image ci image layers (reduces i/o bottlenecks by ~80%)
* dogfood cluster: embed `dind` service into the dogfood gitlab ci deployment with fixes for `/etc/machine-id`, `dind tls`, and runtime dockerd config (see [here](https://repo1.dso.mil/platform-one/big-bang/customers/bigbang/-/blob/master/apps/gitlab-runners/prod/bigbang-gitlab-runner-privileged.yaml#L39))
* dogfood cluster: deploy gitlab ci with `default` dns (instead of the default `ClusterFirst`) dns to properly resolve dind-k3d dns


Closes #258

See merge request platform-one/big-bang/bigbang!328

.gitlab-ci.yml

@@ -101,17 +101,20 @@ pre vars:
 clean install:
   stage: smoke tests
   extends:
-    - .k3s-ci
+    - .k3d-ci
   variables:
     CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}"
   rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master"'
+      when: never
     - *chart_changes
   script:
     - *deploy_bigbang
     - *test_bigbang
 
     # Fetch list of all images ran
-    - curl -sfL k3s:8081/images | jq -r '.images[].repoTags[0]' > images.txt
+    - cid=$(docker ps -aqf "name=k3d-${CI_JOB_ID}-server-0")
+    - docker exec $cid crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)' | tee images.txt
   artifacts:
     paths:
       - images.txt
@@ -122,7 +125,7 @@ upgrade:
   dependencies:
     - pre vars
   extends:
-    - .k3s-ci
+    - .k3d-ci
   rules:
     # skip job when MR title starts with 'Breaking Change'
     - if: '$CI_MERGE_REQUEST_TITLE =~ /^Breaking Change/'

.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml

 .k8s-util:
-  image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k8s-ci:v1.20.2
+  image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k8s-ci:v1.20.4-bb.2
 
 .dind-runner:
   tags:
@@ -7,20 +7,73 @@
     - privileged
     - dogfood
 
+#
+# In cluster k3s using K3D with the docker daemon as a sidecar
+#
+#   This will connect to a remote docker daemon over tls tcp (defined at installation of gitlab runners) and create
+#   a k3d cluster in a custom built docker bridge network.
+#
+.k3d-ci:
+  extends:
+    - .k8s-util
+    - .dind-runner
+  #  services:
+  #      # Added in through gitlab ci configuration, left in incase some poor soul needs to come debug this later
+  #    - name: docker:20.10.5-dind
+  variables:
+    DOCKER_HOST: tcp://localhost:2376
+    DOCKER_TLS_CERTDIR: "/certs"
+    DOCKER_TLS_VERIFY: 1
+    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
+    DOCKER_DRIVER: overlay2
+  before_script:
+    - docker network create ${CI_JOB_ID} --driver=bridge -o "com.docker.network.driver.mtu"="1450"
+    - k3d cluster create ${CI_JOB_ID} --config tests/ci/k3d/config.yaml --network ${CI_JOB_ID}
+    - until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done
+    - kubectl get all -A
+  after_script:
+    - kubectl get all -A
+    - kubectl get gitrepository,helmrelease,kustomizations -A
+    - k3d cluster delete ${CI_JOB_ID}
+    - docker network rm ${CI_JOB_ID}
+
+#
+# In cluster k3s using k3s as a sidecar
+#
+#   This will spin up k3s as a gitlab ci sidecar
+#
 .k3s-ci:
   extends:
     - .k8s-util
     - .dind-runner
   services:
-    - name: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k3s-ci:v1.20.2-k3s1-bb.0
+    - name: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k3s-ci:v1.20.4-k3s1-bb.0
       alias: k3s
       command:
         - "server"
         - "--tls-san=k3s"
-        - "--disable=metrics-server"
         - "--disable=traefik"
   before_script:
     - mkdir -p ~/.kube
     - curl -sfL k3s:8081?service=k3s > ~/.kube/config
     - kubectl version
     - kubectl cluster-info
+  after_script:
+    - kubectl get all -A
+    - kubectl get gitrepository,helmrelease,kustomizations -A
+
+#
+# In cluster k3d using K3D with a dind container
+#
+#   This will spin up a k3d cluster using a dind container as the base container
+#
+.k3d-dind-ci:
+  extends:
+    - .dind-runner
+  image:
+    name: rancher/k3d:v4.3.0-dind
+  before_script:
+    - nohup dockerd-entrypoint.sh &
+    - until docker ps 2>&1 > /dev/null; do sleep 1s; done
+    - k3d cluster create --config tests/ci/k3d/config.yaml
+    - until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done

chart/templates/minio/minio-operator/gitrepository.yaml

-{{- if and (not .Values.offline) .Values.addons.minioOperator.enabled }}
+{{- if and (not .Values.offline) (or .Values.addons.minioOperator.enabled .Values.addons.minio.enabled) }}
 apiVersion: source.toolkit.fluxcd.io/v1beta1
 kind: GitRepository
 metadata:

chart/templates/minio/minio-operator/minio-operator-helmrelease.yaml

-{{- if .Values.addons.minioOperator.enabled }}
+{{- if or .Values.addons.minioOperator.enabled .Values.addons.minio.enabled }}
 apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:

chart/templates/minio/minio-operator/namespace.yaml

-{{- if .Values.addons.minioOperator.enabled }}
+{{- if or .Values.addons.minioOperator.enabled .Values.addons.minio.enabled }}
 apiVersion: v1
 kind: Namespace
 metadata:

scripts/deploy/01_deploy_bigbang.sh

@@ -2,6 +2,8 @@
 
 set -ex
 
+CI_VALUES_FILE="tests/ci/k3d/values.yaml"
+
 # Deploy flux and wait for it to be ready
 echo "Installing Flux"
 flux --version
@@ -26,12 +28,22 @@ kubectl apply -f ./scripts/deploy/flux.yaml
 # wait for flux
 flux check
 
+IFS=","
+for package in $CI_MERGE_REQUEST_LABELS; do
+  if [ "$(yq e ".addons.${package}.enabled" $CI_VALUES_FILE 2>/dev/null)" == "false" ]; then
+    echo "Identified \"$package\" from labels"
+    yq e ".addons.${package}.enabled = "true"" $CI_VALUES_FILE > tmpfile && mv tmpfile $CI_VALUES_FILE
+  fi
+done
+
 # deploy BigBang using dev sized scaling
-echo "Installing BigBang"
+echo "Installing BigBang with the following configurations:"
+cat $CI_VALUES_FILE
+
 helm upgrade -i bigbang chart -n bigbang --create-namespace \
 --set registryCredentials[0].username='robot$bigbang' --set registryCredentials[0].password=${REGISTRY1_PASSWORD} \
 --set registryCredentials[0].registry=registry1.dso.mil \
--f tests/ci/k3d/values.yaml
+-f ${CI_VALUES_FILE}
 
 # apply secrets kustomization pointing to current branch
 echo "Deploying secrets from the ${CI_COMMIT_REF_NAME} branch"

scripts/deploy/02_wait_for_helmreleases.sh

 #!/usr/bin/env bash
 
-set -ex
+set -e
 
 ## This is an array to instantiate the order of wait conditions
 ORDERED_HELMRELEASES="gatekeeper istio-operator istio monitoring eck-operator ek fluent-bit twistlock cluster-auditor authservice argocd gitlab haproxy-sso gitlab-runner minio-operator minio anchore sonarqube mattermost-operator mattermost"
 
-## This the actual deployed helmrelease objects in the cluster
+## This is the actual deployed helmrelease objects in the cluster
 DEPLOYED_HELMRELEASES=$(kubectl get hr --no-headers -n bigbang | awk '{ print $1}')
 
+printf "Identified the following deployed helmreleases:\n%s" "${DEPLOYED_HELMRELEASES}"
+
 ## Function to test an array contains an element
 ## Args:
 ## $1: array to search

scripts/package/synker.yaml

@@ -33,9 +33,6 @@ source:
     - registry1.dso.mil/ironbank/anchore/enterprise/enterprise:2.4.1
     - registry1.dso.mil/ironbank/anchore/enterpriseui/enterpriseui:2.4.1
 
-    # not capturing this image from CI deployment. Adding it explicitly
-    - registry1.dso.mil/ironbank/opensource/fluent/fluent-bit:1.6.6
-
     # Images used by pipeline-templates
     # NOTE: These are __not__ fully inclusive yet, see this issue for updates: https://repo1.dso.mil/platform-one/big-bang/bigbang/-/issues/145
     - registry.il2.dso.mil/platform-one/devops/pipeline-templates/pipeline-job/sonar-scanner-dotnet31:4.10

tests/ci/k3d/config.yaml

+apiVersion: k3d.io/v1alpha2
+kind: Simple
+name: ci
+servers: 1
+options:
+  k3s:
+    extraServerArgs:
+      - --disable=traefik
+  k3d:
+    wait: true
+volumes:
+  - volume: /etc/machine-id:/etc/machine-id
+    nodeFilters:
+      - server[*]
+      - agent[*]
+ports:
+  - port: 80:80
+    nodeFilters:
+      - loadbalancer
+  - port: 443:443
+    nodeFilters:
+      - loadbalancer

tests/ci/k3d/values.yaml

@@ -14,17 +14,17 @@ logging:
         persistence:
           size: 5Gi
         resources:
-          limits:
-            cpu: 1
-            memory: 2Gi
+          requests:
+            cpu: .5
+          limits: {}
       data:
         count: 1
         persistence:
           size: 5Gi
         resources:
-          limits:
-            cpu: 1
-            memory: 2Gi
+          requests:
+            cpu: .5
+          limits: {}
 
 istio:
   enabled: true
@@ -36,14 +36,64 @@ istio:
 
 clusterAuditor:
   enabled: true
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: .5Gi
+      limits: {}
 
 monitoring:
   enabled: true
+  values:
+    alertmanager:
+      alertmanagerSpec:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+    prometheusOperator:
+      resources:
+        requests:
+          cpu: 250m
+          memory: 400Mi
+        limits: {}
+    prometheus:
+      prometheusSpec:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+    grafana:
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits: {}
+    kubeStateMetrics:
+      resources:
+        requests:
+          cpu: 10m
+          memory: 32Mi
+        limits: {}
+    nodeExporter:
+      resources:
+        requests:
+          cpu: 100m
+          memory: 30Mi
+        limits: {}
 
 gatekeeper:
   enabled: true
   values:
     replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits: {}
 
 twistlock:
   enabled: true
@@ -55,27 +105,243 @@ twistlock:
 # Explicitly enable all addons for CI
 addons:
   argocd:
-    enabled: true
+    enabled: false
+    values:
+      controller:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      dex:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      redis:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      server:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      repoServer:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+
   authservice:
-    enabled: true
+    enabled: false
     chains:
       minimal:
         callback_uri: "https://minimal.bigbang.dev"
+    values:
+      resources:
+        requests:
+          cpu: 100m
+          memory: 100Mi
+        limits: {}
+
   gitlab:
-    enabled: true
+    enabled: false
     sso:
       enabled: false
+    values:
+      global:
+        rails:
+          bootstrap:
+            enabled: false
+      gitlab-runner:
+        resources:
+          requests:
+            cpu: 10m
+          limits: {}
+      gitlab:
+        webservice:
+          minReplicas: 1
+          maxReplicas: 1
+        sidekiq:
+          minReplicas: 1
+          maxReplicas: 1
+        gitlab-shell:
+          minReplicas: 1
+          maxReplicas: 1
+        gitaly:
+          resources:
+            requests:
+              cpu: 50m
+            limits: {}
+        shared-secrets:
+          resources:
+            requests:
+              cpu: 10m
+            limits: {}
+        migrations:
+          resources:
+            requests:
+              cpu: 10m
+            limits: {}
+        task-runner:
+          resources:
+            requests:
+              cpu: 10m
+            limits: {}
+      registry:
+        hpa:
+          minReplicas: 1
+          maxReplicas: 1
+      postgresql:
+        metrics:
+          resources:
+            requests:
+              cpu: 10m
+            limits: {}
+      minio:
+        resources:
+          requests:
+            cpu: 50m
+          limits: {}
+
   gitlabRunner:
-    enabled: true
+    enabled: false
+    values:
+      resources:
+        requests:
+          memory: 64Mi
+          cpu: 50m
+        limits: {}
+
   anchore:
-    enabled: true
+    enabled: false
+    values:
+      postgresql:
+        persistence:
+          size: 20Gi
+      anchoreAnalyzer:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreApi:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreCatalog:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchorePolicyEngine:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreSimpleQueue:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreEnterpriseFeeds:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreEnterpriseRbac:
+        authResource:
+          resources:
+            requests:
+              cpu: 100m
+              memory: 200Mi
+            limits: {}
+        managerResources:
+          resources:
+            requests:
+              cpu: 100m
+              memory: 200Mi
+            limits: {}
+      anchoreEnterpriseReports:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreEnterpriseNotifications:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      anchoreEntperpiseUi:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+
   sonarqube:
-    enabled: true
+    enabled: false
+    values:
+      plugins:
+        install: []
+      resources:
+        requests:
+          cpu: 100m
+          memory: 200Mi
+        limits: {}
+      persistence:
+        enabled: false
+        size: 5Gi
+      postgresql:
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+          limits: {}
+      tests:
+        enabled: false
+
   minioOperator:
-    enabled: true
+    enabled: false
+
   minio:
-    enabled: true
+    enabled: false
+    values:
+      zones:
+        servers: 1
+      volumesPerServer: 1
+
   mattermostoperator:
-    enabled: true
+    enabled: false
+
   mattermost:
-    enabled: true
+    enabled: false
+    values:
+      replicaCount: 1
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits: {}
+      minio:
+        zones:
+          servers: 1