Merge branch 'master' into 833-gitlab-object-storage-use_iam_role

chart/templates/gitlab/namespace.yaml

@@ -6,6 +6,6 @@ metadata:
     app.kubernetes.io/name: gitlab
     app.kubernetes.io/component: "developer-tools"
     {{- include "commonLabels" . | nindent 4}}
-    istio-injection: disabled
+    istio-injection: {{ dig "istio" "injection" "enabled" .Values.addons.gitlab }}
   name: gitlab
 {{- end }}

chart/templates/gitlab/values.yaml

@@ -46,6 +46,12 @@ registry:
   {{- end }}
 {{- end }}
 
+{{- if .Values.istio.enabled }}
+shared-secrets:
+  annotations:
+    sidecar.istio.io/inject: "false"
+{{- end }}
+
 gitlab:
   {{- if .Values.addons.gitlab.objectStorage.endpoint }}
   task-runner:
@@ -70,7 +76,18 @@ gitlab:
       iam.amazonaws.com/role: {{ .Values.addons.gitlab.objectStorage.iamProfile }}
   {{- end }}
 
+  {{- if .Values.istio.enabled }}
+  migrations:
+    annotations:
+      sidecar.istio.io/inject: "false"
+  {{- end }}
+  
 global:
+
+  # added to help with Gitlab sub-chart configuration
+  istio:
+    enabled: {{ .Values.istio.enabled }}
+
   hosts:
     domain: {{ $domainName }}
 

chart/values.yaml

@@ -459,7 +459,7 @@ twistlock:
   git:
     repo: https://repo1.dso.mil/platform-one/big-bang/apps/security-tools/twistlock.git
     path: "./chart"
-    tag: "0.0.9-bb.1"
+    tag: "0.0.10-bb.0"
 
   # -- Flux reconciliation overrides specifically for the Twistlock Package
   flux: {}
@@ -591,7 +591,7 @@ addons:
     postRenderers: []
 
   gitlab:
-    # -- Toggle deployment of Gitlab
+    # -- Toggle deployment of Gitlab 
     enabled: false
 
     hostnames:
@@ -683,7 +683,7 @@ addons:
     git:
       repo: https://repo1.dso.mil/platform-one/big-bang/apps/developer-tools/gitlab-runner.git
       path: "./chart"
-      tag: "0.33.1-bb.3"
+      tag: "0.33.1-bb.4"
 
     # -- Flux reconciliation overrides specifically for the Gitlab Runner Package
     flux: {}

docs/developer/package-integration/package-integration-database.md

 # Big Bang Package: Database Integration
 
-If the package you are integrating connects to a database or cache server, you will need to follow the instructions below to integrate this feature into Big Bang
+If the package you are integrating connects to a database, you will need to follow the instructions below to integrate this feature into Big Bang.
 
 ## Prerequisites
 
-TBD
+- Existing database
 
 ## Integration
 
+There are currently 2 typical ways in bigbang that packages connect to a database.
+
+1. Package charts accept values for host, user, pass, etc and the chart makes the necessary secret, configmap etc.
+
+2. Package chart accepts a secret name where all the DB connection info is defined. In these cases we make the secret in the BB chart.
+
+Both ways will first require the following step:
+
+Add database values for the package in bigbang/chart/values.yaml
+
+  Note: Names of key/values may differ based on the application being integrated. Please refer to package chart values to ensure key/values coincide and application documentation for additional information on connecting to a database.
+
+```yml
+<package>
+  database:
+    # -- Hostname of a pre-existing PostgreSQL database to use.
+    host: ""
+    # -- Port of a pre-existing PostgreSQL database to use.
+    port: ""
+    # -- Database name to connect to on host.
+    database: ""
+    # -- Username to connect as to external database, the user must have all privileges on the database.
+    username: ""
+    # -- Database password for the username used to connect to the existing database.
+    password: ""
+```
+Example: [Anchore](https://repo1.dso.mil/platform-one/big-bang/bigbang/-/blob/10d43bea9351b91dfc6f14d3b0c2b2a60fe60c6a/chart/values.yaml#L882)
+
+**Next details the first way packages connect to a pre-existing database.**
+
+1. Package charts accept values for host, user, pass, etc and the chart makes the necessary secret, configmap etc...
+
+- add a conditional statement to `bigbang/chart/templates/<package>/values` that will check if the database values exist and creates the necessary postgresql values.
+
+  If database values are present, then the internal database is disabled by setting `enabled: false` and the server, database, username, and port values are set.
+
+  If database values are NOT present then the internal database is enabled and default values declared in the package are used.
+
+```yml
+# External Postgres config
+{{- with .Values.<package>.database }}
+postgresql:
+  {{- if and .host .username .password .database .port }}
+  # Use external database
+  enabled: false
+  postgresqlServer: {{ .host }}
+  postgresqlDatabase: {{ .database }}
+  postgresqlUsername: {{ .username }}
+  service:
+    port: {{ .port }}
+  {{- else }}
+  # Use internal database, defaults are fine
+  enabled: true
+  {{- end }}
+{{- end }}
+```
+Example: [Anchore](https://repo1.dso.mil/platform-one/big-bang/bigbang/-/blob/10d43bea9351b91dfc6f14d3b0c2b2a60fe60c6a/chart/templates/anchore/values.yaml#L49)
+
+**The alternative way packages connect to a pre-existing database is detailed below.**
+
+2. Package chart accepts a secret name where all the DB connection info is defined. In these cases we make the secret in the BB chart..
+
+- add conditional statement in `chart/templates/<package>/values.yaml` to add values for database secret, if database values exist. Otherwise the internal database is deployed.
+```yml
+{{- with .Values.addons.<package>.database }}
+{{- if and .username .password .host .port .database }}
+database:
+  secret: "<package>-database-secret"
+{{- else }}
+postgresql:
+  image:
+    pullSecrets:
+      - private-registry
+  install: true
+{{- end }}
+{{- end }}
+```
+
+Example: [Mattermost](https://repo1.dso.mil/platform-one/big-bang/bigbang/-/blob/10d43bea9351b91dfc6f14d3b0c2b2a60fe60c6a/chart/templates/mattermost/mattermost/values.yaml#L49)
+
+
+- create manifest that uses database values to create the database secret referenced above
+
+```yml
+{{- if .Values.addons.<package>.enabled }}
+{{- with .Values.addons.<package>.database }}
+{{- if and .username .password .host .port .database }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: <package>-database-secret
+  namespace: <package>
+  labels:
+    {{- include "commonLabels" $ | nindent 4}}
+stringData:
+  DB_CONNECTION_CHECK_URL: "postgres://{{ .username }}:{{ .password }}@{{ .host }}:{{ .port }}/{{ .database }}?connect_timeout=10&sslmode={{ .ssl_mode | default "disable" }}"
+  DB_CONNECTION_STRING: "postgres://{{ .username }}:{{ .password }}@{{ .host }}:{{ .port }}/{{ .database }}?connect_timeout=10&sslmode={{ .ssl_mode | default "disable" }}"
+{{- end }}
+{{- end }}
+{{- end }}
+```
+
+Example: [Mattermost](https://repo1.dso.mil/platform-one/big-bang/bigbang/-/blob/10d43bea9351b91dfc6f14d3b0c2b2a60fe60c6a/chart/templates/mattermost/mattermost/secret-database.yaml)
+
 ## Validation
+
+For validating connection to the external database in your environment or testing in CI pipeline you will need to add the database specific values to your overrides file or `tests/ci/k3d/values.yaml` respectively.
+
+Mattermost Example:
+
+```yml
+addons:
+  mattermost:
+    enabled: true
+    database:
+      host: "mm-postgres.bigbang.dev"
+      port: "5432"
+      username: "admin"
+      password: "Pa55w0rd"
+      database: "db1
+```

docs/developer/package-integration/package-integration-documentation.md

@@ -2,4 +2,8 @@
 
 Big Bang requires some additional documentation for supported packages to help user's understand how it interacts with other components.  The following are documents that should be created or updated for integration into Big Bang:
 
-- TBD
+- Package Architecture: See [Big Bang's Architecture instructions](../../charter/packages/ref-package/Architecture.md).  Examples are included in [charter/packages](../../charter/packages)
+- [Big Bang Packages](../../charter/BigBangPackages.md)
+- [Default Credentials](../guides/using_bigbang/default_credentials.md)
+- [Licensing](../understanding_bigbang/licensing_expectations.md)
+- [Minimum Hardware Requirements](../guides/prerequisites/minimum_hardware_requirements.md)

docs/developer/package-integration/package-integration-pipeline.md

@@ -2,11 +2,80 @@
 
 Big Bang contains a uses a continuous deployment tool to deploy packages using Helm charts sourced from Git.  This document will cover how to integrate a Helm chart from a mission application or other package into the pattern Big Bang requires.  Once complete, you will be able to deploy your package with Big Bang.
 
-
 ## Prerequisites
 
-TBD
+- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+- [Docker CLI](https://docs.docker.com/get-docker/)
+- [Big Bang package project containing your Helm chart](./package-integration-upstream.md)
+   > You will need to have the Container Registry enabled.This can be requested from the Big Bang team.
+
+> Throughout this document, we will be setting up an application called `podinfo` as a demonstration.
+
+## Package Pipeline
+
+Pipelines provide rapid feedback to changes in our Helm chart as we develop and should be put in place as early as possible.  Big Bang has a [generic pipeline](https://repo1.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/-/blob/master/templates/package-tests.yml) that we can reuse for packages.
+
+1. The pipeline **requires** that all images are stored in either Iron Bank (`registry1.dso.mil`) or Repo1 (`registry.dso.mil`).  In some cases, you may be able to substitute images already in Iron Bank for the ones in the Helm chart.  For example, images for `curl`, `kubectl` or `jq` can use `registry1.dso.mil/ironbank/big-bang/base`.  If you have not already submitted your containers to Iron Bank, [start the process](https://repo1.dso.mil/dsop/dccscr/-/blob/master/README.md).  While you are working your way to Iron Bank approval, you can temporarily put the images in `registry.dso.mil` for development by doing the following:
+
+   > Check if the Container Registry is on by navigating to `https://repo1.dso.mil/platform-one/big-bang/apps/sandbox/<your project>/container_registry`.  If you get a 404 error, you need to request a Maintainer turn this feature on in your project via Settings > General > Visibility > Container Registry.
+
+   ```shell
+   # Image Info
+   export IMGSRC_REPO=docker.io
+   export IMGSRC_PROJ=stefanprodan
+   export IMGDST_REPO=registry.dso.mil
+   export IMGDST_PROJ=platform-one/big-bang/apps/sandbox/podinfo
+   export IMGNAME=podinfo
+   export IMGTAG=6.0.0
+
+   # Pull image locally
+   docker pull $IMGSRC_REPO/$IMGSRC_PROJ/$IMGNAME:$IMGTAG
+
+   # Retag image
+   docker tag $IMGSRC_REPO/$IMGSRC_PROJ/$IMGNAME:$IMGTAG $IMGDST_REPO/$IMGDST_PROJ/$IMGNAME:$IMGTAG
+
+   # Login in docker registry
+   docker login $IMGDST_REPO
+
+   # Push to registry
+   docker push $IMGDST_REPO/$IMGDST_PROJ/$IMGNAME:$IMGTAG
+   ```
 
-## Integration
+1. Update `chart/values.yaml` with either the `registry1.dso.mil` or `registry.dso.mil` for images.  For example:
 
-## Validation
+   ```yaml
+   image:
+     repository: registry.dso.mil/platform-one/big-bang/apps/sandbox/podinfo/podinfo
+     tag: 6.0.0
+   ```
+
+1. Add the following to `.gitlab-ci.yml` to call the pipeline.
+
+   ```yaml
+   include:
+     - project: 'platform-one/big-bang/pipeline-templates/pipeline-templates'
+       ref: master
+       file: '/templates/package-tests.yml'
+   ```
+
+1. Add overlay values for testing into `tests/test-values.yaml`.  This will be where you add values needed for running in the pipeline.  For now it can be a blank, placeholder.
+
+1. Commit the changes
+
+   ```shell
+   git add -A
+   git commit -m "feat: package pipeline"
+   git push
+   ```
+
+1. Big Bang requires a Merge Request to run the pipeline.  Open a MR to merge your branch into the main branch.
+
+   > You will need to add `SKIP UPDATE CHECK` and `SKIP UPGRADE` into the title of the first MR or it will fail.  Until you have a baseline Helm chart and CHANGELOG in place, these stages need to be skipped.
+
+1. The pipeline will install the package, run any Helm tests (`chart/tests`), and run any custom tests (`tests`).
+
+1. Troubleshoot and fix any failures from the pipeline.
+
+## Big Bang Pipeline
+
+TBD

docs/guides/using_bigbang/image_pull_policy.md

+# ImagePullPolicy at Big Bang Level
+
+Big Bang is currently working to standardize the adoption of a global image pull policy so that customers can set a single value and have it passed to all packages. This work is not yet complete, but should allow customers easier control over their global pull policy.
+
+In the meantime we have begun to document the package overrides required in preparation for this change.
+
+# ImagePullPolicy per Package
+
+| Package | Default | Value Override |
+|---|---|---|
+| Istio Controlplane | None | <pre lang="yaml">istio:<br>  values:<br>    imagePullPolicy: IfNotPresent</pre> |
+| Istio Operator | IfNotPresent | No override available |
+| Jaeger | Always | <pre lang="yaml">jaeger:<br>  values:<br>    image:<br>      pullPolicy: IfNotPresent</pre> |
+| Kiali | IfNotPresent | <pre lang="yaml">kiali:<br>  values:<br>    image:<br>      pullPolicy: IfNotPresent<br></pre><br><pre lang="yaml">kiali:<br>  values:<br>    cr:<br>      spec:<br>        deployment:<br>          image_pull_policy: IfNotPresent</pre> |
+| Cluster Auditor | Always | <pre lang="yaml">clusterAuditor:<br>  values:<br>    image:<br>      imagePullPolicy: IfNotPresent</pre> |
+| OPA Gatekeeper | IfNotPresent | <pre lang="yaml">gatekeeper:<br>  values:<br>    postInstall:<br>      labelNamespace:<br>        image:<br>          pullPolicy: IfNotPresent<br>    image:<br>      pullPolicy: IfNotPresent</pre> |
+| Elasticsearch / Kibana | None | No override available |
+| ECK Operator | IfNotPresent | <pre lang="yaml">eckoperator:<br>  values:<br>    image:<br>      pullPolicy: IfNotPresent</pre> |
+| Fluentbit | Always | <pre lang="yaml">fluentbit:<br>  values:<br>    image:<br>      pullPolicy: IfNotPresent</pre> |
+| Monitoring | Varies | To be documented |
+| Twistlock | None | No override available |
+| ArgoCD | IfNotPresent | To be documented |
+| Authservice | IfNotPresent | <pre lang="yaml">addons:<br>  authservice:<br>    values:<br>      image:<br>        pullPolicy: IfNotPresent</pre> |
+| MinIO Operator | To be documented | To be documented |
+| MinIO | To be documented | To be documented |
+| Gitlab | To be documented | To be documented |
+| Gitlab Runners | To be documented | To be documented |
+| Nexus | To be documented | To be documented |
+| Sonarqube | To be documented | To be documented |
+| Anchore | To be documented | To be documented |
+| Mattermost Operator | To be documented | To be documented |
+| Mattermost | To be documented | To be documented |
+| Velero | To be documented | To be documented |
+| Keycloak | To be documented | To be documented |

scripts/package/synker.yaml

@@ -28,7 +28,7 @@ source:
     # Include registry image
     - registry:2
 
-    - registry1.dso.mil/ironbank/twistlock/defender/defender:20.12.531
+    - registry1.dso.mil/ironbank/twistlock/defender/defender:21.08.520
     - registry1.dso.mil/ironbank/anchore/enterprise/enterprise:3.1.2
     - registry1.dso.mil/ironbank/anchore/enterpriseui/enterpriseui:3.1.1
     - registry1.dso.mil/ironbank/big-bang/base:8.4