UNCLASSIFIED - NO CUI

Skip to content
Snippets Groups Projects
Commit d75300d3 authored by Thomas Runyon's avatar Thomas Runyon
Browse files

Merge branch 'master' into feature/remove-ns-finalizer

parents cd66d866 087284f1
No related branches found
No related tags found
1 merge request!80Add hack script for removing NS finalizer for stuck namespaces
Pipeline #100424 passed
Showing
with 334 additions and 110 deletions
......@@ -23,9 +23,43 @@ stages:
- cluster down
- network down
#-----------------------------------------------------------------------------------------------------------------------
# Pre Stage Jobs
#
pre vars:
image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3
stage: .pre
artifacts:
reports:
dotenv: variables.env
script:
# obtain MR and master versions
- CHART_MR_VERSION=$(sed -n -e 's/^version. //p' chart/Chart.yaml)
- git fetch && git checkout ${CI_DEFAULT_BRANCH}
- CHART_MA_VERSION=$(sed -n -e 's/^version. //p' chart/Chart.yaml)
- git fetch && git checkout ${CI_COMMIT_REF_NAME}
- echo "CHART_MR_VERSION=$CHART_MR_VERSION" >> variables.env
- echo "CHART_MA_VERSION=$CHART_MA_VERSION" >> variables.env
# obtain semver differences (subtract master version from mr verison)
- chmod +x ./scripts/semver_diff.sh
- CHART_VERSION_DIFF=$(./scripts/semver_diff.sh $CHART_MR_VERSION $CHART_MA_VERSION)
- IFS=. DIFF_ARR=(${CHART_VERSION_DIFF##*-})
- echo "CHART_VERSION_DIFF=$CHART_VERSION_DIFF" >> variables.env
# detect breaking change (first two version sections in semver diff)
- CHART_BREAKING_CHANGE="false"
- if (( ${DIFF_ARR[0]} > 0 )); then CHART_BREAKING_CHANGE="true"; fi
- if (( ${DIFF_ARR[1]} > 0 )); then CHART_BREAKING_CHANGE="true"; fi
# store variables
- echo "CHART_BREAKING_CHANGE=$CHART_BREAKING_CHANGE" >> variables.env
- cat variables.env
#-----------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------
# Smoke Tests
#
.bigbang:
image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/k3d-builder:0.0.1
after_script:
......@@ -33,7 +67,7 @@ stages:
- kubectl get helmrelease -A
.deploy_bigbang: &deploy_bigbang
- for script in ./scripts/*.sh; do chmod +x $script && $script; done
- for script in ./scripts/deploy/*.sh; do chmod +x $script && $script; done
.test_bigbang: &test_bigbang
- for test in ./tests/bash/*.sh; do chmod +x $test && $test; done
......@@ -52,25 +86,26 @@ clean install:
script:
- *deploy_bigbang
- *test_bigbang
after_script:
- kubectl get all -A
- kubectl get helmrelease -A
upgrade:
stage: smoke tests
needs:
- job: pre vars
artifacts: true
extends:
- .k3d
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE !~ /^Breaking Change/'
variables:
CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}"
script:
- if $CHART_BREAKING_CHANGE; then echo "Breaking change detected by chart version difference, skipping job"; exit 0; fi
- echo "Install Big Bang from ${CI_DEFAULT_BRANCH}"
- git fetch && git checkout ${CI_DEFAULT_BRANCH}
- *deploy_bigbang
- *test_bigbang
- echo "Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
- git reset --hard && git clean -fd
- git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
- *deploy_bigbang
- *test_bigbang
......@@ -194,14 +229,11 @@ aws/rke2/bigbang test:
- mkdir -p ~/.kube
- cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config
script:
- sleep 240
- kubectl get all -A
- kubectl get vs -A
- kubectl describe gateway main -n istio-system
- kubectl get gateway main -n istio-system
- kubectl get gateway main -n istio-system -o yaml
- kubectl get service -n istio-system
- kubectl get service istio-ingressgateway -n istio-system -o yaml
## Move this yum install to the dockerfile for the builder
## putting it here now for a quick way to install dig
- yum install bind-utils -y
- chmod +x scripts/hosts.sh && ./scripts/hosts.sh
- *test_bigbang
# Uninstall BigBang on RKE2 cluster on AWS
aws/rke2/bigbang down:
......@@ -233,4 +265,4 @@ aws/rke2/cluster down:
- job: fetch umbrella templates
artifacts: true
- job: aws/rke2/bigbang down
#-----------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------
\ No newline at end of file
# Contributing to Big Bang
Thanks for taking the time to contribute to BigBang!
## Developers Guide
Big Bang is designed in such a way as to be as easily deployed locally as it is in production. In fact, most contributions begin locally.
Follow the steps below to get a complete local instantiation of Big Bang up locally using [k3d](https://k3d.io/).
### Local Development Quickstart
#### Local `k3d` cluster
```bash
# Create a local k3d cluster with the appropriate port forwards
k3d cluster create --k3s-server-arg "--disable=traefik" --k3s-server-arg "--disable=metrics-server" -p 80:80@loadbalancer -p 443:443@loadbalancer
```
#### Deploying Big Bang
The [Big Bang environment template](https://repo1.dsop.io/platform-one/big-bang/customers/bigbang/-/tree/master/bigbang) should be copied locally to start your deployment. Follow the instructions in the [template's readme](https://repo1.dsop.io/platform-one/big-bang/customers/bigbang/-/tree/master/bigbang/README.md) and in the [Big Bang docs](./docs) for configuration.
```bash
# Deploy official, hardened fluxv2 from Iron Bank
# Alternatives:
# - Install non-hardened image: `flux install`
# - Install unofficial images from Big Bang repo: `flux install --registry registry.dsop.io/platform-one/big-bang/apps/sandbox/fluxv2`
hack/flux-install.sh
# Apply the development sops secret
# Modify sops-create.sh if you use your own SOPS secret
hack/sops-create.sh
# The above command creates the 'bigbang' namespace. If you skip it, create your own
kubectl create namespace bigbang
# Apply the necessary dev secrets (e.g. pull secrets, certs)
# The .yaml files used for this are from the Big Bang environment template
sops -d bigbang/base/secrets.enc.yaml | kubectl apply -n bigbang -f -
sops -d bigbang/dev/secrets.enc.yaml | kubectl apply -n bigbang -f -
# Apply a local version of the umbrella chart
# NOTE: This is the alternative to deploying a HelmRelease and having flux manage it, we use a local copy to avoid having to commit every change
# NOTE: Use yq to parse the kustomize values patch and pipe it to the helm values
# The .yaml files used for yq are from the Big Bang environment template
# NOTE: Flux will take care of the reconcilitation and retry loops for us, it is normal to see resources fail to deploy a few times on boot
yq m bigbang/prod/configmap.yaml bigbang/base/configmap.yaml | helm helm upgrade -i bigbang chart -n bigbang --create-namespace -f -
# After making changes to the umbrella chart or values, you can update the chart idempotently
yq m bigbang/prod/configmap.yaml bigbang/base/configmap.yaml | helm helm upgrade -i bigbang chart -n bigbang --create-namespace -f -
# A convenience development script is provided to force fluxv2 to reconcile all helmreleases within the cluster
hack/sync.sh
```
#### DNS
To ease with local development, the TLD `bigbang.dev` has been purchased with the following CNAME record:
`CNAME: *.bigbang.dev -> 127.0.0.1`
All routable endpoints BigBang deploys will use the TLD of `bigbang.dev` by default. It is expected that consumers modify this appropriately for their environment.
#### Secrets & Certificates
A __development only__ gpg key is provided at `hack/bigbang-dev.asc` that is used to encrypt and decrypt the "secret" information in `hack/secrets`.
We cannot stress enough, __do not use this key to encrypt real secret data__. It is a shared key meant to demonstrate the workflow of secrets management within Big Bang.
```bash
# Import the gpg key
gpg --import bigbang-dev.asc
# Decrypt the Big Bang Development Wildcard Cert
sops -d hack/secrets/ingress-cert.yaml
# Encrypt the Big Bang Development Wildcard Cert
sops -e hack/ingress-cert.yaml
```
## Merge requests process
The following is meant to serve as an overview of the pipeline stages required to get a commit merged.
### Pipeline Stages
The pipeline is split into several stages:
#### Linting
Several linting rules are first run to ensure yaml standards are met within the primary `./charts` folder.
This stage is ran on every commit, and is a requirement for merging.
#### Smoke Testing
For fast feedback testing, an ephemeral in cluster pipeline is created using [k3d](https://k3d.io/) that lives for the lifetime of the gitlab ci job. Within that cluster, BigBang is deployed, and an initial set of smoke tests are performed against the deployment to ensure basic conformance.
This stage verifies several easy to check assumptions such as:
* does BigBang successfully install
* does BigBang successfully upgrade (from master)
* are endpoints routable
This stage also serves as a guide for local development, and care is taken to ensure all pipeline actions within this stage are repeatable locally.
This stage is ran on every commit, and is a requirement for merging.
#### Infrastructure Testing
Ultimately, BigBang is designed to deploy production ready workloads on real infrastructure. While local and ephemeral clusters are excellent for fast feedback during development, changes must ultimately be tested on real clusters on real infrastructure.
As part of BigBang's [charter](https://repo1.dsop.io/platform-one/big-bang/charter), it is expected work on any CNCF conformant kubernetes cluster, on multiple clouds, and on premise environments. By very definition, this means infrastructure testing is _slow_. To strive for a pipeline with a happy medium of providing fast feedback while still exhaustively testing against environments that closely mirror production, __infrastructure testing only occurs on manual actions on merge request commits.__
When you are comfortable your branch is ready to be merged, opening up an merge request will trigger the creation of a suite of infrastructure testing jobs which will require a manual action from a project maintainer (assuming previous linting and smoke tests have passed). Once the commit(s) are validated against the infrastructure tests, your changes are ready to be merged!
For _most_ of the infrastructure testing, `terraform` is chosen as the IAC tool of choice for infrastructure that BigBang owns, while the cluster creation process follows the vendors recommended installation process.
The infrastructure pipeline is designed to have _no_ human interaction, and are scoped to the lifecycle of the pipeline. This means a single pipeline is fully responsible for provisioning infrastructure, but just as important, deprovisioning infrastructure, ensuring resources are not orphaned.
More information on the full set of infrastructure tests are below:
##### Network Creation
For each cloud, a BigBang owned network will be created that conform with the appropriate set of tests about to be ran. For example, to validate that Big Bang deploys in a connected environment on AWS, a VPC, subnets, route tables, etc... are created, and the outputs are made available through terraform's remote `data` source.
##### Cluster(s) Creation
Several types of clusters are created within the previously provisioned network(s), and follow the vendors recommended iac approach.
For example, an `rke2` cluster is created that leverages the upstream [terraform modules](https://repo1.dsop.io/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform), and an `eks` cluster is created with the upstream [terraform modules](https://docs.microsoft.com/en-us/azure/developer/terraform/create-k8s-cluster-with-tf-and-aks).
It is a hard requriement at this stage that every cluster outputs an admin scoped `kubeconfig` as a gitlab ci artifact. This artifact will be leveraged in the following stages for interacting with the created cluster.
##### Big Bang Installation
Given the kubeconfig created in the previous stage, BigBang is installed on the cluster using the same installation process used in the smoke tests.
Like any BigBang installation, several cluster requirements (TODO: doc these) must be met before BigBang is installed, and it is up to the vendor to ensure those requirements are met.
##### Big Bang Tests
Assuming BigBang has installed successfully, additional tests residing within the `./tests` folder of this repository are run against the deployed cluster. These tests range from automated UI testing, to internal kubernetes resource validation and verification.
TODO: Document these tests more once they are flushed out.
#### Teardown
Infrastructure teardown happens in the reverse sequence as to which they are deployed, and the pipeline will ensure these teardown jobs are _always_ ran, regardless of whether or not the previous jobs were successful.
Combined with terraform's declarative remote state, the "always on" teardown ensures no orphaned resources are left over once tests are run.
Within the teardown process, the commit scoped terraform workspace is also deleted to ensure the remote state remains clean.
For example, if an EKS cluster fails to provision, a full teardown of BigBang, EKS, and the network will be run, even though BigBang was never deployed. This will result in 2 failing jobs (EKS up and BigBang down), but will ensure that no infrastructure resources become orphaned.
......@@ -13,23 +13,12 @@ to the images, follow the guides below. These steps should NOT be used for prod
## Usage
The following examples expect a cluster with fluxv2 preinstalled. This can be done by [installing the flux cli](https://toolkit.fluxcd.io/get-started/#install-the-flux-cli) and running `flux install`. (TODO: Convert to IB images).
The following examples expect a cluster with fluxv2 preinstalled. This can be done by [installing the flux cli](https://toolkit.fluxcd.io/get-started/#install-the-flux-cli) and running `flux install`. This will install flux from the internet. If you wish to install the Iron Bank approved images, run `hack/flux-install.sh` to install flux from the [Iron Bank Registry](https://registry1.dsop.io).
### Simple Quickstart
### Quickstart
A bare mininmum, simple quickstart is provided under `./examples/simple`:
A quickstart BigBang environment template is provided [here](https://repo1.dsop.io/platform-one/big-bang/customers/bigbang/-/tree/master/bigbang). See the README.md to get started.
```bash
kubectl apply -f examples/simple
```
### Contributing
### Complete Example
While simple to use, Big Bang also allows full flexibility in configuring individual packages, using encrypted secrets, and deploying to multiple environments with the same configuration base.
See the [readme](./examples/complete/README.md) for more information.
### Developers
Developers can use the [Developer Setup](./examples/complete/README.md#development-workflow) to faciliate a local setup for developing improvements to Big Bang.
Please see our [contributing guide](./CONTRIBUTING.md) if you are interested in contributing to Big Bang.
\ No newline at end of file
---
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
name: bigbang
spec:
interval: 1m0s
url: https://repo1.dsop.io/platform-one/big-bang/umbrella.git
\ No newline at end of file
ignore: |
# exclude file extensions
/**/*.md
/**/*.txt
/**/*.sh
interval: 10m
url: https://repo1.dsop.io/platform-one/big-bang/umbrella.git
ref:
tag: 0.0.2
\ No newline at end of file
---
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
name: umbrella
namespace: flux-system
spec:
interval: 1m0s
url: https://repo1.dsop.io/platform-one/big-bang/apps/sandbox/umbrella
ref:
branch: master
---
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: bigbang
namespace: flux-system
spec:
targetNamespace: bigbang
releaseName: bigbang
interval: 10m
chart:
spec:
chart: chart
sourceRef:
kind: GitRepository
name: umbrella
interval: 1m0s
name: bigbang
test:
enable: false
install:
createNamespace: true
remediation:
retries: 3
retries: 5
upgrade:
remediation:
retries: 3
retries: 5
remediateLastFailure: true
cleanupOnFail: true
rollback:
timeout: 10m
cleanupOnFail: false
# Add additional variables by using the ConfigMap and Secrets below
valuesFrom:
- kind: Secret
name: common-bb
- kind: ConfigMap
name: bigbang-values
optional: true
name: common
- kind: Secret
name: bigbang-values
optional: true
# Modify big bang variables in line with the values below
# NOTE: These will take precedence over the `spec.valuesFrom` above
values:
global:
hostname: bigbang.dsop.io
name: environment-bb
- kind: ConfigMap
name: environment
\ No newline at end of file
configurations:
- transformer.yaml
namespace: bigbang
commonLabels:
owner: bigbang
resources:
- namespace.yaml
- gitrepository.yaml
- helmrelease.yaml
configMapGenerator:
- name: common
behavior: create
literals:
- values.yaml=
- name: environment
behavior: create
literals:
- values.yaml=
# Flux combines secrets and configmaps in `valuesFrom`. Kustomize
# cannot distinguish between them when applying suffixes. Therefore,
# the secrets must have different names than the configmaps
# While capital letters or dashes could be used, '-bb' was chosen
# to make the difference obvious at a glance
secretGenerator:
- name: common-bb
behavior: create
literals:
- values.yaml=
- name: environment-bb
behavior: create
literals:
- values.yaml=
\ No newline at end of file
---
apiVersion: v1
kind: Namespace
metadata:
name: bigbang
name: bigbang
\ No newline at end of file
nameReference:
- kind: ConfigMap
fieldSpecs:
- path: spec/valuesFrom/name
kind: HelmRelease
- kind: Secret
fieldSpecs:
- path: spec/valuesFrom/name
kind: HelmRelease
\ No newline at end of file
apiVersion: v2
name: bigbang-umbrella
name: bigbang
version: 0.0.2
......@@ -12,11 +12,24 @@ Build the appropriate spec.ref.{} given git branch, commit values
{{- if not .branch -}}
{{- fail "A valid branch is required when a commit is specified!" -}}
{{- end -}}
branch: {{ .branch }}
branch: {{ .branch | quote }}
commit: {{ .commit }}
{{- else if .semver -}}
semver: {{ .semver | quote }}
{{- else if .tag -}}
tag: {{ .tag }}
{{- else -}}
branch: {{ .branch }}
branch: {{ .branch | quote }}
{{- end -}}
{{- end -}}
{{/*
Build common set of file extensions to include/exclude
*/}}
{{- define "gitIgnore" -}}
ignore: |
# exclude file extensions
/**/*.md
/**/*.txt
/**/*.sh
{{- end -}}
\ No newline at end of file
......@@ -8,7 +8,7 @@ spec:
targetNamespace: argocd
chart:
spec:
chart: chart
chart: {{ .Values.addons.argocd.git.path }}
interval: 5m
sourceRef:
kind: GitRepository
......
......@@ -5,13 +5,9 @@ metadata:
name: argocd
namespace: {{ .Release.Namespace }}
spec:
ignore: |
# exclude file extensions
/**/*.md
/**/*.txt
/**/*.sh
interval: {{ .Values.flux.interval }}
url: {{ .Values.addons.argocd.git.repo }}
ref:
{{- include "validRef" .Values.addons.argocd.git | nindent 4 }}
{{ include "gitIgnore" . }}
{{- end }}
......@@ -8,7 +8,7 @@ spec:
targetNamespace: istio-system
chart:
spec:
chart: chart
chart: {{ .Values.addons.authservice.git.path }}
interval: 5m
sourceRef:
kind: GitRepository
......@@ -40,7 +40,6 @@ spec:
imagePullSecrets:
- name: private-registry
defaultConfig: false
filterLabel: keycloak
dependsOn:
......
......@@ -8,7 +8,7 @@ spec:
targetNamespace: logging
chart:
spec:
chart: charts/application
chart: {{ .Values.clusterAuditor.git.path }}
interval: 5m
sourceRef:
kind: GitRepository
......@@ -44,6 +44,6 @@ spec:
dependsOn:
- name: ek
namespace: {{ .Release.Namespace }}
- name: cluster-auditor-policies
- name: gatekeeper
namespace: {{ .Release.Namespace}}
{{- end }}
\ No newline at end of file
{{- if .Values.clusterAuditor.enabled }}
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: cluster-auditor-policies
namespace: {{ .Release.Namespace }}
spec:
targetNamespace: cluster-auditor
chart:
spec:
chart: charts/policies
interval: 5m
sourceRef:
kind: GitRepository
name: cluster-auditor
namespace: {{ .Release.Namespace }}
{{- with .Values.flux }}
interval: {{ .interval }}
test:
enable: false
install:
remediation:
retries: {{ .install.retries }}
upgrade:
remediation:
retries: {{ .upgrade.retries }}
remediateLastFailure: true
cleanupOnFail: true
rollback:
timeout: {{ .rollback.timeout }}
cleanupOnFail: {{ .rollback.cleanupOnFail }}
{{- end }}
valuesFrom:
- name: values
kind: Secret
valuesKey: "clusterauditor.yaml"
{{- if .Values.gatekeeper.enabled }}
dependsOn:
- name: gatekeeper
namespace: {{ .Release.Namespace }}
{{- end }}
{{- end }}
\ No newline at end of file
......@@ -5,13 +5,9 @@ metadata:
name: cluster-auditor
namespace: {{ .Release.Namespace }}
spec:
ignore: |
# exclude file extensions
/**/*.md
/**/*.txt
/**/*.sh
interval: {{ .Values.flux.interval }}
url: {{ .Values.clusterAuditor.git.repo }}
ref:
{{- include "validRef" .Values.clusterAuditor.git | nindent 4 }}
{{ include "gitIgnore" . }}
{{- end }}
......@@ -8,7 +8,7 @@ spec:
targetNamespace: gatekeeper-system
chart:
spec:
chart: charts/gatekeeper
chart: {{ .Values.gatekeeper.git.path }}
interval: 5m
sourceRef:
kind: GitRepository
......
......@@ -5,13 +5,9 @@ metadata:
name: gatekeeper
namespace: {{ .Release.Namespace }}
spec:
ignore: |
# exclude file extensions
/**/*.md
/**/*.txt
/**/*.sh
interval: {{ .Values.flux.interval }}
url: {{ .Values.gatekeeper.git.repo }}
ref:
{{- include "validRef" .Values.gatekeeper.git | nindent 4 }}
{{ include "gitIgnore" . }}
{{- end }}
......@@ -2,16 +2,12 @@
apiVersion: source.toolkit.fluxcd.io/v1beta1
kind: GitRepository
metadata:
name: istio
name: istio-controlplane
namespace: {{ .Release.Namespace }}
spec:
ignore: |
# exclude file extensions
/**/*.md
/**/*.txt
/**/*.sh
interval: {{ .Values.flux.interval }}
url: {{ .Values.istio.git.repo }}
ref:
{{- include "validRef" .Values.istio.git | nindent 4 }}
{{ include "gitIgnore" . }}
{{- end }}
......@@ -8,11 +8,11 @@ spec:
targetNamespace: istio-system
chart:
spec:
chart: charts/controlplane
chart: {{ .Values.istio.git.path }}
interval: 5m
sourceRef:
kind: GitRepository
name: istio
name: istio-controlplane
namespace: {{ .Release.Namespace }}
{{- with .Values.flux }}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment