diff --git a/chart/dev-k3d-values.yaml b/chart/dev-k3d-values.yaml
index 2790118a2a7169cd5c9f78232a6f56c319811d2a..20e34f525fefd70e6d7dbce52e47fee68fab03d8 100644
--- a/chart/dev-k3d-values.yaml
+++ b/chart/dev-k3d-values.yaml
@@ -2,16 +2,60 @@
# this is required for development deployments to allow the istio loadbalancer daemonset to come up with rancher images rather than images from registry1.dso.mil or registry.dso.mil
gatekeeper:
- enabled: true
values:
violations:
+ allowedCapabilities:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to not drop capabilities
+ - istio-system/lb-port-.*
allowedDockerRegistries:
- match:
- excludedNamespaces:
- # Allows load balancer images for k3d from public repo
- - istio-system
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to pull from public repos
+ - istio-system/lb-port-.*
+ allowedSecCompProfiles:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have an undefined defined seccomp
+ - istio-system/lb-port-.*
+ allowedUsers:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to run as any user/group
+ - istio-system/lb-port-.*
+ containerRatio:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined limits/requests
+ - istio-system/lb-port-.*
hostNetworking:
- match:
- excludedNamespaces:
- # Allows load balancer containers to map ports for k3d
- - istio-system
\ No newline at end of file
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to mount host ports
+ - istio-system/lb-port-.*
+ noBigContainers:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined limits/requests
+ - istio-system/lb-port-.*
+ noPrivilegedEscalation:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined security context
+ - istio-system/lb-port-.*
+ readOnlyRoot:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to mount filesystems read/write
+ - istio-system/lb-port-.*
+ requiredLabels:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer pods to not have required labels
+ - istio-system/svclb-.*
+ requiredProbes:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to not have readiness/liveness probes
+ - istio-system/lb-port-.*
\ No newline at end of file
diff --git a/chart/templates/gatekeeper/values.yaml b/chart/templates/gatekeeper/values.yaml
index 61fee1c578af82ae072416db214fbea8a1653c2f..6ffde500f85aee73f0ebc0c49fb9ff02fc4191e4 100644
--- a/chart/templates/gatekeeper/values.yaml
+++ b/chart/templates/gatekeeper/values.yaml
@@ -36,30 +36,26 @@ violations: # Try to keep this in alpha order to make it easier to find keys
{{- if or .Values.monitoring.enabled (or .Values.fluentbit.enabled .Values.twistlock.enabled) }}
allowedHostFilesystem:
- {{- if .Values.monitoring.enabled }}
- match:
- excludedNamespaces:
- # Prometheus-node-exporter needs access to host to get node metrics
- - monitoring
- {{- end }}
- {{- if or .Values.fluentbit.enabled .Values.twistlock.enabled }}
parameters:
excludedResources:
- {{- if .Values.fluentbit.enabled }}
- # Fluentbit pods need access to host to get log files
- - logging/logging-fluent-bit-.*
- {{- end }}
- {{- if .Values.twistlock.enabled }}
- - twistlock/twistlock-defender-ds-.*
- {{- end }}
- {{- end }}
+ {{- if .Values.monitoring.enabled }}
+ # Prometheus-node-exporter needs access to host to get node metrics
+ - monitoring/monitoring-monitoring-prometheus-node-exporter-.*
+ {{- end }}
+ {{- if .Values.fluentbit.enabled }}
+ # Fluentbit pods need access to host to get log files
+ - logging/logging-fluent-bit-.*
+ {{- end }}
+ {{- if .Values.twistlock.enabled }}
+ - twistlock/twistlock-defender-ds-.*
+ {{- end }}
{{- end }}
{{- if .Values.twistlock.enabled }}
hostNetworking:
parameters:
excludedResources:
- - twistlock/twistlock-defender-ds-.*
+ - twistlock/twistlock-defender
noHostNamespace:
parameters:
excludedResources:
@@ -68,22 +64,22 @@ violations: # Try to keep this in alpha order to make it easier to find keys
{{- if .Values.addons.mattermost.enabled }}
httpsOnly:
- match:
- excludedNamespaces:
+ parameters:
+ excludedResources:
# Mattermost currently does not useIngressTLS hence Ingress is created without TLS field by the operator.
# Adding exemption, pending https://github.com/mattermost/mattermost-operator/issues/235
- - mattermost
+ - mattermost/mattermost
{{- end }}
namespacesHaveIstio:
enabled: {{ .Values.istio.enabled }}
- {{- if .Values.logging.enabled }}
+ {{- if .Values.fluentbit.enabled }}
noPrivilegedContainers:
- match:
- excludedNamespaces:
+ parameters:
+ excludedResources:
# Fluentbit needs privileged to read and store the buffer for tailing logs from the nodes
- - logging
+ - logging/fluent-bit
{{- end }}
podsHaveIstio:
@@ -96,53 +92,49 @@ violations: # Try to keep this in alpha order to make it easier to find keys
{{- if or .Values.monitoring.enabled .Values.twistlock.enabled }}
restrictedTaint:
- {{- if .Values.monitoring.enabled }}
- match:
- excludedNamespaces:
- # Prometheus Node Exporter needs to be able to run on all nodes, regardless of taint, to gather node metrics
- - monitoring
- {{- end }}
- {{- if .Values.twistlock.enabled }}
parameters:
excludedResources:
- - twistlock/twistlock-defender-ds-.*
- {{- end }}
+ {{- if .Values.monitoring.enabled }}
+ # Prometheus Node Exporter needs to be able to run on all nodes, regardless of taint, to gather node metrics
+ - monitoring/monitoring-monitoring-prometheus-node-exporter-.*
+ {{- end }}
+ {{- if .Values.twistlock.enabled }}
+ - twistlock/twistlock-defender-ds-.*
+ {{- end }}
{{- end }}
- {{- if or .Values.logging.enabled .Values.twistlock.enabled }}
+ {{- if or .Values.fluentbit.enabled .Values.twistlock.enabled }}
selinuxPolicy:
- {{- if .Values.logging.enabled }}
- match:
- excludedNamespaces:
- # FluentBit needs selinux option type spc_t
- - logging
- {{- end }}
- {{- if .Values.twistlock.enabled }}
parameters:
excludedResources:
- # Twistlock Defenders need selinux option type spc_t
- - twistlock/twistlock-defender
- {{- end }}
+ {{- if .Values.fluentbit.enabled }}
+ # FluentBit needs selinux option type spc_t
+ - logging/fluent-bit
+ {{- end }}
+ {{- if .Values.twistlock.enabled }}
+ # Twistlock Defenders need selinux option type spc_t
+ - twistlock/twistlock-defender
+ {{- end }}
{{- end }}
{{- if or .Values.fluentbit.enabled (or .Values.twistlock.enabled .Values.monitoring.enabled) }}
volumeTypes:
- match:
- excludedNamespaces:
+ parameters:
+ excludedResources:
{{- if .Values.fluentbit.enabled }}
# fluent-bit container requires certain host level access to ship logs and for keep track of state
# https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#workflow-of-tail-kubernetes-filter
- - logging
+ - logging/logging-fluent-bit-.*
{{- end }}
{{- if .Values.twistlock.enabled }}
# Twistlock requires /dev/log for its syslog daemon.
# https://docs.paloaltonetworks.com/prisma/prisma-cloud/prisma-cloud-admin-compute/audit/logging.html#
- - twistlock
+ - twistlock/twistlock-defender-ds-.*
{{- end }}
{{- if .Values.monitoring.enabled }}
# Prometheus node exported requires hostpath hardcoded in upstream chart on which monitoring pkg has a direct dependency
# https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-node-exporter/templates/daemonset.yaml#L150
- - monitoring
+ - monitoring/monitoring-monitoring-prometheus-node-exporter-.*
{{- end }}
{{- end }}
{{- end -}}
diff --git a/docs/developer/development-environment.md b/docs/developer/development-environment.md
index 9fb524553be73514630abbb4b8e3097f9dd44de0..42f8295fd67ca7016f1f2d2926aa8c9a2d0b37f0 100644
--- a/docs/developer/development-environment.md
+++ b/docs/developer/development-environment.md
@@ -2,7 +2,7 @@
[[_TOC_]]
-BigBang developers use [k3d](https://k3d.io/), a lightweight wrapper to run [k3s](https://github.com/rancher/k3s) (Rancher Lab’s minimal Kubernetes distribution) in docker.
+BigBang developers use [k3d](https://k3d.io/), a lightweight wrapper to run [k3s](https://github.com/rancher/k3s) (Rancher Lab’s minimal Kubernetes distribution) in docker.
It is not recommend to run k3d with BigBang on your local computer. BigBang can be quite resource-intensive and it requires a huge download bandwidth for the images. It is best to use a remote k3d cluster running on an AWS EC2 instance. If you do insist on running k3d locally you should disable certain packages before deploying. You can do this in the values.yaml file by setting the package deploy to false. One of the packages that is most resource-intensive is the logging package. And you should create a local image registry cache to minimize the amount of image downloading. A script that shows how to create a local image cache is in the [BigBang Quick Start](https://repo1.dso.mil/platform-one/quick-start/big-bang/)
@@ -19,7 +19,7 @@ This page contains the manual steps to create your k3d dev environment. Various
### Local Utilities
- [Helm](https://helm.sh/docs/intro/install/)
-- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
## Manual Creation of a Development Environment
@@ -37,12 +37,12 @@ Create an Ubuntu EC2 instance using the AWS console with the following attribute
```shell
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
-
+
--==MYBOUNDARY==
Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
-# Set the vm.max_map_count to 262144.
+# Set the vm.max_map_count to 262144.
# Required for Elastic to run correctly without OOM errors.
echo 'vm.max_map_count=524288' > /etc/sysctl.d/vm-max_map_count.conf
echo 'fs.file-max=131072' > /etc/sysctl.d/fs-file-max.conf
@@ -57,7 +57,7 @@ modprobe xt_statistic
- 50 Gigs of disk space
- Tags: ```Name: <firstname.lastname>```
- Security Group: All TCP limited to your local IP address. If you already have a security group, select it. Otherwise create a new one. See addendum for more secure way with only port 22 for ssh traffic using sshuttle.
-- If you have created an existing key pair that you still have access to, select it. If not, create a new key pair. Be sure to save the pem file.
+- If you have created an existing key pair that you still have access to, select it. If not, create a new key pair. Be sure to save the pem file.
### Step 2
@@ -204,23 +204,69 @@ cd ./bigbang
./scripts/install_flux.sh -u your-user-name -p your-pull-secret
```
-**Note1:** When deploying to k3d, istio-system should be added from `excludedNamespaces` under the `allowedDockerRegistries` violations for gatekeeper. This can be done by modifying `chart/values.yaml` file or passing an override file with the values set as seen below. This is for development purposes only: production should not allow containers in the `istio-system` namespace to be pulled from outside of Registry1.
+**Note1:** When deploying to k3d, the load balancer must be added to `excludedResources` under several violations for gatekeeper. This can be done by modifying `chart/values.yaml` file or passing an override file (e.g. `chart/k3d-dev-values.yaml`) with the values set below. This is for development purposes only.
```yaml
gatekeeper:
values:
violations:
+ allowedCapabilities:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to not drop capabilities
+ - istio-system/lb-port-.*
allowedDockerRegistries:
- match:
- excludedNamespaces:
- # Allows load balancer images for k3d from public repo
- - istio-system
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to pull from public repos
+ - istio-system/lb-port-.*
+ allowedSecCompProfiles:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have an undefined defined seccomp
+ - istio-system/lb-port-.*
+ allowedUsers:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to run as any user/group
+ - istio-system/lb-port-.*
+ containerRatio:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined limits/requests
+ - istio-system/lb-port-.*
hostNetworking:
- match:
- excludedNamespaces:
- # Allows load balancer containers to map ports for k3d
- - istio-system
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to mount host ports
+ - istio-system/lb-port-.*
+ noBigContainers:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined limits/requests
+ - istio-system/lb-port-.*
+ noPrivilegedEscalation:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined security context
+ - istio-system/lb-port-.*
+ readOnlyRoot:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to mount filesystems read/write
+ - istio-system/lb-port-.*
+ requiredLabels:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer pods to not have required labels
+ - istio-system/svclb-.*
+ requiredProbes:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to not have readiness/liveness probes
+ - istio-system/lb-port-.*
```
+
**Note2:** The information in this note is simply to give you awareness in advance. You should create local directory on your workstation where you store your helm values override files. Development changes made in the code for testing could accidentally be committed. That is why you should create a separate local directory to hold your override values for testing. The location can be anywhere on your workstation but it is most convenient to place them in a sibling directory next to the BigBang repos. Below is an example directory structure. The directory names are fake (for example only). Other documents will give more specific detail as needed.
```text
├── BigBangCodeRepo/
@@ -274,7 +320,7 @@ If the hosts shown in the HOSTS column from `kubectl get vs -A` don't resolve to
<IP of the EC2 instance> kibana.bigbang.dev prometheus.bigbang.dev grafana.bigbang.dev alertmanager.bigbang.dev kiali.bigbang.dev tracing.bigbang.dev
```
-It is important to use hostnames when accessing cluster apps in a browser instead of IPs as the hostname sent by the browser in its HTTP GET request is used by the load balancers (see: kubectl get svc -n istio-system) to direct the traffic to the correct app.
+It is important to use hostnames when accessing cluster apps in a browser instead of IPs as the hostname sent by the browser in its HTTP GET request is used by the load balancers (see: kubectl get svc -n istio-system) to direct the traffic to the correct app.
### Multi Ingress-gateway Support with MetalLB and K3D
diff --git a/tests/ci/k3d/values.yaml b/tests/ci/k3d/values.yaml
index 27b97fb8dcf5b413e3213f445433f418f2c338b3..3e312de275e37857e356a41e29c6c83906788c97 100644
--- a/tests/ci/k3d/values.yaml
+++ b/tests/ci/k3d/values.yaml
@@ -161,16 +161,61 @@ gatekeeper:
memory: 256Mi
limits: {}
violations:
+ allowedCapabilities:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to not drop capabilities
+ - istio-system/lb-port-.*
allowedDockerRegistries:
- match:
- excludedNamespaces:
- # Allows load balancer images for k3d from public repo
- - istio-system
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to pull from public repos
+ - istio-system/lb-port-.*
+ allowedSecCompProfiles:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have an undefined defined seccomp
+ - istio-system/lb-port-.*
+ allowedUsers:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to run as any user/group
+ - istio-system/lb-port-.*
+ containerRatio:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined limits/requests
+ - istio-system/lb-port-.*
hostNetworking:
- match:
- excludedNamespaces:
- # Allows load balancer containers to map ports for k3d
- - istio-system
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to mount host ports
+ - istio-system/lb-port-.*
+ noBigContainers:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined limits/requests
+ - istio-system/lb-port-.*
+ noPrivilegedEscalation:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to have undefined security context
+ - istio-system/lb-port-.*
+ readOnlyRoot:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to mount filesystems read/write
+ - istio-system/lb-port-.*
+ requiredLabels:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer pods to not have required labels
+ - istio-system/svclb-.*
+ requiredProbes:
+ parameters:
+ excludedResources:
+ # Allows k3d load balancer containers to not have readiness/liveness probes
+ - istio-system/lb-port-.*
bbtests:
# TODO: Test will need to be refactored at BB level to properly run since we can't turn everything to deny
# https://repo1.dso.mil/platform-one/big-bang/apps/core/policy/-/issues/133
@@ -359,7 +404,7 @@ addons:
size: 256Mi
slave:
persistence:
- size: 256Mi
+ size: 256Mi
bbtests:
enabled: true
cypress:
@@ -438,8 +483,8 @@ addons:
resources:
requests:
cpu: 100m
- memory: 200Mi
- limits: {}
+ memory: 200Mi
+ limits: {}
metrics:
resources:
requests:
@@ -573,7 +618,7 @@ addons:
size: 5Gi
postgresql:
persistence:
- size: 256Mi
+ size: 256Mi
resources:
requests:
cpu: 100m