diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index 1d4abf3232ad84c9b5cb8c9323a258b2a1dc0aaa..e36092dd2852924b833abeea99635cb4b619804b 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -398,3 +398,13 @@ DEPRECATION NOTICE: Please reconfigure your values overrides to use .Values.addons.mattermostOperator {{- end }} +{{- if and $.Values.addons.thanos.enabled (not (dig "values" "prometheus" "prometheusSpec" "replicas" "" $.Values.monitoring)) }} +PLATFORM ONE THANOS WARNING: + You have enabled Thanos with the default Prometheus replicas set to 1. For production deployments, + you can increase the number of replicas by adding: + monitoring: + values: + prometheus: + prometheusSpec: + replicas: 3 +{{- end }} diff --git a/chart/templates/grafana/values.yaml b/chart/templates/grafana/values.yaml index 0ab158652ab1ba889bf5f4a8cdf74aebb5e103da..7f19e78a59a03f5f9cef8bce63b5e4b8ceb89cd3 100644 --- a/chart/templates/grafana/values.yaml +++ b/chart/templates/grafana/values.yaml @@ -99,6 +99,14 @@ datasources: apiVersion: 1 datasources: {{- if .Values.monitoring.enabled }} + {{- if .Values.addons.thanos.enabled }} + - name: Thanos + type: prometheus + uid: prometheus + access: proxy + url: http://thanos-query.thanos.svc:9090 + editable: true + {{- else }} - name: Prometheus type: prometheus uid: prometheus @@ -106,6 +114,7 @@ datasources: url: http://monitoring-monitoring-kube-prometheus.monitoring.svc:9090 editable: true {{- end }} + {{- end }} {{- if .Values.addons.argocd.enabled }} - name: Argo Master type: redis-datasource diff --git a/chart/templates/monitoring/thanos-objstore-secret.yaml b/chart/templates/monitoring/thanos-objstore-secret.yaml new file mode 100644 index 0000000000000000000000000000000000000000..035db13830e175a0f58d6d385f6b03faa8d461d9 --- /dev/null +++ b/chart/templates/monitoring/thanos-objstore-secret.yaml @@ -0,0 +1,9 @@ +{{- if .Values.monitoring.thanos.objstoreConfig }} +apiVersion: v1 +kind: Secret +metadata: + name: thanos-objstore-secret + namespace: monitoring +data: + objstore.yml: {{- tpl .Values.monitoring.thanos.objstoreConfig . | trim | b64enc | indent 2 }} +{{ end }} diff --git a/chart/templates/monitoring/values.yaml b/chart/templates/monitoring/values.yaml index e57d3892d953564003579ec16d075941104c38d6..d31766ad049a06645b8bee4de897a6a1dca326ad 100644 --- a/chart/templates/monitoring/values.yaml +++ b/chart/templates/monitoring/values.yaml @@ -12,6 +12,7 @@ domain: {{ $domainName }} {{- $gitlabRedis := (and (ne .Values.addons.gitlab.redis.password "" ) (or .Values.addons.gitlab.enabled .Values.addons.gitlabRunner.enabled)) }} {{- $authserviceRedisEnabled := (and (dig "values" "redis" "enabled" false .Values.addons.authservice) .Values.addons.authservice.enabled) }} {{- $redisDatasource := (or $gitlabRedis .Values.addons.argocd.enabled $authserviceRedisEnabled) }} +{{- $thanosEnabled := (.Values.addons.thanos.enabled) }} flux: enabled: true @@ -79,7 +80,50 @@ alertmanager: {{- end }} prometheus: + # Note: We need to change the portName in order for istio to correctly detect TCP is being used + # for the headless service + + {{- if $thanosEnabled }} + thanosService: + enabled: true + + {{- if $istioInjection }} + portName: "tcp-grpc" + {{- end }} + + thanosServiceMonitor: + enabled: true + {{- end }} + prometheusSpec: + + {{- if $thanosEnabled }} + thanos: + + # by default, the object stores only sync every 2 hours + # increase the frequency so the tests will pass + {{- if (dig "values" "bbtests" "enabled" false .Values.addons.thanos) }} + blockSize: 1m + {{- end }} + + {{- if .Values.monitoring.thanos.objstoreConfig }} + objectStorageConfig: + key: objstore.yml + name: thanos-objstore-secret + {{- end }} + + {{- if and $istioInjection .Values.monitoring.thanos.objstoreConfig }} + # Add volume/mount on thanos sidecar for Istio certs for mTLS scraping + volumes: + - emptyDir: + medium: Memory + name: istio-certs + volumeMounts: + - mountPath: /etc/prom-certs/ + name: istio-certs + {{- end }} + {{- end }} + # The operator performs a strategic merge to add our imagePullPolicy definition to the default containers # NOTE: This functionality is not actively maintained upstream and may not work in a future monitoring upgrade containers: diff --git a/chart/templates/thanos/gitrepository.yaml b/chart/templates/thanos/gitrepository.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c22920f6eadcdef23767f3a7aa810de2c864f0a --- /dev/null +++ b/chart/templates/thanos/gitrepository.yaml @@ -0,0 +1,15 @@ +{{- $pkg := "thanos" }} +{{- if and (eq (get .Values.addons $pkg).sourceType "git") (not .Values.offline) (get .Values.addons $pkg).enabled }} +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: {{ $pkg }} + namespace: {{ .Release.Namespace }} +spec: + interval: {{ .Values.flux.interval }} + url: {{ (get .Values.addons $pkg).git.repo }} + ref: + {{- include "validRef" (get .Values.addons $pkg).git | nindent 4 }} + {{ include "gitIgnore" . }} + {{- include "gitCreds" . | nindent 2 }} +{{- end }} \ No newline at end of file diff --git a/chart/templates/thanos/helmrelease.yaml b/chart/templates/thanos/helmrelease.yaml new file mode 100644 index 0000000000000000000000000000000000000000..005ab52a0327e4a38ec11b78c701932f8eebaa5e --- /dev/null +++ b/chart/templates/thanos/helmrelease.yaml @@ -0,0 +1,67 @@ +{{- $pkg := "thanos" }} +{{- $fluxSettingsThanos := merge (get .Values.addons $pkg).flux .Values.flux -}} +{{- if (get .Values.addons $pkg).enabled }} +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: {{ $pkg }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ $pkg }} + {{- include "commonLabels" . | nindent 4}} + annotations: + checksum/bigbang-values: {{ include (print $.Template.BasePath "/thanos/values.yaml") . | sha256sum }} +spec: + releaseName: {{ $pkg }} + targetNamespace: {{ $pkg }} + chart: + spec: + {{- if eq (get .Values.addons $pkg).sourceType "git" }} + chart: {{ (get .Values.addons $pkg).git.path }} + sourceRef: + kind: GitRepository + name: thanos + namespace: {{ .Release.Namespace }} + {{- else }} + chart: {{ (get .Values.addons $pkg).helmRepo.chartName }} + version: {{ (get .Values.addons $pkg).helmRepo.tag }} + sourceRef: + kind: HelmRepository + name: {{ (get .Values.addons $pkg).helmRepo.repoName }} + namespace: {{ .Release.Namespace }} + {{- end }} + interval: 5m + + {{- toYaml $fluxSettingsThanos | nindent 2 }} + + {{- if (get .Values.addons $pkg).postRenderers }} + postRenderers: + {{ toYaml (get .Values.addons $pkg).postRenderers | nindent 4 }} + {{- end }} + valuesFrom: + - name: {{ .Release.Name }}-{{ $pkg }}-values + kind: Secret + valuesKey: "common" + - name: {{ .Release.Name }}-{{ $pkg }}-values + kind: Secret + valuesKey: "defaults" + - name: {{ .Release.Name }}-{{ $pkg }}-values + kind: Secret + valuesKey: "overlays" + + {{- if or .Values.istio.enabled .Values.kyvernoPolicies.enabled .Values.monitoring.enabled }} + dependsOn: + {{- if .Values.istio.enabled }} + - name: istio + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.kyvernoPolicies.enabled }} + - name: kyverno-policies + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.monitoring.enabled }} + - name: monitoring + namespace: {{ .Release.Namespace }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/chart/templates/thanos/imagepullsecret.yaml b/chart/templates/thanos/imagepullsecret.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62cc56fbdc997e1435be941341f62df26413c33d --- /dev/null +++ b/chart/templates/thanos/imagepullsecret.yaml @@ -0,0 +1,14 @@ +{{- $pkg := "thanos" }} +{{- if and (get .Values.addons $pkg).enabled ( include "imagePullSecret" . ) }} +apiVersion: v1 +kind: Secret +metadata: + name: private-registry + namespace: {{ $pkg }} + labels: + app.kubernetes.io/name: {{ $pkg }} + {{- include "commonLabels" . | nindent 4}} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ template "imagePullSecret" . }} +{{- end }} \ No newline at end of file diff --git a/chart/templates/thanos/namespace.yaml b/chart/templates/thanos/namespace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..520336b73d3ea3c09e9c8323ae0ba86ea38197e5 --- /dev/null +++ b/chart/templates/thanos/namespace.yaml @@ -0,0 +1,14 @@ +{{- $pkg := "thanos" }} +{{- if (get .Values.addons $pkg).enabled }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $pkg }} + labels: + {{- if .Values.istio.enabled }} + istio-injection: enabled + {{- end }} + app.kubernetes.io/name: {{ $pkg }} + app.kubernetes.io/component: "core" + {{- include "commonLabels" . | nindent 4}} +{{- end }} \ No newline at end of file diff --git a/chart/templates/thanos/values.yaml b/chart/templates/thanos/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e80cc1bdebb8ae9f791e3547f23739675a132d9 --- /dev/null +++ b/chart/templates/thanos/values.yaml @@ -0,0 +1,62 @@ +{{- $pkg := "thanos" }} + +{{- /* Create secret */ -}} +{{- if (get .Values.addons $pkg).enabled }} +{{- include "values-secret" (dict "root" $ "package" (get .Values.addons $pkg) "name" $pkg "defaults" (include (printf "bigbang.defaults.%s" $pkg) .)) }} +{{- end }} + +{{- define "bigbang.defaults.thanos" -}} + +imagePullSecrets: +- name: private-registry +imagePullPolicy: {{ .Values.imagePullPolicy }} + +externalURL: https://thanos.{{ .Values.domain }} + +domain: {{ .Values.domain }} + +istio: + enabled: {{ .Values.istio.enabled }} + thanos: + gateways: + - istio-system/{{ default "public" .Values.addons.thanos.ingress.gateway }} + +{{- with .Values.addons.thanos.objectStorage }} +{{- if and (eq $.Values.addons.thanos.strategy "scalable") (not (and .endpoint .region)) }} +minio: + enabled: true +{{- end }} +{{- end }} + +networkPolicies: + enabled: {{ .Values.networkPolicies.enabled }} + controlPlaneCidr: {{ .Values.networkPolicies.controlPlaneCidr }} + ingressLabels: + {{- $gateway := default "public" .Values.addons.thanos.ingress.gateway }} + {{- $default := dict "app" (dig "gateways" $gateway "ingressGateway" nil .Values.istio) "istio" nil }} + {{- toYaml (dig "values" "gateways" $gateway "selector" $default .Values.istio) | nindent 4 }} + +monitoring: + enabled: {{ .Values.monitoring.enabled }} + +query: + dnsDiscovery: + # to allow lookups to work with and without Istio enabled, we disable k8s dns service + # discovery and manually set stores: below. + # + # With Istio, the combination of headless service + TCP port will create an entry + # for each pod IP:PORT and that makes communication via IP:PORT viable + enabled: false + {{- if or .Values.monitoring.enabled (dig "values" "storegateway" "enabled" false .Values.addons.thanos) }} + stores: + {{- end }} + {{- if .Values.monitoring.enabled }} + - dns+monitoring-monitoring-kube-thanos-discovery.monitoring.svc.cluster.local:10901 + {{- end }} + {{- if (dig "values" "storegateway" "enabled" false .Values.addons.thanos) }} + - dns+thanos-storegateway.thanos.svc.cluster.local:10901 + {{- end }} + +objstoreConfig: {{- toYaml (tpl (dig "values" "objstoreConfig" "" .Values.addons.thanos) .) | indent 2 }} + +{{- end }} \ No newline at end of file diff --git a/chart/values.schema.json b/chart/values.schema.json index 84828d039fc0d113eb132e5497b2f0a6c3ff13be..e3942294c327ec0fb94883af93233fba076f6bda 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -685,6 +685,9 @@ "ingress": { "$ref": "#/$defs/ingress" }, + "thanos": { + "type": "object" + }, "sso": { "type": "object", "properties": { @@ -1065,6 +1068,13 @@ "$ref": "#/$defs/basePackage" } ] + }, + "thanos": { + "allOf": [ + { + "$ref": "#/$defs/basePackage" + } + ] } } }, diff --git a/chart/values.yaml b/chart/values.yaml index 1003a1a90fd8b9cbbeb47382b1f50c1f429b1797..fe2e895fda5b073f1845816d799887c1946c3cab 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -851,14 +851,17 @@ monitoring: # -- Choose source type of "git" or "helmRepo" sourceType: "git" + thanos: + objstoreConfig: "" + git: repo: https://repo1.dso.mil/big-bang/product/packages/monitoring.git path: "./chart" - tag: "51.10.0-bb.0" + tag: "51.10.0-bb.1" helmRepo: repoName: "registry1" chartName: "monitoring" - tag: "51.10.0-bb.0" + tag: "51.10.0-bb.1" # -- Flux reconciliation overrides specifically for the Monitoring Package flux: @@ -909,11 +912,11 @@ grafana: git: repo: https://repo1.dso.mil/big-bang/product/packages/grafana.git path: "./chart" - tag: "6.60.6-bb.0" + tag: "6.60.6-bb.1" helmRepo: repoName: "registry1" chartName: "grafana" - tag: "6.60.6-bb.0" + tag: "6.60.6-bb.1" # -- Flux reconciliation overrides specifically for the Monitoring Package flux: {} @@ -1895,6 +1898,39 @@ addons: # -- Post Renderers. See docs/postrenders.md postRenderers: [] + # ---------------------------------------------------------------------------------------------------------------------- + # thanos + # + thanos: + # -- Toggle deployment of thanos + enabled: false + + # -- Choose source type of "git" or "helmRepo" + sourceType: "git" + + git: + repo: https://repo1.dso.mil/big-bang/apps/sandbox/thanos.git + tag: "12.13.12-bb.2" + path: "./chart" + helmRepo: + repoName: "registry1" + chartName: "thanos" + tag: "12.13.12-bb.2" + + # -- Flux reconciliation overrides specifically for the Jaeger Package + flux: {} + + # -- Redirect the package ingress to a specific Istio Gateway (listed in `istio.gateways`). The default is "public". + ingress: + gateway: "" + + # -- Configure the object storage for Thanos. + # The monitoring.prometheus thanos-sidecar and Thanos will use this configuration if defined + objstoreConfig: "" + + values: {} + + postRenderers: [] # -- Wrapper chart for integrating Big Bang components alongside a package wrapper: diff --git a/tests/test-values.yaml b/tests/test-values.yaml index e0d18316140d359c395efeb4c25462fccf641313..a2bef6d49ba295b3fc99b8f91acba970b44acd8e 100644 --- a/tests/test-values.yaml +++ b/tests/test-values.yaml @@ -482,6 +482,7 @@ kyvernoPolicies: - neuvector - harbor - fortify + - thanos names: - "*-cypress-test*" parameters: @@ -518,6 +519,7 @@ kyvernoPolicies: - neuvector - harbor - fortify + - thanos names: - "*-cypress-test*" - resources: @@ -570,6 +572,7 @@ kyvernoPolicies: - neuvector - harbor - fortify + - thanos names: - "*-cypress-test*" update-image-pull-policy: @@ -781,9 +784,27 @@ monitoring: client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_prometheus alertmanager: client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_alertmanager + thanos: + objstoreConfig: |- + type: s3 + config: + bucket: "thanos" + endpoint: minio.thanos.svc.cluster.local:80 + access_key: "minio" + secret_key: "minio123" + insecure: false + trace: + enable: true + http_config: + tls_config: + key_file: /etc/prom-certs/key.pem + ca_file: /etc/prom-certs/root-cert.pem + cert_file: /etc/prom-certs/cert-chain.pem + insecure_skip_verify: true values: prometheus: prometheusSpec: + replicas: 1 resources: requests: cpu: 100m @@ -1969,3 +1990,58 @@ addons: image: "registry1.dso.mil/bigbang-ci/gitlab-tester:0.0.4" envs: HARBOR_REGISTRY: "harbor.bigbang.dev" + + # ---------------------------------------------------------------------------------------------------------------------- + # Thanos + # + thanos: + # -- Toggle deployment of thanos + enabled: false + values: + minio: + enabled: true + tenant: + pools: + - servers: 1 + volumesPerServer: 4 + size: 256Mi + resources: + requests: + cpu: 250m + memory: 2Gi + limits: + cpu: 250m + memory: 2Gi + securityContext: + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + runAsNonRoot: true + containerSecurityContext: + runAsUser: 1001 + runAsGroup: 1001 + runAsNonRoot: true + storegateway: + enabled: true + bbtests: + enabled: true + cypress: + artifacts: true + envs: + cypress_url: "https://thanos.bigbang.dev" + cypress_prometheus_integration_enabled: "true" + cypress_objstorage_integration_enabled: "true" + scripts: + image: "registry1.dso.mil/bigbang-ci/gitlab-tester:0.0.4" + envs: + THANOS_REGISTRY: "thanos.bigbang.dev" + objstoreConfig: |- + type: s3 + config: + bucket: "thanos" + endpoint: minio.thanos.svc.cluster.local:80 + access_key: "minio" + secret_key: "minio123" + insecure: true + trace: + enable: true