SKIP UPGRADE Add Thanos to BB Chart

chart/templates/NOTES.txt

@@ -398,3 +398,13 @@ DEPRECATION NOTICE:
   Please reconfigure your values overrides to use .Values.addons.mattermostOperator
 {{- end }}
 
+{{- if and $.Values.addons.thanos.enabled (not (dig "values" "prometheus" "prometheusSpec" "replicas" "" $.Values.monitoring)) }}
+PLATFORM ONE THANOS WARNING:
+  You have enabled Thanos with the default Prometheus replicas set to 1.  For production deployments, 
+  you can increase the number of replicas by adding:
+  monitoring:
+    values:
+      prometheus:
+        prometheusSpec:
+          replicas: 3
+{{- end }}

chart/templates/grafana/values.yaml

@@ -99,6 +99,14 @@ datasources:
     apiVersion: 1
     datasources:
     {{- if .Values.monitoring.enabled }}
+    {{- if .Values.addons.thanos.enabled }}
+    - name: Thanos
+      type: prometheus
+      uid: prometheus
+      access: proxy
+      url: http://thanos-query.thanos.svc:9090
+      editable: true
+    {{- else }}
     - name: Prometheus
       type: prometheus
       uid: prometheus
@@ -106,6 +114,7 @@ datasources:
       url: http://monitoring-monitoring-kube-prometheus.monitoring.svc:9090
       editable: true
     {{- end }}
+    {{- end }}
     {{- if .Values.addons.argocd.enabled }}
     - name: Argo Master
       type: redis-datasource

chart/templates/monitoring/thanos-objstore-secret.yaml

+{{- if .Values.monitoring.thanos.objstoreConfig }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: thanos-objstore-secret
+  namespace: monitoring
+data:
+  objstore.yml: {{- tpl .Values.monitoring.thanos.objstoreConfig . | trim | b64enc | indent 2 }}
+{{ end }}

chart/templates/monitoring/values.yaml

@@ -12,6 +12,7 @@ domain: {{ $domainName }}
 {{- $gitlabRedis := (and (ne .Values.addons.gitlab.redis.password "" ) (or .Values.addons.gitlab.enabled .Values.addons.gitlabRunner.enabled)) }}
 {{- $authserviceRedisEnabled := (and (dig "values" "redis" "enabled" false .Values.addons.authservice) .Values.addons.authservice.enabled) }}
 {{- $redisDatasource := (or $gitlabRedis .Values.addons.argocd.enabled $authserviceRedisEnabled) }}
+{{- $thanosEnabled := (.Values.addons.thanos.enabled) }}
 
 flux:
   enabled: true
@@ -79,7 +80,50 @@ alertmanager:
     {{- end }}
 
 prometheus:
+    # Note: We need to change the portName in order for istio to correctly detect TCP is being used
+    #       for the headless service
+
+  {{- if $thanosEnabled }}
+  thanosService:
+    enabled: true
+
+    {{- if $istioInjection }}
+    portName: "tcp-grpc"       
+    {{- end }}
+
+  thanosServiceMonitor:
+    enabled: true
+  {{- end }}
+
   prometheusSpec:
+
+    {{- if $thanosEnabled }}
+    thanos:
+
+      # by default, the object stores only sync every 2 hours
+      # increase the frequency so the tests will pass 
+      {{- if (dig "values" "bbtests" "enabled" false .Values.addons.thanos) }}
+      blockSize: 1m
+      {{- end }}
+
+      {{- if .Values.monitoring.thanos.objstoreConfig }}
+      objectStorageConfig:
+        key: objstore.yml
+        name: thanos-objstore-secret
+      {{- end }}
+
+      {{- if and $istioInjection .Values.monitoring.thanos.objstoreConfig }}    
+      # Add volume/mount on thanos sidecar for Istio certs for mTLS scraping
+      volumes:
+        - emptyDir:
+            medium: Memory
+          name: istio-certs
+      volumeMounts:
+        - mountPath: /etc/prom-certs/
+          name: istio-certs
+      {{- end }}
+    {{- end }}
+
     # The operator performs a strategic merge to add our imagePullPolicy definition to the default containers
     # NOTE: This functionality is not actively maintained upstream and may not work in a future monitoring upgrade
     containers:

chart/templates/thanos/gitrepository.yaml

+{{- $pkg := "thanos" }}
+{{- if and (eq (get .Values.addons $pkg).sourceType "git") (not .Values.offline) (get .Values.addons $pkg).enabled }}
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: {{ $pkg }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  interval: {{ .Values.flux.interval }}
+  url: {{ (get .Values.addons $pkg).git.repo }}
+  ref:
+    {{- include "validRef" (get .Values.addons $pkg).git | nindent 4 }}
+  {{ include "gitIgnore" . }}
+  {{- include "gitCreds" . | nindent 2 }}
+{{- end }}
\ No newline at end of file

chart/templates/thanos/helmrelease.yaml

+{{- $pkg := "thanos" }}
+{{- $fluxSettingsThanos := merge (get .Values.addons $pkg).flux .Values.flux -}}
+{{- if (get .Values.addons $pkg).enabled }}
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: {{ $pkg }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ $pkg }}
+    {{- include "commonLabels" . | nindent 4}}
+  annotations:
+    checksum/bigbang-values: {{ include (print $.Template.BasePath "/thanos/values.yaml") . | sha256sum }}
+spec:
+  releaseName: {{ $pkg }}
+  targetNamespace: {{ $pkg }}
+  chart:
+    spec:
+      {{- if eq (get .Values.addons $pkg).sourceType "git" }}
+      chart: {{ (get .Values.addons $pkg).git.path }}
+      sourceRef:
+        kind: GitRepository
+        name: thanos
+        namespace: {{ .Release.Namespace }}
+      {{- else }}
+      chart: {{ (get .Values.addons $pkg).helmRepo.chartName }}
+      version: {{ (get .Values.addons $pkg).helmRepo.tag }}
+      sourceRef:
+        kind: HelmRepository
+        name: {{ (get .Values.addons $pkg).helmRepo.repoName }}
+        namespace: {{ .Release.Namespace }}
+      {{- end }}
+      interval: 5m
+
+  {{- toYaml $fluxSettingsThanos | nindent 2 }}
+
+  {{- if (get .Values.addons $pkg).postRenderers }}
+  postRenderers:
+  {{ toYaml (get .Values.addons $pkg).postRenderers | nindent 4 }}
+  {{- end }}
+  valuesFrom:
+    - name: {{ .Release.Name }}-{{ $pkg }}-values
+      kind: Secret
+      valuesKey: "common"
+    - name: {{ .Release.Name }}-{{ $pkg }}-values
+      kind: Secret
+      valuesKey: "defaults"
+    - name: {{ .Release.Name }}-{{ $pkg }}-values
+      kind: Secret
+      valuesKey: "overlays"
+
+  {{- if or .Values.istio.enabled .Values.kyvernoPolicies.enabled .Values.monitoring.enabled }}
+  dependsOn:
+  {{- if .Values.istio.enabled }}
+    - name: istio
+      namespace: {{ .Release.Namespace }}
+  {{- end }}
+  {{- if .Values.kyvernoPolicies.enabled }}
+    - name: kyverno-policies
+      namespace: {{ .Release.Namespace }}
+  {{- end }}
+  {{- if .Values.monitoring.enabled }}
+    - name: monitoring
+      namespace: {{ .Release.Namespace }}
+  {{- end }}
+  {{- end }}
+{{- end }}
\ No newline at end of file

chart/templates/thanos/imagepullsecret.yaml

+{{- $pkg := "thanos" }}
+{{- if and (get .Values.addons $pkg).enabled ( include "imagePullSecret" . ) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: private-registry
+  namespace: {{ $pkg }}
+  labels:
+    app.kubernetes.io/name: {{ $pkg }}
+    {{- include "commonLabels" . | nindent 4}}
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: {{ template "imagePullSecret" . }}
+{{- end }}
\ No newline at end of file

chart/templates/thanos/namespace.yaml

+{{- $pkg := "thanos" }}
+{{- if (get .Values.addons $pkg).enabled }}
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ $pkg }}
+  labels:
+    {{- if .Values.istio.enabled }}
+    istio-injection: enabled
+    {{- end }}
+    app.kubernetes.io/name: {{ $pkg }}
+    app.kubernetes.io/component: "core"
+    {{- include "commonLabels" . | nindent 4}}
+{{- end }}
\ No newline at end of file

chart/templates/thanos/values.yaml

+{{- $pkg := "thanos" }}
+
+{{- /* Create secret */ -}}
+{{- if (get .Values.addons $pkg).enabled }}
+{{- include "values-secret" (dict "root" $ "package" (get .Values.addons $pkg) "name" $pkg "defaults" (include (printf "bigbang.defaults.%s" $pkg) .)) }}
+{{- end }}
+
+{{- define "bigbang.defaults.thanos" -}}
+
+imagePullSecrets:
+- name: private-registry
+imagePullPolicy: {{ .Values.imagePullPolicy }}
+
+externalURL: https://thanos.{{ .Values.domain }}
+
+domain: {{ .Values.domain }}
+
+istio:
+  enabled: {{ .Values.istio.enabled }}
+  thanos:
+    gateways:
+    - istio-system/{{ default "public" .Values.addons.thanos.ingress.gateway }}
+    
+{{- with .Values.addons.thanos.objectStorage }}
+{{- if and (eq $.Values.addons.thanos.strategy "scalable") (not (and .endpoint .region)) }}
+minio:
+  enabled: true
+{{- end }}
+{{- end }}
+
+networkPolicies:
+  enabled: {{ .Values.networkPolicies.enabled }}
+  controlPlaneCidr: {{ .Values.networkPolicies.controlPlaneCidr }}
+  ingressLabels:
+    {{- $gateway := default "public" .Values.addons.thanos.ingress.gateway }}
+    {{- $default := dict "app" (dig "gateways" $gateway "ingressGateway" nil .Values.istio) "istio" nil }}
+    {{- toYaml (dig "values" "gateways" $gateway "selector" $default .Values.istio) | nindent 4 }}
+
+monitoring:
+  enabled: {{ .Values.monitoring.enabled }}
+
+query:
+  dnsDiscovery:
+    # to allow lookups to work with and without Istio enabled, we disable k8s dns service
+    # discovery and manually set stores: below.
+    #
+    # With Istio, the combination of headless service + TCP port will create an entry
+    # for each pod IP:PORT and that makes communication via IP:PORT viable
+    enabled: false
+  {{- if or .Values.monitoring.enabled (dig "values" "storegateway" "enabled" false .Values.addons.thanos) }}
+  stores:
+  {{- end }}
+    {{- if .Values.monitoring.enabled }}
+    - dns+monitoring-monitoring-kube-thanos-discovery.monitoring.svc.cluster.local:10901
+    {{- end }}
+    {{- if (dig "values" "storegateway" "enabled" false .Values.addons.thanos) }}
+    - dns+thanos-storegateway.thanos.svc.cluster.local:10901
+    {{- end }}
+    
+objstoreConfig: {{- toYaml (tpl (dig "values" "objstoreConfig" "" .Values.addons.thanos) .) | indent 2 }}
+
+{{- end }}
\ No newline at end of file

chart/values.schema.json

@@ -685,6 +685,9 @@
         "ingress": {
           "$ref": "#/$defs/ingress"
         },
+        "thanos": {
+          "type": "object"
+        },
         "sso": {
           "type": "object",
           "properties": {
@@ -1065,6 +1068,13 @@
               "$ref": "#/$defs/basePackage"
             }
           ]
+        },
+        "thanos": {
+          "allOf": [
+            {
+              "$ref": "#/$defs/basePackage"
+            }
+          ]
         }
       }
     },

chart/values.yaml

@@ -851,14 +851,17 @@ monitoring:
   # -- Choose source type of "git" or "helmRepo"
   sourceType: "git"
 
+  thanos:
+    objstoreConfig: ""
+
   git:
     repo: https://repo1.dso.mil/big-bang/product/packages/monitoring.git
     path: "./chart"
-    tag: "51.10.0-bb.0"
+    tag: "51.10.0-bb.1"
   helmRepo:
     repoName: "registry1"
     chartName: "monitoring"
-    tag: "51.10.0-bb.0"
+    tag: "51.10.0-bb.1"
 
   # -- Flux reconciliation overrides specifically for the Monitoring Package
   flux:
@@ -909,11 +912,11 @@ grafana:
   git:
     repo: https://repo1.dso.mil/big-bang/product/packages/grafana.git
     path: "./chart"
-    tag: "6.60.6-bb.0"
+    tag: "6.60.6-bb.1"
   helmRepo:
     repoName: "registry1"
     chartName: "grafana"
-    tag: "6.60.6-bb.0"
+    tag: "6.60.6-bb.1"
 
   # -- Flux reconciliation overrides specifically for the Monitoring Package
   flux: {}
@@ -1895,6 +1898,39 @@ addons:
     # -- Post Renderers.  See docs/postrenders.md
     postRenderers: []
 
+  # ----------------------------------------------------------------------------------------------------------------------
+  # thanos
+  #
+  thanos:
+    # -- Toggle deployment of thanos
+    enabled: false
+
+    # -- Choose source type of "git" or "helmRepo"
+    sourceType: "git"
+
+    git:
+      repo: https://repo1.dso.mil/big-bang/apps/sandbox/thanos.git
+      tag: "12.13.12-bb.2"
+      path: "./chart"
+    helmRepo:
+      repoName: "registry1"
+      chartName: "thanos"
+      tag: "12.13.12-bb.2"
+
+    # -- Flux reconciliation overrides specifically for the Jaeger Package
+    flux: {}
+
+    # -- Redirect the package ingress to a specific Istio Gateway (listed in `istio.gateways`).  The default is "public".
+    ingress:
+      gateway: ""
+      
+    # -- Configure the object storage for Thanos.  
+    # The monitoring.prometheus thanos-sidecar and Thanos will use this configuration if defined 
+    objstoreConfig: ""
+
+    values: {}
+
+    postRenderers: []
 
 # -- Wrapper chart for integrating Big Bang components alongside a package
 wrapper:

tests/test-values.yaml

@@ -482,6 +482,7 @@ kyvernoPolicies:
               - neuvector
               - harbor
               - fortify
+              - thanos
               names:
               - "*-cypress-test*"
         parameters:
@@ -518,6 +519,7 @@ kyvernoPolicies:
               - neuvector
               - harbor
               - fortify
+              - thanos
               names:
               - "*-cypress-test*"
           - resources:
@@ -570,6 +572,7 @@ kyvernoPolicies:
               - neuvector
               - harbor
               - fortify
+              - thanos
               names:
               - "*-cypress-test*"
       update-image-pull-policy:
@@ -781,9 +784,27 @@ monitoring:
       client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_prometheus
     alertmanager:
       client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_alertmanager
+  thanos:
+    objstoreConfig: |-
+      type: s3
+      config:
+        bucket: "thanos"
+        endpoint: minio.thanos.svc.cluster.local:80
+        access_key: "minio"
+        secret_key: "minio123"
+        insecure: false
+        trace:
+          enable: true
+        http_config:
+          tls_config:
+            key_file: /etc/prom-certs/key.pem
+            ca_file: /etc/prom-certs/root-cert.pem
+            cert_file: /etc/prom-certs/cert-chain.pem
+          insecure_skip_verify: true  
   values:
     prometheus:
       prometheusSpec:
+        replicas: 1
         resources:
           requests:
             cpu: 100m
@@ -1969,3 +1990,58 @@ addons:
           image: "registry1.dso.mil/bigbang-ci/gitlab-tester:0.0.4"
           envs:
             HARBOR_REGISTRY: "harbor.bigbang.dev"
+
+  # ----------------------------------------------------------------------------------------------------------------------
+  # Thanos
+  #
+  thanos:
+    # -- Toggle deployment of thanos
+    enabled: false
+    values:
+      minio:
+        enabled: true
+        tenant:
+          pools:
+          - servers: 1
+            volumesPerServer: 4
+            size: 256Mi
+            resources:
+              requests:
+                cpu: 250m
+                memory: 2Gi
+              limits:
+                cpu: 250m
+                memory: 2Gi
+            securityContext:
+              runAsUser: 1001
+              runAsGroup: 1001
+              fsGroup: 1001
+              runAsNonRoot: true
+            containerSecurityContext:
+              runAsUser: 1001
+              runAsGroup: 1001
+              runAsNonRoot: true
+      storegateway:
+        enabled: true            
+      bbtests:
+        enabled: true
+        cypress:
+          artifacts: true
+          envs:
+            cypress_url: "https://thanos.bigbang.dev"
+            cypress_prometheus_integration_enabled: "true"
+            cypress_objstorage_integration_enabled: "true"
+        scripts:
+          image: "registry1.dso.mil/bigbang-ci/gitlab-tester:0.0.4"
+          envs:
+            THANOS_REGISTRY: "thanos.bigbang.dev"
+      objstoreConfig: |-
+        type: s3
+        config:
+          bucket: "thanos"
+          endpoint: minio.thanos.svc.cluster.local:80
+          access_key: "minio"
+          secret_key: "minio123"
+          insecure: true
+          trace:
+            enable: true