Merge branch 'thanos-sidecar' into 'master'

Enable Thanos Object Level Storage

Closes big-bang/product/packages/thanos#49

See merge request !4503

chart/templates/monitoring/secret-objectstore.yaml

+{{- if and .Values.monitoring.enabled (not ( .Values.addons.thanos.objectStorage.endpoint | empty)) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: monitoring-objstore-secret
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: monitoring
+    app.kubernetes.io/component: "core"
+    {{- include "commonLabels" . | nindent 4}}
+type: kubernetes.io/opaque
+stringData:
+  objstore.yml: |
+    type: s3
+    config:
+      bucket: {{ .Values.addons.thanos.objectStorage.bucket }}
+      endpoint: {{ .Values.addons.thanos.objectStorage.endpoint }}
+      access_key: {{ .Values.addons.thanos.objectStorage.accessKey }}
+      secret_key: {{ .Values.addons.thanos.objectStorage.accessSecret }}
+      insecure: {{ .Values.addons.thanos.objectStorage.insecure }}
+{{- end }}

chart/templates/monitoring/values.yaml

@@ -16,6 +16,7 @@ domain: {{ $domainName }}
 {{- $lokiEnabled := (.Values.loki.enabled) }}
 {{- $clusterName := ( default "logging-loki" .Values.loki.clusterName ) }}
 
+
 flux:
   enabled: true
 
@@ -44,6 +45,20 @@ istio:
     {{- else }}
     enabled: false
     {{- end }}
+    {{- if and (dig "values" "istio" "hardened" "enabled" false .Values.monitoring) (contains "s3" .Values.addons.thanos.objectStorage.endpoint) }}
+    customServiceEntries:
+      - name: egress-object-store
+        enabled: true
+        spec:
+          hosts:
+           - "{{ .Values.addons.thanos.objectStorage.bucket }}.{{ .Values.addons.thanos.objectStorage.endpoint }}"
+          location: MESH_EXTERNAL
+          ports:
+            - number: 443
+              protocol: TLS
+              name: https
+          resolution: DNS
+     {{- end }}
   prometheus:
     enabled: true
     {{- if and .Values.monitoring.sso.enabled (eq $monitoringInjection "disabled") }}
@@ -114,7 +129,7 @@ prometheus:
 
     {{- if $thanosEnabled }}
     thanos:
-
+    
       {{- if (dig "values" "bbtests" "enabled" false .Values.addons.thanos) }}
       objectStorageConfig:
         secret:
@@ -137,6 +152,13 @@ prometheus:
       # by default, the object stores only sync every 2 hours
       # increase the frequency so the tests will pass 
       blockSize: 1m
+      
+      {{- else if not ( .Values.addons.thanos.objectStorage.endpoint | empty) }}
+      objectStorageConfig:
+        existingSecret:
+          key: objstore.yml
+          name: monitoring-objstore-secret
+
       {{- end }}
 
       {{- if $istioInjection }}    

chart/templates/thanos/values.yaml

@@ -6,6 +6,7 @@
 {{- end }}
 
 {{- define "bigbang.defaults.thanos" -}}
+{{- $thanosS3Endpoint := (printf "%s.s3.dualstack.%s.amazonaws.com" .Values.addons.thanos.objectStorage.bucket .Values.addons.thanos.objectStorage.region) }}
 
 imagePullSecrets:
 - name: private-registry
@@ -17,10 +18,24 @@ domain: {{ .Values.domain }}
 
 istio:
   enabled: {{ .Values.istio.enabled }}
+  {{- if and ( dig "values" "istio" "hardened" "enabled" false .Values.addons.thanos) (contains "s3" .Values.addons.thanos.objectStorage.endpoint) }}
+  hardened:
+    customServiceEntries:
+      - name: egress-object-store
+        enabled: true
+        spec:
+          hosts:
+          - {{ $thanosS3Endpoint }}
+          location: MESH_EXTERNAL
+          ports:
+            - number: 443
+              protocol: TLS
+              name: https
+          resolution: DNS
+  {{- end }}
   thanos:
     gateways:
     - istio-system/{{ default "public" .Values.addons.thanos.ingress.gateway }}
-    
 {{- with .Values.addons.thanos.objectStorage }}
 {{- if and (eq $.Values.addons.thanos.strategy "scalable") (not (and .endpoint .region)) }}
 minio:
@@ -62,6 +77,26 @@ query:
   podLabels:
     {{ $thanosAuthserviceKey }}: {{ $thanosAuthserviceValue }}
   {{- end }}
-    
 
-{{- end }}
\ No newline at end of file
+{{- if not (.Values.addons.thanos.objectStorage.endpoint | empty) }}
+
+objstoreConfig: |-
+  type: s3
+  config:
+    bucket: {{ .Values.addons.thanos.objectStorage.bucket }}
+    endpoint: {{ .Values.addons.thanos.objectStorage.endpoint }}
+    access_key: {{ .Values.addons.thanos.objectStorage.accessKey }}
+    secret_key: {{ .Values.addons.thanos.objectStorage.accessSecret }}
+    insecure: {{ .Values.addons.thanos.objectStorage.insecure }}
+
+storegateway:
+  enabled: true
+  useEndpointGroup: true
+  endpoint: {{ .Values.addons.thanos.objectStorage.endpoint }}
+
+query:
+  extraFlags:
+    - "--endpoint=dns+monitoring-monitoring-kube-thanos-discovery.monitoring.svc.cluster.local:{{- dig "values" "query" "containerPorts" "grpc" 10901 .Values.addons.thanos }}"
+
+{{- end }}
+{{- end }}

chart/values.yaml

@@ -2017,6 +2017,30 @@ addons:
 
       # -- OIDC Client Secret to use for Thanos
       client_secret: ""
+    
+    #Enable S3 Object Storage for Thanos-sidecar https://thanos.io/tip/components/sidecar.md/ and enables Thanos Store Gateway by default https://thanos.io/tip/components/store.md/ 
+    objectStorage:
+      # -- S3 compatible endpoint to use for connection information.
+      # examples: "s3.amazonaws.com" "s3.us-gov-west-1.amazonaws.com" "minio.minio.svc.cluster.local:9000"
+      # Note: Thanos does not require protocol prefix for URL.
+      endpoint: ""
+
+      # -- S3 compatible region to use for connection information.
+      region: ""
+
+      # -- Access key for connecting to object storage endpoint.
+      accessKey: ""
+
+      # -- Secret key for connecting to object storage endpoint.
+      # Unencoded string data. This should be placed in the secret values and then encrypted
+      accessSecret: ""
+
+      # -- Bucket Name for Thanos
+      # examples: "Thanos-metrics"
+      bucket: ""
+
+      # -- Whether or not objectStorage connection should require HTTPS, if connecting to in-cluster object
+      insecure: false
 
     # -- Choose source type of "git" or "helmRepo"
     sourceType: "git"