Resolve "Add Global Promethus Label Based ExtraScrapeConfig"

chart/templates/monitoring/values.yaml

@@ -139,8 +139,99 @@ prometheus:
       static_configs:
       - targets: [vault.{{ .Values.domain }}]
       {{- end }}
+    {{- end }}           
+    # Scrape config for service endpoints.
+    #
+    # The relabeling allows the actual service scrape endpoint to be configured
+    # via the following annotations:
+    #
+    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
+    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
+    # to set this to `https` & most likely set the `tls_config` of the scrape config.
+    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+    # * `prometheus.io/port`: If the metrics are exposed on a different port to the
+    # service then set this appropriately.
+    {{- if (dig "globalServiceEndpointMetrics" "enabled" false .Values.monitoring) }}
+    - job_name: 'kubernetes-service-endpoints'
+      kubernetes_sd_configs:
+        - role: endpoints
+      scheme: https
+      tls_config:
+        insecure_skip_verify: true
+      follow_redirects: true
+      enable_http2: true 
+      relabel_configs:
+        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+          action: keep
+          regex: true
+        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+          action: replace
+          target_label: __scheme__
+          regex: (https?)
+        - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+          action: replace
+          target_label: __metrics_path__
+          regex: (.+)
+        - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+          action: replace
+          target_label: __address__
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+        - action: labelmap
+          regex: __meta_kubernetes_service_label_(.+)
+        - source_labels: [__meta_kubernetes_namespace]
+          action: replace
+          target_label: kubernetes_namespace
+        - source_labels: [__meta_kubernetes_service_name]
+          action: replace
+          target_label: kubernetes_name
+        - source_labels: [__meta_kubernetes_service_name]
+          action: drop
+          regex: '(.+)node-exporter'
+        - source_labels: [__meta_kubernetes_service_name]
+          action: drop
+          regex: '(.+)dns'
+        - source_labels: [__meta_kubernetes_service_name]
+          action: drop
+          regex: '(.+)kube-state-metrics'
+    {{- end }}
+    # Example scrape config for pods
+    #
+    # The relabeling allows the actual pod scrape endpoint to be configured via the
+    # following annotations:
+    #
+    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
+    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the default of `9102`.
+    {{- if (dig "globalPodEndpointMetrics" "enabled" false .Values.monitoring) }}
+    - job_name: 'kubernetes-pods'
+
+      kubernetes_sd_configs:
+        - role: pod
+    
+      relabel_configs:
+        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+          action: keep
+          regex: true
+        - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+          action: replace
+          target_label: __metrics_path__
+          regex: (.+)
+        - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+          action: replace
+          regex: ([^:]+)(?::\d+)?;(\d+)
+          replacement: $1:$2
+          target_label: __address__
+        - action: labelmap
+          regex: __meta_kubernetes_pod_label_(.+)
+        - source_labels: [__meta_kubernetes_namespace]
+          action: replace
+          target_label: kubernetes_namespace
+        - source_labels: [__meta_kubernetes_pod_name]
+          action: replace
+          target_label: kubernetes_pod_name
     {{- end }}
-    {{- if $istioInjection }}
+    {{- if $istioInjection }}    
     # Add volume/mount for Istio certs for mTLS scraping
     volumes:
       - emptyDir:
@@ -151,7 +242,6 @@ prometheus:
         name: istio-certs
     {{- end }}
 
-
 anchore:
   enabled: {{ .Values.addons.anchore.enabled }}
 
@@ -258,7 +348,7 @@ prometheusOperator:
       {{- if and (dig "additionalScrapeConfigs"  list $default) (dig "additionalScrapeConfigs"  list $overlay) }}
          {{ $_ := set $overlay "additionalScrapeConfigs" (concat $default.additionalScrapeConfigs $overlay.additionalScrapeConfigs) }}
       {{- end }}
-
+      
     {{- end }}
   {{- end }}
 {{ toYaml $overlays }}

docs/guides/using-bigbang/monitoring-applications-with-annotations-scraping.md

+# Integrating Prometheus Label Scraping for sample application
+
+## Introduction
+
+Integrating Prometheus metrics scraping with label scraping services is helpful for monitoring monitoring applications that either don't ship with a a ServiceMonitor, for testing in researching to build out a serviceMonitor resource or in-development applications where creating and managing annotations is simpler than building and managing ServiceMonitors. This guide will explain how to integrate metrics scraping for your annotated service endpoints or specific pods with Big Bang.
+
+By default, global endpoint monitoring is behind a hidden value in Big Bang. If you wish to enable it the sections below are for each configuration.
+
+The available Global serviceMonitor for annotated endpoints service is defined by the prometheus job name `kubernetes-service-endpoints` with following hidden value `monitoring.globalServiceEndpointMetrics` in `values.yaml`:
+```
+monitoring:
+  globalServiceEndpointMetrics:
+    enabled: true
+```
+
+The available Global serviceMonitor for annotated pods is defined by the prometheus job name `kubernetes-pods` with following hidden value `monitoring.globalPodEndpointMetrics` in `values.yaml`:
+```
+monitoring:
+  globalPodEndpointMetrics:
+    enabled: true
+```
+
+## Prerequisites
+
+Before integrating with Prometheus, ensure the following:
+
+- Determine if the application supports Prometheus metrics exporting. If not, find a Prometheus exporter to provide this service.
+- Identify the path and port used to scrape metrics on the application or exporter.
+- List the services and/or pods that should be monitored.
+
+## Integration Steps
+
+### 1. Define Placeholder Values for a Service (Recommended)
+
+Add placeholders in `chart/values.yaml` to configure whether the monitoring stack (Prometheus) is enabled:
+the following is an example of a placeholder for allowing  scraping of the `metrics-server` service endpoint:
+```yaml
+addons:
+  metrics-server:
+    values: 
+      service:
+        annotations:
+          prometheus.io/scrape: "true"
+          prometheus.io/port: "10250"
+          prometheus.io/path: "/metrics"
+      serviceMonitor:
+        enabled: false  # set to true to enable monitoring if the service is not already being scraped. This is an easily re-producible example but ideally the app will not ship with a serviceMonitor.
+
+note: The example above is for the metrics-server application, you will need to update the service annotations to match your application. Also note that metrics-server is already being scraped by a local serviceMonitor, so you will need to disable it so label scraping can be enabled.
+```
+
+Applications configured correctly will be scraped by Prometheus under the `target->` `kubernetes-service-endpoints` (see image below) in the Prometheus UI. If you do not know the correct port and path for your application, you may use `Service Discovery` in the Prometheus UI to find the correct port and path for your application.
+
+![Prometeus GUI](../../assets/imgs/developer/metrics-server-scraping.png)
+