configure `monitors` and `dashboards` for flux, istio, and cert-manager

base/cert-manager/helmrelease.yaml

@@ -17,10 +17,11 @@ spec:
   values:
     installCRDs: true
     prometheus:
+      enabled: true
       servicemonitor:
         labels:
           release: monitoring
-        enabled: false
+        enabled: true
   valuesFrom:
   - name: env-values
     kind: ConfigMap

base/flux/monitoring/dashboards/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+commonLabels:
+  grafana_dashboard: "1"
+  release: monitoring
+configMapGenerator:
+- name: flux-dashboards
+  files:
+  - cluster.json
+  - control-plane.json

base/flux/monitoring/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- monitors
+- dashboards

base/flux/monitoring/monitors/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- podmonitors.yaml
+commonLabels:
+  release: monitoring

base/flux/monitoring/monitors/podmonitors.yaml

+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: helm-controller
+  namespace: flux-system
+spec:
+  selector:
+    matchLabels:
+      app: helm-controller
+  podMetricsEndpoints:
+  - port: http-prom
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: kustomize-controller
+  namespace: flux-system
+spec:
+  selector:
+    matchLabels:
+      app: kustomize-controller
+  podMetricsEndpoints:
+  - port: http-prom
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: notification-controller
+  namespace: flux-system
+spec:
+  selector:
+    matchLabels:
+      app: notification-controller
+  podMetricsEndpoints:
+  - port: http-prom
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: source-controller
+  namespace: flux-system
+spec:
+  selector:
+    matchLabels:
+      app: source-controller
+  podMetricsEndpoints:
+  - port: http-prom

base/istio/kustomization.yaml

-resources:
-- istio-operator
-- istio-system

base/istio/monitoring/dashboards/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+commonLabels:
+  grafana_dashboard: "1"
+  release: monitoring
+configMapGenerator:
+- name: istio-dashboards-1
+  files:
+  - pilot-dashboard.json
+  - mixer-dashboard.json
+  - istio-performance-dashboard.json
+- name: istio-dashboards-2
+  files:
+  - istio-service-dashboard.json
+  - istio-workload-dashboard.json
+  - istio-mesh-dashboard.json

base/istio/monitoring/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- monitors
+- dashboards

base/istio/monitoring/monitors/kustomization.yaml

+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- servicemonitors.yaml
+commonLabels:
+  release: monitoring

base/istio/monitoring/monitors/servicemonitors.yaml

+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: istio-mesh-monitor
+  labels:
+    monitoring: istio-mesh
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio, operator: In, values: [mixer]}
+  endpoints:
+  - port: prometheus
+    interval: 15s
+  namespaceSelector:
+    matchNames:
+    - istio-system
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: istio-component-monitor
+  labels:
+    monitoring: istio-components
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio, operator: In, values: [mixer, pilot, galley, citadel, sidecar-injector]}
+  endpoints:
+  - port: http-monitoring
+    interval: 15s
+  - port: http-policy-monitoring
+    interval: 15s
+  jobLabel: istio
+  namespaceSelector:
+    any: true
+  targetLabels: [app]
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: envoy-stats-monitor
+  labels:
+    monitoring: istio-proxies
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio-prometheus-ignore, operator: DoesNotExist}
+  endpoints:
+  - targetPort: 15090
+    interval: 15s
+    path: /stats/prometheus
+    relabelings:
+    - action: keep
+      regex: '.*-envoy-prom'
+      sourceLabels: [__meta_kubernetes_pod_container_port_name]
+    - action: labeldrop
+      regex: "__meta_kubernetes_pod_label_(.+)"
+    - action: replace
+      sourceLabels: [__meta_kubernetes_namespace]
+      targetLabel: namespace
+    - action: replace
+      sourceLabels: [__meta_kubernetes_pod_name]
+      targetLabel: pod_name
+  jobLabel: envoy-stats
+  namespaceSelector:
+    any: true
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kubernetes-pods-monitor
+  labels:
+    monitoring: kube-pods
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio-prometheus-ignore, operator: DoesNotExist}
+  endpoints:
+  - interval: 15s
+    relabelings:
+    - action: keep
+      regex: 'true'
+      sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+    - action: keep
+      regex: '((;.*)|(.*;http)|(.??))'
+      sourceLabels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_prometheus_io_scheme]
+    - action: drop
+      regex: 'true'
+      sourceLabels: [__meta_kubernetes_pod_annotation_istio_mtls]
+    - action: replace
+      regex: '(.+)'
+      sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      targetLabel: __metrics_path__
+    - action: replace
+      regex: '([^:]+)(?::\d+)?;(\d+)'
+      replacement: $1:$2
+      sourceLabels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      targetLabel: __address__
+    - action: labelmap
+      regex: '__meta_kubernetes_pod_label_(.+)'
+    - action: replace
+      sourceLabels: [__meta_kubernetes_namespace]
+      targetLabel: namespace
+    - action: replace
+      sourceLabels: [__meta_kubernetes_pod_name]
+      targetLabel: pod_name
+  jobLabel: kubernetes-pods
+  namespaceSelector:
+    any: true
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kubernetes-pods-secure-monitor
+  labels:
+    monitoring: kube-pods-secure
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio-prometheus-ignore, operator: DoesNotExist}
+  endpoints:
+  - interval: 15s
+    relabelings:
+    - action: keep
+      regex: 'true'
+      sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+    - action: keep
+      regex: '(([^;]+);([^;]*))|(([^;]*);(true))'
+      # sidecar status annotation is added by sidecar injector and
+      # istio_workload_mtls_ability can be specifically placed on a pod to indicate its ability to receive mtls traffic.
+      sourceLabels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_istio_mtls]
+    - action: drop
+      regex: '(http)'
+      sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+    - action: replace
+      regex: '(.+)'
+      sourceLabels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+      targetLabel: __metrics_path__
+    - action: keep # otherwise an extra target with ':443' is added for https scheme
+      regex: '([^:]+):(\d+)'
+      sourceLabels: [__address__] # Only keep address that is host:port
+    - action: replace
+      regex: '([^:]+)(?::\d+)?;(\d+)'
+      replacement: $1:$2
+      sourceLabels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+      targetLabel: __address__
+    - action: labelmap
+      regex: '__meta_kubernetes_pod_label_(.+)'
+    - action: replace
+      sourceLabels: [__meta_kubernetes_namespace]
+      targetLabel: namespace
+    - action: replace
+      sourceLabels: [__meta_kubernetes_pod_name]
+      targetLabel: pod_name
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/secrets/istio.prometheus/root-cert.pem
+      certFile: /etc/prometheus/secrets/istio.prometheus/cert-chain.pem
+      insecureSkipVerify: true # prometheus does not support secure naming.
+      keyFile: /etc/prometheus/secrets/istio.prometheus/key.pem
+  jobLabel: kubernetes-pods-secure
+  namespaceSelector:
+    any: true
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kubernetes-services-monitor
+  labels:
+    monitoring: kube-services
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio-prometheus-ignore, operator: DoesNotExist}
+  endpoints:
+  - interval: 15s
+    relabelings:
+    - action: keep
+      regex: 'true'
+      sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+    - action: keep
+      regex: '((;.*)|(.*;http)|(.??))'
+      sourceLabels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_service_annotation_prometheus_io_scheme]
+    - action: drop
+      regex: 'true'
+      sourceLabels: [__meta_kubernetes_pod_annotation_istio_mtls]
+    - action: replace
+      regex: '(.+)'
+      sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+      targetLabel: __metrics_path__
+    - action: replace
+      regex: '([^:]+)(?::\d+)?;(\d+)'
+      replacement: $1:$2
+      sourceLabels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+      targetLabel: __address__
+    - action: labelmap
+      regex: '__meta_kubernetes_pod_label_(.+)'
+    - action: replace
+      sourceLabels: [__meta_kubernetes_namespace]
+      targetLabel: namespace
+    - action: replace
+      sourceLabels: [__meta_kubernetes_pod_name]
+      targetLabel: pod_name
+  jobLabel: kubernetes-services
+  namespaceSelector:
+    any: true
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kubernetes-services-secure-monitor
+  labels:
+    monitoring: kube-services-secure
+    release: monitoring
+spec:
+  selector:
+    matchExpressions:
+    - {key: istio-prometheus-ignore, operator: DoesNotExist}
+  endpoints:
+  - interval: 15s
+    relabelings:
+    - action: keep
+      regex: 'true'
+      sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+    - action: keep
+      regex: '(([^;]+);([^;]*))|(([^;]*);(true))'
+      # sidecar status annotation is added by sidecar injector and
+      # istio_workload_mtls_ability can be specifically placed on a pod to indicate its ability to receive mtls traffic.
+      sourceLabels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_istio_mtls]
+    - action: drop
+      regex: '(http)'
+      sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+    - action: replace
+      regex: '(.+)'
+      sourceLabels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+      targetLabel: __metrics_path__
+    - action: keep # otherwise an extra target with ':443' is added for https scheme
+      regex: '([^:]+):(\d+)'
+      sourceLabels: [__address__] # Only keep address that is host:port
+    - action: replace
+      regex: '([^:]+)(?::\d+)?;(\d+)'
+      replacement: $1:$2
+      sourceLabels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+      targetLabel: __address__
+    - action: labelmap
+      regex: '__meta_kubernetes_pod_label_(.+)'
+    - action: replace
+      sourceLabels: [__meta_kubernetes_namespace]
+      targetLabel: namespace
+    - action: replace
+      sourceLabels: [__meta_kubernetes_pod_name]
+      targetLabel: pod_name
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/secrets/istio.prometheus/root-cert.pem
+      certFile: /etc/prometheus/secrets/istio.prometheus/cert-chain.pem
+      insecureSkipVerify: true # prometheus does not support secure naming.
+      keyFile: /etc/prometheus/secrets/istio.prometheus/key.pem
+  jobLabel: kubernetes-services-secure
+  namespaceSelector:
+    any: true
+---
+# Source: prometheusOperator/templates/servicemonitors.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: kubelet
+  labels:
+    monitoring: kubelet-monitor
+    release: monitoring
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kubelet
+  endpoints:
+  - port: http-metrics
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 15s
+    scheme: http
+    tlsConfig:
+      insecureSkipVerify: true
+  - port: http-metrics
+    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 15s
+    metricRelabelings:
+    - action: drop
+      regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)
+      sourceLabels:
+      - __name__
+    path: /metrics/cadvisor
+    relabelings:
+    - action: replace
+      replacement: kubernetes-cadvisor
+      sourceLabels: [job]
+      targetLabel: job
+    scheme: http
+    tlsConfig:
+      insecureSkipVerify: true
+  jobLabel: k8s-app
+  namespaceSelector:
+    matchNames:
+    - kube-system

instance/flux-system/kustomizations/cert-manager.yaml

@@ -9,3 +9,6 @@ spec:
     apiVersion: helm.toolkit.fluxcd.io/v2beta1
     kind: HelmRelease
   path: './instance/cert-manager' # {"$kpt-set":"env-cert-manager"}
+  dependsOn:
+    - name: bigbang-monitoring
+      namespace: flux-system

instance/istio-system/Kptfile

@@ -33,3 +33,11 @@ openAPI:
           values:
           - marker: ${hostname}
             ref: '#/definitions/io.k8s.cli.setters.hostname'
+    io.k8s.cli.substitutions.kiali-hostname:
+      x-k8s-cli:
+        substitution:
+          name: kiali-hostname
+          pattern: kiali.${hostname}
+          values:
+          - marker: ${hostname}
+            ref: '#/definitions/io.k8s.cli.setters.hostname'