Merge branch 'feat/flux_observability' into 'master'

Feat: updating flux observability

See merge request platform-one/big-bang/bigbang!1896

chart/dashboards/flux/Kptfile

 apiVersion: kpt.dev/v1alpha1
 kind: Kptfile
 metadata:
-  name: dashboards
+  name: flux
 upstream:
   type: git
   git:
-    commit: 4b4e6b1be3413358654e74f98bac5339aed28868
+    commit: fe3e0efcf1060bd5b3332b1d717243401b34c305
     repo: https://github.com/fluxcd/flux2
-    directory: /manifests/monitoring/grafana/dashboards
-    ref: v0.26.1
+    directory: /manifests/monitoring/monitoring-config/dashboards
+    ref: v0.31.5

chart/dashboards/flux/cluster.json

@@ -9,13 +9,30 @@
         "iconColor": "rgba(0, 211, 255, 1)",
         "name": "Annotations & Alerts",
         "type": "dashboard"
+      },
+      {
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "iconColor": "red",
+        "name": "flux events",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "flux"
+          ],
+          "type": "tags"
+        }
       }
     ]
   },
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "iteration": 1636369574387,
+  "iteration": 1652337714814,
   "links": [],
   "panels": [
     {
@@ -290,6 +307,8 @@
       "id": 8,
       "options": {
         "displayMode": "gradient",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -352,6 +371,8 @@
       "id": 31,
       "options": {
         "displayMode": "gradient",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -398,25 +419,21 @@
       "fieldConfig": {
         "defaults": {
           "custom": {
-            "align": null,
-            "filterable": true
+            "displayMode": "auto",
+            "filterable": true,
+            "inspect": false
           },
           "mappings": [
             {
-              "from": "",
-              "id": 1,
-              "text": "Ready",
-              "to": "",
-              "type": 1,
-              "value": "0"
-            },
-            {
-              "from": "",
-              "id": 2,
-              "text": "Not Ready",
-              "to": "",
-              "type": 1,
-              "value": "1"
+              "options": {
+                "0": {
+                  "text": "Ready"
+                },
+                "1": {
+                  "text": "Not Ready"
+                }
+              },
+              "type": "value"
             }
           ],
           "thresholds": {
@@ -453,14 +470,27 @@
         ]
       },
       "gridPos": {
-        "h": 8,
+        "h": 11,
         "w": 12,
         "x": 0,
         "y": 10
       },
       "id": 33,
       "options": {
-        "showHeader": true
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Status"
+          }
+        ]
       },
       "pluginVersion": "7.5.5",
       "targets": [
@@ -487,11 +517,12 @@
               "app": true,
               "container": true,
               "endpoint": true,
-              "exported_namespace": true,
+              "exported_namespace": false,
               "instance": true,
               "job": true,
               "kubernetes_namespace": true,
               "kubernetes_pod_name": true,
+              "namespace": true,
               "pod": true,
               "pod_template_hash": true,
               "status": true,
@@ -500,9 +531,10 @@
             "indexByName": {},
             "renameByName": {
               "Value": "Status",
+              "exported_namespace": "Namespace",
               "kind": "Kind",
               "name": "Name",
-              "namespace": "Namespace"
+              "namespace": "Operator Namespace"
             }
           }
         }
@@ -515,25 +547,21 @@
       "fieldConfig": {
         "defaults": {
           "custom": {
-            "align": null,
-            "filterable": true
+            "displayMode": "auto",
+            "filterable": true,
+            "inspect": false
           },
           "mappings": [
             {
-              "from": "",
-              "id": 1,
-              "text": "Ready",
-              "to": "",
-              "type": 1,
-              "value": "0"
-            },
-            {
-              "from": "",
-              "id": 2,
-              "text": "Not Ready",
-              "to": "",
-              "type": 1,
-              "value": "1"
+              "options": {
+                "0": {
+                  "text": "Ready"
+                },
+                "1": {
+                  "text": "Not Ready"
+                }
+              },
+              "type": "value"
             }
           ],
           "thresholds": {
@@ -570,14 +598,27 @@
         ]
       },
       "gridPos": {
-        "h": 8,
+        "h": 11,
         "w": 12,
         "x": 12,
         "y": 10
       },
       "id": 34,
       "options": {
-        "showHeader": true
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "Status"
+          }
+        ]
       },
       "pluginVersion": "7.5.5",
       "targets": [
@@ -604,11 +645,12 @@
               "app": true,
               "container": true,
               "endpoint": true,
-              "exported_namespace": true,
+              "exported_namespace": false,
               "instance": true,
               "job": true,
               "kubernetes_namespace": true,
               "kubernetes_pod_name": true,
+              "namespace": true,
               "pod": true,
               "pod_template_hash": true,
               "status": true,
@@ -617,9 +659,10 @@
             "indexByName": {},
             "renameByName": {
               "Value": "Status",
+              "exported_namespace": "Namespace",
               "kind": "Kind",
               "name": "Name",
-              "namespace": "Namespace"
+              "namespace": "Operator Namespace"
             }
           }
         }
@@ -633,7 +676,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 18
+        "y": 21
       },
       "id": 17,
       "panels": [],
@@ -657,7 +700,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 19
+        "y": 22
       },
       "hiddenSeries": false,
       "id": 27,
@@ -757,7 +800,7 @@
         "h": 8,
         "w": 24,
         "x": 0,
-        "y": 27
+        "y": 30
       },
       "hiddenSeries": false,
       "id": 35,
@@ -841,34 +884,14 @@
       }
     }
   ],
-  "refresh": "",
-  "schemaVersion": 27,
+  "refresh": "30s",
+  "schemaVersion": 36,
   "style": "light",
   "tags": [
     "flux"
   ],
   "templating": {
     "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "Prometheus",
-          "value": "Prometheus"
-        },
-        "description": null,
-        "error": null,
-        "hide": 2,
-        "includeAll": false,
-        "label": null,
-        "multi": false,
-        "name": "DS_PROMETHEUS",
-        "options": [],
-        "query": "prometheus",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
       {
         "allValue": "",
         "current": {
@@ -916,7 +939,7 @@
             "$__all"
           ]
         },
-        "datasource": null,
+        "datasource": "$DS_PROMETHEUS",
         "definition": "label_values(gotk_reconcile_condition, exported_namespace)",
         "description": null,
         "error": null,
@@ -939,6 +962,24 @@
         "tagsQuery": "",
         "type": "query",
         "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
       }
     ]
   },
@@ -959,8 +1000,7 @@
       "1d"
     ]
   },
-  "timezone": "",
   "title": "Flux Cluster Stats",
   "uid": "flux-cluster",
-  "version": 1
+  "version": 3
 }

chart/dashboards/flux/control-plane.json

@@ -15,6 +15,23 @@
           "type": "dashboard"
         },
         "type": "dashboard"
+      },
+      {
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "iconColor": "red",
+        "name": "flux events",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "flux"
+          ],
+          "type": "tags"
+        }
       }
     ]
   },

chart/dashboards/flux/logs.json

+{
+  "__inputs": [
+    {
+      "name": "DS_LOKI",
+      "label": "Loki",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "loki",
+      "pluginName": "Loki"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      },
+      {
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "iconColor": "red",
+        "name": "flux events",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "flux"
+          ],
+          "type": "tags"
+        }
+      }
+    ]
+  },
+  "description": "Flux logs collected from Kubernetes, stored in Loki",
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 29,
+  "iteration": 1653748775696,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": "${DS_LOKI}",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "hidden",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": "${DS_LOKI}",
+          "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))",
+          "instant": false,
+          "legendFormat": "Log count",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_LOKI}",
+      "description": "Logs from services running in Kubernetes",
+      "gridPos": {
+        "h": 25,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "numbers",
+        "enableLogDetails": false,
+        "prettifyLogMessage": true,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": "${DS_LOKI}",
+          "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"",
+          "refId": "A"
+        }
+      ],
+      "type": "logs"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 36,
+  "style": "light",
+  "tags": [
+    "flux"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "",
+          "value": ""
+        },
+        "description": "String to search for",
+        "hide": 0,
+        "label": "Search Query",
+        "name": "query",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "skipUrlSync": false,
+        "type": "textbox"
+      },
+      {
+        "allValue": "info|error",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "level",
+        "options": [
+          {
+            "selected": true,
+            "text": "All",
+            "value": "$__all"
+          },
+          {
+            "selected": false,
+            "text": "info",
+            "value": "info"
+          },
+          {
+            "selected": false,
+            "text": "error",
+            "value": "error"
+          }
+        ],
+        "query": "info,error",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${DS_LOKI}",
+        "definition": "label_values(app)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "controller",
+        "options": [],
+        "query": "label_values(app)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": [
+            "flux-system"
+          ],
+          "value": [
+            "flux-system"
+          ]
+        },
+        "datasource": "${DS_LOKI}",
+        "definition": "label_values(namespace)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "namespace",
+        "options": [],
+        "query": "label_values(namespace)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": "${DS_LOKI}",
+        "definition": "label_values(stream)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "stream",
+        "options": [],
+        "query": "label_values(stream)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "Loki",
+          "value": "Loki"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_LOKI",
+        "options": [],
+        "query": "loki",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Flux Logs",
+  "uid": "flux-logs",
+  "version": 2
+}

chart/templates/monitoring/flux-dashboards.yaml

@@ -11,4 +11,8 @@ data:
     {{ .Files.Get "dashboards/flux/cluster.json"  | nindent 4 }}
   flux-control-plane-dashboard.json: |
     {{ .Files.Get "dashboards/flux/control-plane.json" | nindent 4 }}
+  {{- if .Values.loki.enabled }}
+  flux-logs-dashboard.json: |
+    {{ .Files.Get "dashboards/flux/logs.json" | nindent 4 }}
+  {{- end }}
 {{- end }}

chart/templates/monitoring/flux/alert.yaml

+{{- if and .Values.monitoring.enabled (dig "grafana" "enabled" true .Values.monitoring.values) }}
+apiVersion: notification.toolkit.fluxcd.io/v1beta1
+kind: Alert
+metadata:
+  name: grafana
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: monitoring
+    app.kubernetes.io/component: "core"
+    {{- include "commonLabels" . | nindent 4}}
+spec:
+  providerRef:
+    name: grafana
+  eventSeverity: info
+  eventSources:
+    - kind: GitRepository
+      name: '*'
+      namespace: bigbang
+{{- end }}
\ No newline at end of file

chart/templates/monitoring/flux/grafana-auth-secret.yaml

+{{- if and .Values.monitoring.enabled (dig "grafana" "enabled" true .Values.monitoring.values) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: grafana-flux-auth
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: monitoring
+    app.kubernetes.io/component: "core"
+    {{- include "commonLabels" . | nindent 4}}
+type: kubernetes.io/opaque
+stringData:
+{{- if (dig "grafana" "admin" "existingSecret" "" .Values.monitoring.values) }}
+  {{- $passwordKey := (dig "grafana" "admin" "passwordKey" "admin-password" .Values.monitoring.values) }}
+  {{- $userKey := (dig "grafana" "admin" "userKey" "admin-user" .Values.monitoring.values) }}
+  {{- with lookup "v1" "Secret" "monitoring" .Values.monitoring.values.grafana.admin.existingSecret }}
+  username: {{ (get .data $userKey | b64dec) | default "admin" }}
+  password: {{ (get .data $passwordKey | b64dec) | default "prom-operator" }}
+  {{- end }}
+{{- else }}
+  username: {{ dig "grafana" "adminUser" "admin" .Values.monitoring.values }}
+  password: {{ dig "grafana" "adminPassword" "prom-operator" .Values.monitoring.values }}
+{{- end }}
+{{- end }}
\ No newline at end of file

chart/templates/monitoring/flux/ingress-flux.yaml

+{{- if and .Values.monitoring.enabled (dig "grafana" "enabled" true .Values.monitoring.values) }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-from-flux
+  namespace: monitoring
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: grafana
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          app: notification-controller
+      namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: flux-system
+    ports:
+    - port: 3000
+      protocol: TCP
+{{- end }}
\ No newline at end of file

chart/templates/monitoring/flux/provider.yaml

+{{- if and .Values.monitoring.enabled (dig "grafana" "enabled" true .Values.monitoring.values) }}
+apiVersion: notification.toolkit.fluxcd.io/v1beta1
+kind: Provider
+metadata:
+  name: grafana
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: monitoring
+    app.kubernetes.io/component: "core"
+    {{- include "commonLabels" . | nindent 4}}
+spec:
+  type: grafana
+  address: "http://monitoring-monitoring-grafana.monitoring/api/annotations"
+  secretRef:
+    name: grafana-flux-auth
+{{- end }}
\ No newline at end of file