From cb0a3ad6e57d889ca82710375c712b42a5d65ed8 Mon Sep 17 00:00:00 2001
From: joshwolf <joshua.wolf@aero.org>
Date: Mon, 2 Nov 2020 02:37:52 +0000
Subject: [PATCH] wire in fluentd/fluentbit with logging-operator logging
pipelines
---
base/istio/flows/ingress-gateway.yaml | 15 +++++++++++
base/istio/flows/kustomization.yaml | 2 ++
base/istio/istio-system/kustomization.yaml | 1 +
base/logging/efk/elasticsearch.yaml | 6 ++++-
base/logging/efk/kibana.yaml | 6 ++---
base/logging/efk/kustomization.yaml | 2 ++
.../logging-operator/es-clusteroutput.yaml | 21 +++++++++++++++
.../efk/logging-operator/helmrelease.yaml | 20 ++++++++++++++
.../efk/logging-operator/kustomization.yaml | 5 ++++
.../logging/efk/logging-operator/logging.yaml | 11 ++++++++
base/monitoring/helmrelease.yaml | 2 +-
instance/flux-system/gitrepositories/init.sh | 27 ++++++++++++++-----
12 files changed, 106 insertions(+), 12 deletions(-)
create mode 100644 base/istio/flows/ingress-gateway.yaml
create mode 100644 base/istio/flows/kustomization.yaml
create mode 100644 base/logging/efk/logging-operator/es-clusteroutput.yaml
create mode 100644 base/logging/efk/logging-operator/helmrelease.yaml
create mode 100644 base/logging/efk/logging-operator/kustomization.yaml
create mode 100644 base/logging/efk/logging-operator/logging.yaml
diff --git a/base/istio/flows/ingress-gateway.yaml b/base/istio/flows/ingress-gateway.yaml
new file mode 100644
index 0000000000..1c8988e6d1
--- /dev/null
+++ b/base/istio/flows/ingress-gateway.yaml
@@ -0,0 +1,15 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+ name: istio-ingressgateway
+ namespace: istio-system
+spec:
+ filters:
+ - tag_normaliser: {}
+ globalOutputRefs:
+ - es
+ match:
+ - select:
+ labels:
+ app: istio-ingressgateway
+ istio: ingressgateway
diff --git a/base/istio/flows/kustomization.yaml b/base/istio/flows/kustomization.yaml
new file mode 100644
index 0000000000..80b1840497
--- /dev/null
+++ b/base/istio/flows/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+- ingress-gateway.yaml
diff --git a/base/istio/istio-system/kustomization.yaml b/base/istio/istio-system/kustomization.yaml
index 5c698ac90e..81250ebb04 100644
--- a/base/istio/istio-system/kustomization.yaml
+++ b/base/istio/istio-system/kustomization.yaml
@@ -1,3 +1,4 @@
+namespace: istio-system
resources:
- namespace.yaml
- istio.yaml
diff --git a/base/logging/efk/elasticsearch.yaml b/base/logging/efk/elasticsearch.yaml
index b5db22e557..c873c75cff 100644
--- a/base/logging/efk/elasticsearch.yaml
+++ b/base/logging/efk/elasticsearch.yaml
@@ -1,7 +1,7 @@
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
- name: elasticsearch
+ name: bigbang
spec:
image: registry1.dsop.io/ironbank/elastic/elasticsearch/elasticsearch:7.9.2
nodeSets:
@@ -36,8 +36,11 @@ spec:
# memory: 3Gi
# cpu: 2
+ sidecar.istio.io/rewriteAppHTTPProbers: "true"
traffic.sidecar.istio.io/excludeInboundPorts: "9300"
traffic.sidecar.istio.io/excludeOutboundPorts: "9300"
+ spec:
+ automountServiceAccountToken: true
- name: data
volumeClaimTemplates:
- metadata:
@@ -63,6 +66,7 @@ spec:
annotations:
fluentbit.io/exclude-istio-proxy: "true"
prometheus.istio.io/merge-metrics: "false"
+ sidecar.istio.io/rewriteAppHTTPProbers: "true"
traffic.sidecar.istio.io/excludeInboundPorts: "9300"
traffic.sidecar.istio.io/excludeOutboundPorts: "9300"
spec:
diff --git a/base/logging/efk/kibana.yaml b/base/logging/efk/kibana.yaml
index f2be2a67e1..2e8835f1b7 100644
--- a/base/logging/efk/kibana.yaml
+++ b/base/logging/efk/kibana.yaml
@@ -1,11 +1,11 @@
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
- name: kibana
+ name: bigbang
spec:
count: 1
elasticsearchRef:
- name: elasticsearch
+ name: bigbang
http:
tls:
selfSignedCertificate:
@@ -16,4 +16,4 @@ spec:
sidecar.istio.io/rewriteAppHTTPProbers: "true"
spec:
automountServiceAccountToken: true
- version: 7.8.1
+ version: 7.9.2
diff --git a/base/logging/efk/kustomization.yaml b/base/logging/efk/kustomization.yaml
index 6888218405..9b9352bf0e 100644
--- a/base/logging/efk/kustomization.yaml
+++ b/base/logging/efk/kustomization.yaml
@@ -2,3 +2,5 @@ resources:
- namespace.yaml
- elasticsearch.yaml
- kibana.yaml
+# Logging operator (declarative logging pipelines)
+- logging-operator
diff --git a/base/logging/efk/logging-operator/es-clusteroutput.yaml b/base/logging/efk/logging-operator/es-clusteroutput.yaml
new file mode 100644
index 0000000000..d8b2ae4976
--- /dev/null
+++ b/base/logging/efk/logging-operator/es-clusteroutput.yaml
@@ -0,0 +1,21 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: ClusterOutput
+metadata:
+ name: es
+spec:
+ elasticsearch:
+ port: 9200
+ buffer:
+ timekey: 1m
+ timekey_use_utc: true
+ timekey_wait: 30s
+ host: bigbang-es-http.logging.svc.cluster.local
+ password:
+ valueFrom:
+ secretKeyRef:
+ name: bigbang-es-elastic-user
+ key: elastic
+ scheme: https
+ ssl_verify: false
+ ssl_version: TLSv1_2
+ user: elastic
diff --git a/base/logging/efk/logging-operator/helmrelease.yaml b/base/logging/efk/logging-operator/helmrelease.yaml
new file mode 100644
index 0000000000..df5cf024cc
--- /dev/null
+++ b/base/logging/efk/logging-operator/helmrelease.yaml
@@ -0,0 +1,20 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+ name: logging-operator
+spec:
+ chart:
+ spec:
+ chart: logging-operator
+ interval: 5m
+ sourceRef:
+ name: banzaicloud
+ namespace: flux-system
+ kind: HelmRepository
+ version: 3.7.0
+ interval: 2m
+ values:
+ createCustomResource: false
+ monitoring:
+ serviceMonitor:
+ enabled: true
diff --git a/base/logging/efk/logging-operator/kustomization.yaml b/base/logging/efk/logging-operator/kustomization.yaml
new file mode 100644
index 0000000000..227bbdf9fb
--- /dev/null
+++ b/base/logging/efk/logging-operator/kustomization.yaml
@@ -0,0 +1,5 @@
+resources:
+- helmrelease.yaml
+# system wide pipeline
+- es-clusteroutput.yaml
+- logging.yaml
diff --git a/base/logging/efk/logging-operator/logging.yaml b/base/logging/efk/logging-operator/logging.yaml
new file mode 100644
index 0000000000..c208ccf62a
--- /dev/null
+++ b/base/logging/efk/logging-operator/logging.yaml
@@ -0,0 +1,11 @@
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Logging
+metadata:
+ name: bigbang
+spec:
+ controlNamespace: logging
+ fluentbit:
+ image:
+ repository: fluent/fluent-bit
+ tag: 1.6.1
+ fluentd: {}
diff --git a/base/monitoring/helmrelease.yaml b/base/monitoring/helmrelease.yaml
index 40476b2dda..b9eb2e5b7f 100644
--- a/base/monitoring/helmrelease.yaml
+++ b/base/monitoring/helmrelease.yaml
@@ -12,7 +12,7 @@ spec:
name: prometheus-community
namespace: flux-system
kind: HelmRepository
- version: 10.1.2
+ version: 10.3.4
interval: 2m
values:
alertmanager:
diff --git a/instance/flux-system/gitrepositories/init.sh b/instance/flux-system/gitrepositories/init.sh
index e17e920113..4d5f221697 100755
--- a/instance/flux-system/gitrepositories/init.sh
+++ b/instance/flux-system/gitrepositories/init.sh
@@ -31,18 +31,30 @@ deploy_flux() {
kubectl wait --for=condition=available --timeout=60s --all deployments -n flux-system
}
-deploy_umbrella() {
- info "Bootstrapping from the current repo"
-
- # apply the repository with the current branch
+deploy_this_repo() {
export branch=$(git rev-parse --abbrev-ref HEAD)
export repo=$(git config --get remote.origin.url)
export env="dev"
- curdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ info "Deploying the current repo: ${branch} targetting the branch: ${repo}"
kustomize build "${curdir}" | envsubst | kubectl apply -f -
- kustomize build "${curdir}/.." | kubectl apply -f -
+}
+
+bootstrap() {
+ curdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+ deploy_this_repo
+
+ case "$1" in
+ "dev")
+ kustomize build base/flux/chart-repositories | kubectl apply -f -
+ info "Stopping at empty flux"
+ ;;
+ *)
+ info "Bootstrapping from the current repo"
+ kustomize build "${curdir}/.." | kubectl apply -f -
+ esac
}
{
@@ -52,5 +64,6 @@ deploy_umbrella() {
need "git"
deploy_flux
- deploy_umbrella
+
+ bootstrap "$1"
}
\ No newline at end of file
--
GitLab