Resolve "Elasticsearch AutoRollingUpgrade is getting error"

General MR

Summary

This MR, though originally intended to "fix" the elastic upgrade job, is actually removing the job altogether. The job is unnecessary as the ECK operator (which this package depends on) already handles upgrade logic for ElasticSearch and Kibana instances deployed via its CRDs.

Relevant logs/screenshots

After an upgrade from 1.18.0-bb.5 to 1.19.0-bb.1 with autoRollingUpgrade.enabled set to false:

❯ kubectl get es -n logging
NAME         HEALTH   NODES   VERSION   PHASE   AGE
logging-ek   green    7       8.15.1    Ready   100m

❯ kubectl get pod -n logging
NAME                                  READY   STATUS    RESTARTS   AGE
logging-ek-es-data-0                  2/2     Running   0          62m
logging-ek-es-data-1                  2/2     Running   0          64m
logging-ek-es-data-2                  2/2     Running   0          66m
logging-ek-es-data-3                  2/2     Running   0          69m
logging-ek-es-master-0                2/2     Running   0          54m
logging-ek-es-master-1                2/2     Running   0          57m
logging-ek-es-master-2                2/2     Running   0          59m
logging-ek-kb-55865bdcf-46ptm         2/2     Running   0          54m
logging-ek-kb-55865bdcf-df2tp         2/2     Running   0          54m
logging-ek-kb-55865bdcf-ktrjt         2/2     Running   0          54m

Linked Issue

#210 (closed)

Upgrade Notices

N/A

Closes #210 (closed)

chart/templates/bigbang/upgrade-job.yaml

-{{- if .Values.autoRollingUpgrade.enabled }}
-{{- if .Values.networkPolicies.enabled }}
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: api-egress-upgrade-job
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-10"
-    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed,before-hook-creation
-spec:
-  egress:
-  - to:
-    - ipBlock:
-        cidr: {{ .Values.networkPolicies.controlPlaneCidr }}
-        {{- if eq .Values.networkPolicies.controlPlaneCidr "0.0.0.0/0" }}
-        # ONLY Block requests to AWS metadata IP
-        except:
-        - 169.254.169.254/32
-        {{- end }}
-  podSelector:
-    matchLabels:
-      app.kubernetes.io/name: bigbang-ek-upgrade-job
-  policyTypes:
-  - Egress
-{{- end }}
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ .Release.Name }}-bb-upgrade
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-10"
-    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed,before-hook-creation
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: bb-{{ .Release.Name }}-upgrade-view
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-10"
-    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed,before-hook-creation
-rules:
-- apiGroups: ["elasticsearch.k8s.elastic.co"]
-  resources: ["elasticsearches"]
-  verbs: ["get", "list", "watch"]
-- apiGroups: ["kibana.k8s.elastic.co"]
-  resources: ["kibanas"]
-  verbs: ["get", "list", "update", "patch"]
-- apiGroups: ["apps"]
-  resources: ["deployments"]
-  verbs: ["get", "list", "delete"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ .Release.Name }}-bb-upgrade
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-10"
-    "helm.sh/hook-delete-policy": hook-succeeded,hook-failed,before-hook-creation
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: bb-{{ .Release.Name }}-upgrade-view
-subjects:
-- kind: ServiceAccount
-  name: {{ .Release.Name }}-bb-upgrade
-  namespace: {{ .Release.Namespace }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: bb-{{ .Release.Name }}-upgrade
-  namespace: {{ .Release.Namespace }}
-  annotations:
-    "helm.sh/hook": post-upgrade
-    "helm.sh/hook-weight": "-5"
-spec:
-  backoffLimit: 3
-  ttlSecondsAfterFinished: 480
-  template:
-    metadata:
-      name: bb-{{ .Release.Name }}-upgrade
-      labels:
-        app.kubernetes.io/name: bigbang-ek-upgrade-job
-    spec:
-      securityContext:
-        runAsUser: 1000
-        runAsGroup: 1000
-      serviceAccountName: {{ .Release.Name }}-bb-upgrade
-      containers:
-        - name: bb-{{ .Release.Name }}-upgrade
-          image: {{ $.Values.upgradeJob.image.repository }}:{{ $.Values.upgradeJob.image.tag }}
-          command:
-            - /bin/bash
-            - -ec
-            - |
-              if [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.9.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 7.10.* ]]; then
-                export ES_DESIRED_VERSION="7.10.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.10.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 7.12.* ]]; then
-                export ES_DESIRED_VERSION="7.12.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.12.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 7.13.* ]]; then
-                export ES_DESIRED_VERSION="7.13.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.13.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 7.14.* ]]; then
-                export ES_DESIRED_VERSION="7.14.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.13.* || $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.14.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 7.16.* ]]; then
-                export ES_DESIRED_VERSION="7.16.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.16.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 7.17.* ]]; then
-                export ES_DESIRED_VERSION="7.17.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 7.17.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.2.* ]]; then
-                export ES_DESIRED_VERSION="8.2.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.2.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.3.* ]]; then
-                export ES_DESIRED_VERSION="8.3.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.3.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.4.* ]]; then
-                export ES_DESIRED_VERSION="8.4.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.4.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.5.* ]]; then
-                export ES_DESIRED_VERSION="8.5.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.5.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.6.* ]]; then
-                export ES_DESIRED_VERSION="8.6.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.6.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.7.* ]]; then
-                export ES_DESIRED_VERSION="8.7.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.7.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.8.* ]]; then
-                export ES_DESIRED_VERSION="8.8.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.8.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.9.* ]]; then
-                export ES_DESIRED_VERSION="8.9.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.9.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.10.* ]]; then
-                export ES_DESIRED_VERSION="8.10.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.10.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.11.* ]]; then
-                export ES_DESIRED_VERSION="8.11.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.11.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.12.* ]]; then
-                export ES_DESIRED_VERSION="8.12.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.12.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.13.* ]]; then
-                export ES_DESIRED_VERSION="8.13.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.13.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.14.* ]]; then
-                export ES_DESIRED_VERSION="8.14.*"
-                export ROLLING_UPGRADE="true"
-              elif [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == 8.14.* ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.spec.version}') == 8.15.* ]]; then
-                export ES_DESIRED_VERSION="8.15.*"
-                export ROLLING_UPGRADE="true"
-              fi
-
-              if [[ "$ROLLING_UPGRADE" == "true" ]]; then
-                echo "Running Rolling Upgrade Prep Commands"
-                kubectl annotate --overwrite kibana {{ .Release.Name }} -n {{ .Release.Namespace }} 'eck.k8s.elastic.co/managed=false'
-                kubectl delete deployment -l kibana.k8s.elastic.co/name={{ .Release.Name }},common.k8s.elastic.co/type=kibana -n {{ .Release.Namespace }}
-
-                curl -XPUT -ku "elastic:$elastic" "https://{{ .Release.Name }}-es-http.{{ .Release.Namespace }}.svc:9200/_cluster/settings?pretty" -H 'Content-Type: application/json' -d' { "persistent": { "cluster.routing.allocation.enable": "primaries" } }'
-                curl -XPOST -ku "elastic:$elastic" "https://{{ .Release.Name }}-es-http.{{ .Release.Namespace }}.svc:9200/_flush/synced?pretty"
-                echo "Rolling Upgrade Prep Commands Completed"
-              else
-                echo "No Upgrade Prep Necessary :D"
-                if {{ .Values.istio.enabled }}; then
-                  echo "Killing Istio Sidecar"
-                  curl -X POST http://localhost:15020/quitquitquit
-                fi
-                exit 0
-              fi
-
-              until [[ $( kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.phase}' ) == "Ready" ]] && [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.version}') == $ES_DESIRED_VERSION ]]; do
-                echo "ES cluster version $ES_DESIRED_VERSION not yet Ready" && sleep 10;
-              done
-              if [[ $( curl -ku "elastic:$elastic" -k "https://{{ .Release.Name }}-es-http.{{ .Release.Namespace }}.svc:9200/_cluster/settings?pretty" | jq '.persistent.cluster.routing.allocation.enable' | tr -d '"' ) == "primaries" ]]; then
-                echo "Running Post-Upgrade Commands"
-                curl -XPUT -ku "elastic:$elastic" "https://{{ .Release.Name }}-es-http.{{ .Release.Namespace }}.svc:9200/_cluster/settings?pretty" -H 'Content-Type: application/json' -d' { "persistent": { "cluster.routing.allocation.enable": null } }'
-                until [[ $(kubectl get elasticsearch {{ .Release.Name }} -n {{ .Release.Namespace }} -o jsonpath='{.status.health}') == "green" ]]; do
-                  echo "Waiting for ES cluster to be green" && sleep 5;
-                done
-                kubectl annotate kibana {{ .Release.Name }} -n {{ .Release.Namespace }} 'eck.k8s.elastic.co/managed-'
-                echo "Post-Upgrade Commands completed"
-                if {{ .Values.istio.enabled }}; then
-                  echo "Killing Istio Sidecar"
-                  curl -X POST http://localhost:15020/quitquitquit
-                fi
-                exit 0
-              else
-                kubectl annotate kibana {{ .Release.Name }} -n {{ .Release.Namespace }} 'eck.k8s.elastic.co/managed-'
-                echo "No post-upgrade commands necessary"
-                if {{ .Values.istio.enabled }}; then
-                  echo "Killing Istio Sidecar"
-                  curl -X POST http://localhost:15020/quitquitquit
-                fi
-                exit 0
-              fi
-          resources:
-            requests:
-              cpu: 100m
-              memory: 256Mi
-            limits:
-              cpu: 100m
-              memory: 256Mi
-          envFrom:
-            - secretRef:
-                name: {{ .Release.Name }}-es-elastic-user
-          securityContext:
-            capabilities:
-              drop:
-                - ALL
-      {{- with $.Values.elasticsearch.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      restartPolicy: OnFailure
-{{- end }}