From e22a9ea70f0e9cdee27a09288c577a4e659ec75e Mon Sep 17 00:00:00 2001 From: Michael Martin <michaelmartin@seed-innovations.com> Date: Wed, 24 Jan 2024 09:24:05 -0700 Subject: [PATCH] Revert "Revert "Holocron"" This reverts commit bc17cebefa7164ae6bd1f1f298ba8df36e4581cd. --- chart/templates/holocron/gitlab-ingress.yaml | 22 +++ chart/templates/holocron/gitrepository.yaml | 15 ++ chart/templates/holocron/helmrelease.yaml | 71 +++++++ chart/templates/holocron/imagepullsecret.yaml | 14 ++ chart/templates/holocron/jira-ingress.yaml | 22 +++ chart/templates/holocron/namespace.yaml | 14 ++ .../templates/holocron/sonarqube-ingress.yaml | 22 +++ chart/templates/holocron/values.yaml | 183 ++++++++++++++++++ chart/templates/kyverno-policies/values.yaml | 16 +- chart/values.schema.json | 7 + chart/values.yaml | 83 +++++++- .../package-architecture/holocron.md | 20 ++ 12 files changed, 483 insertions(+), 6 deletions(-) create mode 100644 chart/templates/holocron/gitlab-ingress.yaml create mode 100644 chart/templates/holocron/gitrepository.yaml create mode 100644 chart/templates/holocron/helmrelease.yaml create mode 100644 chart/templates/holocron/imagepullsecret.yaml create mode 100644 chart/templates/holocron/jira-ingress.yaml create mode 100644 chart/templates/holocron/namespace.yaml create mode 100644 chart/templates/holocron/sonarqube-ingress.yaml create mode 100644 chart/templates/holocron/values.yaml create mode 100644 docs/understanding-bigbang/package-architecture/holocron.md diff --git a/chart/templates/holocron/gitlab-ingress.yaml b/chart/templates/holocron/gitlab-ingress.yaml new file mode 100644 index 0000000000..9656926323 --- /dev/null +++ b/chart/templates/holocron/gitlab-ingress.yaml @@ -0,0 +1,22 @@ +{{- $pkg := "holocron" }} +{{- if and .Values.networkPolicies.enabled .Values.addons.gitlab.enabled (get .Values.addons $pkg).enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $pkg }}-allow-ingress + namespace: gitlab +spec: + podSelector: + matchLabels: + app: webservice + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + app.kubernetes.io/name: holocron + podSelector: + matchLabels: + role: collector +{{- end }} diff --git a/chart/templates/holocron/gitrepository.yaml b/chart/templates/holocron/gitrepository.yaml new file mode 100644 index 0000000000..616274f5c3 --- /dev/null +++ b/chart/templates/holocron/gitrepository.yaml @@ -0,0 +1,15 @@ +{{- $pkg := "holocron" }} +{{- if and (eq (get .Values.addons $pkg).sourceType "git") (not .Values.offline) (get .Values.addons $pkg).enabled }} +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: {{ $pkg }} + namespace: {{ .Release.Namespace }} +spec: + interval: {{ .Values.flux.interval }} + url: {{ (get .Values.addons $pkg).git.repo }} + ref: + {{- include "validRef" (get .Values.addons $pkg).git | nindent 4 }} + {{ include "gitIgnore" . }} + {{- include "gitCreds" . | nindent 2 }} +{{- end }} diff --git a/chart/templates/holocron/helmrelease.yaml b/chart/templates/holocron/helmrelease.yaml new file mode 100644 index 0000000000..3384999f86 --- /dev/null +++ b/chart/templates/holocron/helmrelease.yaml @@ -0,0 +1,71 @@ +{{- $pkg := "holocron" }} +{{- $fluxSettingsHolocron := merge (get .Values.addons $pkg).flux .Values.flux -}} +{{- if (get .Values.addons $pkg).enabled }} +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: {{ $pkg }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ $pkg }} + {{- include "commonLabels" . | nindent 4}} + annotations: + checksum/bigbang-values: {{ include (print $.Template.BasePath "/holocron/values.yaml") . | sha256sum }} +spec: + releaseName: {{ $pkg }} + targetNamespace: {{ $pkg }} + chart: + spec: + {{- if eq (get .Values.addons $pkg).sourceType "git" }} + chart: {{ (get .Values.addons $pkg).git.path }} + sourceRef: + kind: GitRepository + name: holocron + namespace: {{ .Release.Namespace }} + {{- else }} + chart: {{ (get .Values.addons $pkg).helmRepo.chartName }} + version: {{ (get .Values.addons $pkg).helmRepo.tag }} + sourceRef: + kind: HelmRepository + name: {{ (get .Values.addons $pkg).helmRepo.repoName }} + namespace: {{ .Release.Namespace }} + {{- end }} + interval: 5m + + {{- toYaml $fluxSettingsHolocron | nindent 2 }} + + {{- if (get .Values.addons $pkg).postRenderers }} + postRenderers: + {{ toYaml (get .Values.addons $pkg).postRenderers | nindent 4 }} + {{- end }} + valuesFrom: + - name: {{ .Release.Name }}-{{ $pkg }}-values + kind: Secret + valuesKey: "common" + - name: {{ .Release.Name }}-{{ $pkg }}-values + kind: Secret + valuesKey: "defaults" + - name: {{ .Release.Name }}-{{ $pkg }}-values + kind: Secret + valuesKey: "overlays" + + {{- if or .Values.istio.enabled .Values.kyvernoPolicies.enabled .Values.monitoring.enabled .Values.addons.gitlab.enabled}} + dependsOn: + {{- if .Values.istio.enabled }} + - name: istio + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.kyvernoPolicies.enabled }} + - name: kyverno-policies + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.monitoring.enabled }} + - name: monitoring + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.addons.gitlab.enabled }} + - name: gitlab + namespace: {{ .Release.Namespace }} + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/templates/holocron/imagepullsecret.yaml b/chart/templates/holocron/imagepullsecret.yaml new file mode 100644 index 0000000000..8b4fae63f9 --- /dev/null +++ b/chart/templates/holocron/imagepullsecret.yaml @@ -0,0 +1,14 @@ +{{- $pkg := "holocron" }} +{{- if and (get .Values.addons $pkg).enabled ( include "imagePullSecret" . ) }} +apiVersion: v1 +kind: Secret +metadata: + name: private-registry + namespace: {{ $pkg }} + labels: + app.kubernetes.io/name: {{ $pkg }} + {{- include "commonLabels" . | nindent 4 }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ template "imagePullSecret" . }} +{{- end }} diff --git a/chart/templates/holocron/jira-ingress.yaml b/chart/templates/holocron/jira-ingress.yaml new file mode 100644 index 0000000000..84e5b7052a --- /dev/null +++ b/chart/templates/holocron/jira-ingress.yaml @@ -0,0 +1,22 @@ +{{- $pkg := "holocron" }} +{{- if and .Values.networkPolicies.enabled (get .Values.addons $pkg).jira.enabled (get .Values.addons $pkg).enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $pkg }}-allow-ingress + namespace: jira +spec: + podSelector: + matchLabels: + app: jira + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + app.kubernetes.io/name: holocron + podSelector: + matchLabels: + role: collector +{{- end }} \ No newline at end of file diff --git a/chart/templates/holocron/namespace.yaml b/chart/templates/holocron/namespace.yaml new file mode 100644 index 0000000000..c95b54af80 --- /dev/null +++ b/chart/templates/holocron/namespace.yaml @@ -0,0 +1,14 @@ +{{- $pkg := "holocron" }} +{{- if and (get .Values.addons $pkg).enabled (not (get .Values.addons $pkg).collectorAuth.existingSecret) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $pkg }} + labels: + {{- if .Values.istio.enabled }} + istio-injection: enabled + {{- end }} + app.kubernetes.io/name: {{ $pkg }} + app.kubernetes.io/component: "core" + {{- include "commonLabels" . | nindent 4}} +{{- end }} diff --git a/chart/templates/holocron/sonarqube-ingress.yaml b/chart/templates/holocron/sonarqube-ingress.yaml new file mode 100644 index 0000000000..5787f7fa72 --- /dev/null +++ b/chart/templates/holocron/sonarqube-ingress.yaml @@ -0,0 +1,22 @@ +{{- $pkg := "holocron" }} +{{- if and .Values.networkPolicies.enabled .Values.addons.sonarqube.enabled (get .Values.addons $pkg).enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $pkg }}-allow-ingress + namespace: sonarqube +spec: + podSelector: + matchLabels: + app: sonarqube + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + app.kubernetes.io/name: holocron + podSelector: + matchLabels: + role: collector +{{- end }} diff --git a/chart/templates/holocron/values.yaml b/chart/templates/holocron/values.yaml new file mode 100644 index 0000000000..ac9ea16902 --- /dev/null +++ b/chart/templates/holocron/values.yaml @@ -0,0 +1,183 @@ +{{- $pkg := "holocron" }} + +{{- /* Create secret */ -}} +{{- if (get .Values.addons $pkg).enabled }} +{{- include "values-secret" (dict "root" $ "package" (get .Values.addons $pkg) "name" $pkg "defaults" (include (printf "bigbang.defaults.%s" $pkg) .)) }} +{{- end }} + +{{- define "bigbang.defaults.holocron" -}} + +imagePullSecrets: +- name: private-registry +imagePullPolicy: {{ .Values.imagePullPolicy }} + +externalURL: https://holocron.{{ .Values.domain }} + +domain: {{ .Values.domain }} + +api: + env: + REQUEST_IP_HEADER: Host + SSO_ENABLED: {{ .Values.addons.holocron.sso.enabled }} + SSO_ADMIN_GROUP: {{ .Values.addons.holocron.sso.groups.admin }} + SSO_LEADERSHIP_GROUP: {{ .Values.addons.holocron.sso.groups.leadership }} + SSO_GROUPS_KEY: {{ .Values.sso.oidc.claims.groups }} + SSO_USERNAME_KEY: {{ .Values.sso.oidc.claims.username }} + +{{- if .Values.addons.gitlab.enabled }} +{{- $gitlabURL := "http://gitlab-webservice-default.gitlab.svc.cluster.local:8080" }} +collectorGitlabSCM: + replicas: 1 + image: + repository: registry1.dso.mil/ironbank/holocron/collector-gitlab-scm + tag: "3.0.2" + pullPolicy: IfNotPresent + instances: + - accessTokenSecretKey: gitlab-scm-0 + env: + COLLECTOR_NAME: gitlab-scm-0 + COLLECTOR_INTERVAL_SECONDS: 600 + LOOK_BACK_DAYS: 365 + TARGET_URL: {{ $gitlabURL }} + COLLECTOR_TARGETS_INTRVL_SECS: 86400 + MAX_REQUESTS_PER_MINUTE: 500 +collectorGitlabBuild: + replicas: 1 + image: + repository: registry1.dso.mil/ironbank/holocron/collector-gitlab-build + tag: "3.0.2" + pullPolicy: IfNotPresent + instances: + - accessTokenSecretKey: gitlab-build-0 + env: + COLLECTOR_NAME: gitlab-build-0 + COLLECTOR_INTERVAL_SECONDS: 600 + LOOK_BACK_DAYS: 365 + TARGET_URL: {{ $gitlabURL }} + COLLECTOR_TARGETS_INTRVL_SECS: 86400 + MAX_REQUESTS_PER_MINUTE: 500 +collectorGitlabWorkflow: + replicas: 1 + image: + repository: registry1.dso.mil/ironbank/holocron/collector-gitlab-workflow + tag: "3.0.2" + pullPolicy: IfNotPresent + instances: + - accessTokenSecretKey: gitlab-workflow-0 + env: + COLLECTOR_NAME: gitlab-workflow-0 + COLLECTOR_INTERVAL_SECONDS: 600 + LOOK_BACK_DAYS: 365 + TARGET_URL: {{ $gitlabURL }} + COLLECTOR_TARGETS_INTRVL_SECS: 86400 + MAX_REQUESTS_PER_MINUTE: 500 + HIGHEST_PRIORITY_LABELS: priority::1,highest + HIGH_PRIORITY_LABELS: priority::2,high + MEDIUM_PRIORITY_LABELS: priority::3,medium + LOW_PRIORITY_LABELS: priority::4,low + LOWEST_PRIORITY_LABELS: priority::5,lowest + DEFAULT_TICKET_PRIORITY: lowest + FEATURE_LABELS: kind::feature,feature + DEFECT_LABELS: kind::bug,kind::defect,bug,defect + MAINTENANCE_LABELS: kind::maintenance,kind::docs,maintenance,docs,documentation + UNPLANNED_LABELS: kind::unplanned + DEFAULT_TICKET_TYPE: feature +{{- else }} +collectorGitlabSCM: + instances: [] +collectorGitlabBuild: + instances: [] +collectorGitlabWorkflow: + instances: [] +{{- end }} +{{- if .Values.addons.holocron.jira.enabled }} +collectorJiraWorkflow: + replicas: 1 + image: + repository: registry1.dso.mil/ironbank/holocron/collector-jira-workflow + tag: "3.0.2" + pullPolicy: IfNotPresent + instances: + - accessTokenSecretKey: jira-workflow-0 + env: + COLLECTOR_NAME: jira-workflow-0 + COLLECTOR_INTERVAL_SECONDS: 600 + LOOK_BACK_DAYS: 365 + TARGET_URL: {{ .Values.addons.holocron.jira.service }} + COLLECTOR_TARGETS_INTRVL_SECS: 86400 + MAX_REQUESTS_PER_MINUTE: 500 +{{- else }} +collectorJiraWorkflow: + instances: [] +{{- end }} +{{- if .Values.addons.sonarqube.enabled }} +collectorSonarQubeProjectAnalysis: + replicas: 1 + image: + repository: registry1.dso.mil/ironbank/holocron/collector-sonarqube-project-analysis + tag: "3.0.2" + pullPolicy: IfNotPresent + instances: + - accessTokenSecretKey: sonarqube-project-analysis-0 + env: + COLLECTOR_NAME: sonarqube-project-analysis-0 + COLLECTOR_INTERVAL_SECONDS: 600 + LOOK_BACK_DAYS: 365 + TARGET_URL: "http://sonarqube-sonarqube.gitlab.svc.cluster.local:9000" + COLLECTOR_TARGETS_INTRVL_SECS: 86400 + MAX_REQUESTS_PER_MINUTE: 500 +{{- else }} +collectorSonarQubeProjectAnalysis: + instances: [] +{{- end }} + +postgresql: + {{- if .Values.addons.holocron.database.host }} + enabled: false + service: + name: {{ .Values.addons.holocron.database.host }} + {{- else }} + enabled: true + {{- end }} + service: + ports: + postgresql: {{ .Values.addons.holocron.database.port }} + + image: + repository: ironbank/opensource/postgres/postgresql + tag: "15.5" + auth: + secretKey: password + username: {{ .Values.addons.holocron.database.username }} + password: {{ .Values.addons.holocron.database.password }} + database: {{ .Values.addons.holocron.database.database }} + tls: + enabled: true + autoGenerated: true + +istio: + enabled: {{ .Values.istio.enabled }} + injection: enabled + holocron: + gateways: + - istio-system/{{ default "public" .Values.addons.holocron.ingress.gateway }} + hosts: + - holocron.{{ .Values.domain }} + +networkPolicies: + enabled: {{ .Values.networkPolicies.enabled }} + egressHostLabels: + {{- if .Values.addons.gitlab.enabled }} + app.kubernetes.io/name: gitlab + {{- end }} + {{- if .Values.addons.holocron.jira.enabled }} + {{ toYaml .Values.addons.holocron.jira.service.label | nindent 4 }} + {{- end }} + {{- if .Values.addons.sonarqube.enabled }} + app: sonarqube + {{- end }} + +monitoring: + enabled: {{ .Values.monitoring.enabled }} + +{{- end }} diff --git a/chart/templates/kyverno-policies/values.yaml b/chart/templates/kyverno-policies/values.yaml index 73d20ca226..3d72f74424 100644 --- a/chart/templates/kyverno-policies/values.yaml +++ b/chart/templates/kyverno-policies/values.yaml @@ -214,10 +214,17 @@ policies: require-non-root-group: validationFailureAction: Enforce - {{- if .Values.istio.enabled }} + {{ if .Values.istio.enabled }} parameters: excludeContainers: - istio-init + {{ if not .Values.addons.holocron.database.host }} + - init-chmod-data + {{- end }} + {{ else if not .Values.addons.holocron.database.host }} + parameters: + excludeContainers: + - init-chmod-data {{- end }} {{- if or $deployNodeAgent .Values.twistlock.enabled .Values.fluentbit.enabled .Values.promtail.enabled .Values.neuvector.enabled }} exclude: @@ -283,6 +290,13 @@ policies: parameters: excludeContainers: - istio-init + {{ if not .Values.addons.holocron.database.host }} + - init-chmod-data + {{- end }} + {{ else if not .Values.addons.holocron.database.host }} + parameters: + excludeContainers: + - init-chmod-data {{- end }} exclude: any: diff --git a/chart/values.schema.json b/chart/values.schema.json index 3501fd8c5a..6aac62b6ba 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -1087,6 +1087,13 @@ } ] }, + "holocron": { + "allOf": [ + { + "$ref": "#/$defs/basePackage" + } + ] + }, "thanos": { "allOf": [ { diff --git a/chart/values.yaml b/chart/values.yaml index aca7dc0859..5c5ecf4681 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -505,7 +505,6 @@ kyverno: # -- Post Renderers. See docs/postrenders.md postRenderers: [] - kyvernoPolicies: # -- Toggle deployment of Kyverno policies enabled: true @@ -741,7 +740,7 @@ loki: # ---------------------------------------------------------------------------------------------------------------------- neuvector: - # -- Toggle deployment of Neuvector. + # -- Toggle deployment of Neuvector. enabled: true # -- Choose source type of "git" or "helmRepo" @@ -1206,7 +1205,7 @@ addons: # -- Gitlab SSO Scopes, default is ["Gitlab"] scopes: - - Gitlab + - Gitlab database: # -- Hostname of a pre-existing PostgreSQL database to use for Gitlab. @@ -1264,12 +1263,11 @@ addons: # with a random password. # -- This needs to be set to a non-empty value in order for the Grafana Redis Datasource and Dashboards to be installed. password: "" - + # -- Rails plain text secret to define. If empty (""), the gitlab charts will create the gitlab-rails-secret with randomized data. # Read the following for more information on setting Gitlab rails secrets: https://docs.gitlab.com/charts/installation/secrets#gitlab-rails-secret railsSecret: "" - # -- Values to passthrough to the gitlab chart: https://repo1.dso.mil/big-bang/product/packages/gitlab.git values: {} @@ -1914,6 +1912,81 @@ addons: # -- Post Renderers. See docs/postrenders.md postRenderers: [] + + holocron: + # -- Toggle deployment of Holocron. + enabled: false + + # -- Choose source type of "git" or "helmRepo" + sourceType: "git" + + git: + repo: https://repo1.dso.mil/big-bang/product/packages/holocron.git + tag: "0.1.2" + path: "./chart" + helmRepo: + repoName: "registry1" + chartName: "holocron" + tag: "0.1.2" + + collectorAuth: + # -- Name of existing secret with auth tokens for collector services: https://repo1.dso.mil/groups/big-bang/apps/sandbox/holocron/-/wikis/Administrator-Guide + # -- Default keys for secret are: + # -- gitlab-scm-0, gitlab-workflow-0, gitlab-build-0, jira-workflow-0 + # -- If not provided, one will be created + existingSecret: "" + # -- Tokens for the secret to be created + gitlabToken: "" + jiraToken: "" + + jira: + # -- If there is a Jira deployment, enable a collector for it + enabled: false + service: + # -- The service name to communicate with + name: "" + # -- If network policies are enabled, a label to match the namespace for egress policy + label: + key: value + + # -- Flux reconciliation overrides specifically for the Holocron Package + flux: {} + + # -- Redirect the package ingress to a specific Istio Gateway (listed in `istio.gateways`). The default is "public". + ingress: + gateway: "" + + sso: + # -- Toggle SSO for Holocron on and off + enabled: false + + # -- Holocron SSO group roles: https://repo1.dso.mil/groups/big-bang/apps/sandbox/holocron/-/wikis/Administrator-Guide + groups: + admin: "" + leadership: "" + + database: + # -- Hostname of a pre-existing PostgreSQL database to use for Gitlab. + # -- Entering connection info will disable the deployment of an internal database and will auto-create any required secrets. + host: "" + + # -- Port of a pre-existing PostgreSQL database to use for Gitlab. + port: 5432 + + # -- Database name to connect to on host. + database: "holocron" + + # -- Username to connect as to external database, the user must have all privileges on the database. + username: "holocron" + + # -- Database password for the username used to connect to the existing database. + password: "holocron" + + # -- Post Renderers. See docs/postrenders.md + postRenderers: [] + + # -- Values to passthrough to the Holocron chart: https://repo1.dso.mil/big-bang/product/packages/holocron.git + values: {} # ---------------------------------------------------------------------------------------------------------------------- # thanos diff --git a/docs/understanding-bigbang/package-architecture/holocron.md b/docs/understanding-bigbang/package-architecture/holocron.md new file mode 100644 index 0000000000..692d2e038e --- /dev/null +++ b/docs/understanding-bigbang/package-architecture/holocron.md @@ -0,0 +1,20 @@ +# Holocron + +## Overview +Holocron is a software delivery metrics tracker and aggregator. It is designed to provide key insights into potential pain points for software delivery teams and help facilitate more efficient development. Holocron is an opinionated tool displaying goals for work in progress, continuous integration, and keeping piplines, branches, and merge requests from becoming stale. It also tracks how much effort is spent on different types of work, how often bugs are introduced, and how quickly tickets are resolved. It is an open-source project developed in-house at Platform One. + +### Architecture + +| Component | Description | +| --- | --- | +| Holocron Dashboard | Frontend allowing communication with the API to configure teams, value streams, and organizations and view their collected metrics. | +| Holocron API | Backend server connecting to the database, receives requests from the frontend and can potentially be integrated with a custom UI solution. | +| Postgresql Database | Holocron requires a Postgresql database for persistence. | +| SSO | (Optional) Holocron can utilize RBAC if there is an SSO service enabled. | +| Collector GitLab Build | (Optional) Periodically collects build data (pipelines) from a targeted GitLab instance. | +| Collector GitLab SCM |(Optional) Periodically collects SCM data (commits, branches, etc.) from a targeted GitLab instance. | +| Collector GitLab Workflow | (Optional) Periodically collects workflow data (tickets) from a targeted GitLab instance. | +| Collector Jira Workflow | (Optional) Periodically collects workflow data (tickets) from a targeted Jira instance. | +| Collector SonarQube Project Analysis | (Optional) Periodically collects project issue data (code smells, vulnerabilities, etc.) from a targeted SonarQube instance. | + +**Note: While all collectors are optional, Holocron won't have any metrics and as such no value if none are utilized.** -- GitLab