diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index d4029fa444018d4cb6855ac521242ca203074a50..c562595fd1630792fcc8d80c47a7f18c8a0becd6 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -194,7 +194,7 @@ DEPRECATION NOTICE: {{- $nexusValues := merge $nexusOldValues .Values.addons.nexusRepositoryManager -}} {{- with .Values }} -{{- if and .sso.url (coalesce .sso.oidc.host .sso.oidc.realm .sso.certificate_authority .sso.jwks .sso.jwks_uri .sso.client_id .sso.client_secret .sso.token_url .sso.auth_url .sso.secretName .elasticsearchKibana.sso.issuer .elasticsearchKibana.sso.auth_url .elasticsearchKibana.sso.token_url .elasticsearchKibana.sso.userinfo_url .elasticsearchKibana.sso.jwkset_url .elasticsearchKibana.sso.claims_principal .elasticsearchKibana.sso.endsession_url .elasticsearchKibana.sso.claims_group .elasticsearchKibana.sso.claims_mail .monitoring.sso.grafana.auth_url .monitoring.sso.grafana.token_url .monitoring.sso.grafana.api_url .twistlock.sso.provider_name .twistlock.sso.issuer_uri .twistlock.sso.idp_url .twistlock.sso.console_url .twistlock.sso.cert .addons.argocd.sso.provider_name .addons.gitlab.sso.label .addons.gitlab.sso.issuer_uri .addons.gitlab.sso.end_session_uri .addons.gitlab.sso.uid_field .addons.mattermost.sso.auth_endpoint .addons.mattermost.sso.token_endpoint .addons.mattermost.sso.user_api_endpoint $nexusValues.sso.idp_data.idpMetadata .addons.sonarqube.sso.provider_name .addons.sonarqube.sso.certificate) }} +{{- if and .sso.url (coalesce .sso.oidc.host .sso.oidc.realm .sso.certificate_authority .sso.jwks .sso.jwks_uri .sso.client_id .sso.client_secret .sso.token_url .sso.auth_url .sso.secretName .elasticsearchKibana.sso.issuer .elasticsearchKibana.sso.auth_url .elasticsearchKibana.sso.token_url .elasticsearchKibana.sso.userinfo_url .elasticsearchKibana.sso.jwkset_url .elasticsearchKibana.sso.claims_principal .elasticsearchKibana.sso.endsession_url .elasticsearchKibana.sso.claims_group .elasticsearchKibana.sso.claims_mail .grafana.sso.grafana.auth_url .grafana.sso.grafana.token_url .grafana.sso.grafana.api_url .twistlock.sso.provider_name .twistlock.sso.issuer_uri .twistlock.sso.idp_url .twistlock.sso.console_url .twistlock.sso.cert .addons.argocd.sso.provider_name .addons.gitlab.sso.label .addons.gitlab.sso.issuer_uri .addons.gitlab.sso.end_session_uri .addons.gitlab.sso.uid_field .addons.mattermost.sso.auth_endpoint .addons.mattermost.sso.token_endpoint .addons.mattermost.sso.user_api_endpoint $nexusValues.sso.idp_data.idpMetadata .addons.sonarqube.sso.provider_name .addons.sonarqube.sso.certificate) }} DEPRECATION NOTICE: The following SSO keys have been deprecated. Deprecated keys will continue to work, but will be removed in a future release. Please update your overrides. {{- if coalesce .sso.oidc.host .sso.oidc.realm .sso.certificate_authority .sso.jwks .sso.jwks_uri .sso.client_id .sso.client_secret .sso.token_url .sso.auth_url .sso.secretName }} @@ -283,21 +283,21 @@ DEPRECATION NOTICE: claims_mail: {{ .elasticsearchKibana.sso.claims_mail }} {{- end }} {{- end }} - {{- if coalesce .monitoring.sso.grafana.auth_url .monitoring.sso.grafana.token_url .monitoring.sso.grafana.api_url }} - monitoring: + {{- if coalesce .grafana.sso.grafana.auth_url .grafana.sso.grafana.token_url .grafana.sso.grafana.api_url }} + grafana: sso: grafana: - {{- if .monitoring.sso.grafana.auth_url }} + {{- if .grafana.sso.grafana.auth_url }} # "auth_url" moved to "sso.oidc.authorization" - auth_url: {{ .monitoring.sso.grafana.auth_url }} + auth_url: {{ .grafana.sso.grafana.auth_url }} {{- end }} - {{- if .monitoring.sso.grafana.token_url }} + {{- if .grafana.sso.grafana.token_url }} # "token_url" moved to "sso.oidc.token" - token_url: {{ .monitoring.sso.grafana.token_url }} + token_url: {{ .grafana.sso.grafana.token_url }} {{- end }} - {{- if .monitoring.sso.grafana.api_url }} + {{- if .grafana.sso.grafana.api_url }} # "api_url" moved to "sso.oidc.userinfo" - api_url: {{ .monitoring.sso.grafana.api_url }} + api_url: {{ .grafana.sso.grafana.api_url }} {{- end }} {{- end }} {{- if coalesce .twistlock.sso.provider_name .twistlock.sso.issuer_uri .twistlock.sso.idp_url .twistlock.sso.console_url .twistlock.sso.cert }} @@ -397,3 +397,4 @@ DEPRECATION NOTICE: .Values.addons.mattermostoperator has been deprecated and will be removed in a future Big Bang release. Please reconfigure your values overrides to use .Values.addons.mattermostOperator {{- end }} + diff --git a/chart/templates/monitoring/flux-dashboards.yaml b/chart/templates/grafana/flux-dashboards.yaml similarity index 88% rename from chart/templates/monitoring/flux-dashboards.yaml rename to chart/templates/grafana/flux-dashboards.yaml index 9eddb81401b122a2a10db2ae49c578e4f00a7110..5499e4578120ebdb42e4830bf3f44f3fd7f62768 100644 --- a/chart/templates/monitoring/flux-dashboards.yaml +++ b/chart/templates/grafana/flux-dashboards.yaml @@ -1,4 +1,4 @@ -{{- if .Values.monitoring.enabled }} +{{- if and .Values.monitoring.enabled .Values.grafana.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -16,3 +16,4 @@ data: {{ .Files.Get "dashboards/flux/logs.json" | nindent 4 }} {{- end }} {{- end }} + diff --git a/chart/templates/monitoring/flux/alert.yaml b/chart/templates/grafana/flux/alert.yaml similarity index 100% rename from chart/templates/monitoring/flux/alert.yaml rename to chart/templates/grafana/flux/alert.yaml diff --git a/chart/templates/monitoring/flux/grafana-auth-secret.yaml b/chart/templates/grafana/flux/grafana-auth-secret.yaml similarity index 100% rename from chart/templates/monitoring/flux/grafana-auth-secret.yaml rename to chart/templates/grafana/flux/grafana-auth-secret.yaml diff --git a/chart/templates/monitoring/flux/ingress-flux.yaml b/chart/templates/grafana/flux/ingress-flux.yaml similarity index 100% rename from chart/templates/monitoring/flux/ingress-flux.yaml rename to chart/templates/grafana/flux/ingress-flux.yaml diff --git a/chart/templates/monitoring/flux/provider.yaml b/chart/templates/grafana/flux/provider.yaml similarity index 100% rename from chart/templates/monitoring/flux/provider.yaml rename to chart/templates/grafana/flux/provider.yaml diff --git a/chart/templates/grafana/gitrepository.yaml b/chart/templates/grafana/gitrepository.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c55bd899cba63d22a936e27a23636d1103863e83 --- /dev/null +++ b/chart/templates/grafana/gitrepository.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.grafana.sourceType "git") (not .Values.offline) .Values.grafana.enabled }} +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: GitRepository +metadata: + name: grafana + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/component: "core" + {{- include "commonLabels" . | nindent 4}} +spec: + interval: {{ .Values.flux.interval }} + url: {{ .Values.grafana.git.repo }} + ref: + {{- include "validRef" .Values.grafana.git | nindent 4 }} + {{ include "gitIgnore" . }} + {{- include "gitCreds" . | nindent 2 }} +{{- end }} + diff --git a/chart/templates/grafana/grafana-env-secret.yaml b/chart/templates/grafana/grafana-env-secret.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6a743fd838f843107450e5c95c4170d80b84799 --- /dev/null +++ b/chart/templates/grafana/grafana-env-secret.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled (ne .Values.addons.gitlab.redis.password "") }} +apiVersion: v1 +kind: Secret +metadata: + name: grafana-env-secret + namespace: monitoring + labels: + grafana_datasource: "1" +type: Opaque +stringData: + GITLAB_REDIS_PASSWORD: {{ .Values.addons.gitlab.redis.password }} +{{- end }} + diff --git a/chart/templates/grafana/helmrelease.yaml b/chart/templates/grafana/helmrelease.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ed0eec35a8799827e2dffa0883b28c636d35f5b --- /dev/null +++ b/chart/templates/grafana/helmrelease.yaml @@ -0,0 +1,72 @@ +{{- $fluxSettingsMonitoring := merge .Values.grafana.flux .Values.flux -}} +{{- if .Values.grafana.enabled }} +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: grafana + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/component: "core" + {{- include "commonLabels" . | nindent 4}} + annotations: + checksum/bigbang-values: {{ include (print $.Template.BasePath "/grafana/values.yaml") . | sha256sum }} +spec: + targetNamespace: monitoring + chart: + spec: + {{- if eq .Values.grafana.sourceType "git" }} + chart: {{ .Values.grafana.git.path }} + sourceRef: + kind: GitRepository + name: grafana + namespace: {{ .Release.Namespace }} + {{- else }} + chart: {{ .Values.grafana.helmRepo.chartName }} + version: {{ .Values.grafana.helmRepo.tag }} + sourceRef: + kind: HelmRepository + name: {{ .Values.grafana.helmRepo.repoName }} + namespace: {{ .Release.Namespace }} + {{- end }} + interval: 5m + + {{- toYaml $fluxSettingsMonitoring | nindent 2 }} + + {{- if .Values.grafana.postRenderers }} + postRenderers: + {{ toYaml .Values.grafana.postRenderers | nindent 4 }} + {{- end }} + valuesFrom: + - name: {{ .Release.Name }}-grafana-values + kind: Secret + valuesKey: "common" + - name: {{ .Release.Name }}-grafana-values + kind: Secret + valuesKey: "defaults" + - name: {{ .Release.Name }}-grafana-values + kind: Secret + valuesKey: "overlays" + + # TODO: DRY this up + {{- if or .Values.gatekeeper.enabled .Values.istio.enabled .Values.kyvernoPolicies.enabled .Values.monitoring.enabled }} + dependsOn: + {{- if .Values.istio.enabled }} + - name: istio + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.gatekeeper.enabled }} + - name: gatekeeper + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.kyvernoPolicies.enabled }} + - name: kyverno-policies + namespace: {{ .Release.Namespace }} + {{- end }} + {{- if .Values.monitoring.enabled }} + - name: monitoring + namespace: {{ .Release.Namespace }} + {{- end }} + {{- end }} +{{- end }} + diff --git a/chart/templates/grafana/imagepullsecret.yaml b/chart/templates/grafana/imagepullsecret.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9a9f0c0adaa6efaa1fd26a7db2da3b7b93ef9c9 --- /dev/null +++ b/chart/templates/grafana/imagepullsecret.yaml @@ -0,0 +1,17 @@ +{{- if and (not .Values.monitoring.enabled) .Values.grafana.enabled }} +{{- if ( include "imagePullSecret" . ) }} +apiVersion: v1 +kind: Secret +metadata: + name: private-registry + namespace: monitoring + labels: + app.kubernetes.io/name: monitoring + app.kubernetes.io/component: "core" + {{- include "commonLabels" . | nindent 4}} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ template "imagePullSecret" . }} +{{- end }} +{{- end }} + diff --git a/chart/templates/grafana/namespace.yaml b/chart/templates/grafana/namespace.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fef1de924e46e90b57fe8d000e55df911090d38 --- /dev/null +++ b/chart/templates/grafana/namespace.yaml @@ -0,0 +1,12 @@ +{{- if and (not .Values.monitoring.enabled) .Values.grafana.enabled }} +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + app.kubernetes.io/name: monitoring + app.kubernetes.io/component: "core" + {{- include "commonLabels" . | nindent 4}} + istio-injection: {{ dig "istio" "injection" "enabled" .Values.grafana }} +{{- end }} + diff --git a/chart/templates/grafana/secret-ca.yaml b/chart/templates/grafana/secret-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c088f163ceb1e6b2af1adf64c97453c00931d6f3 --- /dev/null +++ b/chart/templates/grafana/secret-ca.yaml @@ -0,0 +1,11 @@ +{{- if and .Values.grafana.enabled .Values.grafana.sso.enabled (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default (dig "certificateAuthority" "secretName" "" .Values.sso) .Values.sso.secretName }}-grafana + namespace: monitoring +type: Opaque +data: + ca.pem: {{ default (dig "certificateAuthority" "cert" "" .Values.sso) .Values.sso.certificate_authority | b64enc }} +{{- end }} + diff --git a/chart/templates/grafana/secret-sso.yaml b/chart/templates/grafana/secret-sso.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b5218c0dcc7a6befdce6b6a0c326430325509f1 --- /dev/null +++ b/chart/templates/grafana/secret-sso.yaml @@ -0,0 +1,20 @@ +{{- if or (and .Values.grafana.enabled .Values.grafana.sso.enabled .Values.grafana.sso.grafana.client_id) (and .Values.monitoring.sso.enabled (dig "grafana" "client_id" false .Values.monitoring.sso)) }} +apiVersion: v1 +kind: Secret +metadata: + name: grafana-sso + namespace: monitoring +type: kubernetes.io/opaque +stringData: + {{- if (dig "grafana" "client_id" false .Values.monitoring.sso) }} + client_id: {{ (dig "grafana" "client_id" false .Values.monitoring.sso) }} + {{- else if .Values.grafana.sso.grafana.client_id }} + client_id: {{ .Values.grafana.sso.grafana.client_id }} + {{- end }} + {{- if (dig "grafana" "client_secret" false .Values.monitoring.sso) }} + client_secret: {{ (dig "grafana" "client_secret" false .Values.monitoring.sso) }} + {{- else if .Values.grafana.sso.grafana.client_secret }} + client_secret: {{ .Values.grafana.sso.grafana.client_secret }} + {{- end }} +{{- end }} + diff --git a/chart/templates/grafana/values.yaml b/chart/templates/grafana/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8bb923b8538da6b7b3dd251de62b0669416ce89a --- /dev/null +++ b/chart/templates/grafana/values.yaml @@ -0,0 +1,317 @@ +{{- if .Values.grafana.enabled }} +{{- include "values-secret" (dict "root" $ "package" (dict "values" (fromYaml (include "bigbang.overlays.grafana" .))) "name" "grafana" "defaults" (include "bigbang.defaults.grafana" .)) }} +{{- end }} + +{{- define "bigbang.defaults.grafana" -}} +# hostname is deprecated and replaced with domain. But if hostname exists then use it. +{{- $domainName := default .Values.domain .Values.hostname }} +hostname: {{ $domainName }} +domain: {{ $domainName }} + +{{- $istioInjection := (and (eq (dig "istio" "injection" "enabled" .Values.grafana) "enabled") .Values.istio.enabled) }} +{{- $gitlabRedis := (and (ne .Values.addons.gitlab.redis.password "" ) (or .Values.addons.gitlab.enabled .Values.addons.gitlabRunner.enabled)) }} +{{- $authserviceRedisEnabled := (and (dig "values" "redis" "enabled" false .Values.addons.authservice) .Values.addons.authservice.enabled) }} +{{- $redisDatasource := (or $gitlabRedis .Values.addons.argocd.enabled $authserviceRedisEnabled) }} + +flux: + enabled: true + +networkPolicies: + enabled: {{ .Values.networkPolicies.enabled }} + controlPlaneCidr: {{ .Values.networkPolicies.controlPlaneCidr }} + ingressLabels: + {{- $gateway := default "public" .Values.grafana.ingress.gateway }} + {{- $default := dict "app" (dig "gateways" $gateway "ingressGateway" nil .Values.istio) "istio" nil }} + {{- toYaml (dig "values" "gateways" $gateway "selector" $default .Values.istio) | nindent 4 }} + +openshift: {{ .Values.openshift }} + +minioOperator: + enabled: {{ .Values.addons.minioOperator.enabled }} + +gitlabRunner: + enabled: {{ .Values.addons.gitlabRunner.enabled }} + +istio: + {{- $grafanaInjection := dig "istio" "injection" "enabled" .Values.grafana }} + enabled: {{ .Values.istio.enabled }} + grafana: + enabled: true + gateways: + - istio-system/{{ default "public" .Values.grafana.ingress.gateway }} + injection: {{ dig "istio" "injection" "enabled" .Values.grafana }} + +anchore: + enabled: {{ .Values.addons.anchore.enabled }} + +kiali: + enabled: {{ .Values.kiali.enabled }} + +loki: + enabled: {{ .Values.loki.enabled }} + +tempo: + enabled: {{ .Values.tempo.enabled }} + +{{- if or $gitlabRedis $authserviceRedisEnabled $redisDatasource }} +redis: + enabled: true +{{- end }} + +vault: + enabled: {{ .Values.addons.vault.enabled }} + tlsDisable: {{ dig "global" "tlsDisable" true .Values.addons.vault.values }} + +global: + imagePullSecrets: + - name: private-registry + +sso: + enabled: {{ or .Values.grafana.sso.enabled .Values.grafana.sso.enabled }} + +{{- if $gitlabRedis }} +envFromSecret: grafana-env-secret +{{- end }} + +{{- if .Values.tempo.enabled }} +env: + GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor tempoSearch tempoServiceGraph" +{{- end }} + +image: + pullPolicy: {{ .Values.imagePullPolicy }} + pullSecrets: + - private-registry +sidecar: + imagePullPolicy: {{ .Values.imagePullPolicy }} + +{{- if $istioInjection }} +podAnnotations: + {{ include "istioAnnotation" . }} + {{- if $gitlabRedis }} + checksum/gitlabRedisPassword: {{ sha256sum .Values.addons.gitlab.redis.password }} + {{- end }} +{{- end }} + +{{- if or .Values.loki.enabled .Values.tempo.enabled $gitlabRedis $authserviceRedisEnabled .Values.addons.argocd.enabled }} +datasources: + datasourcesbb.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://monitoring-monitoring-kube-prometheus.monitoring.svc:9090 + editable: true + {{- if .Values.addons.argocd.enabled }} + - name: Argo Master + type: redis-datasource + access: proxy + url: argocd-argocd-redis-bb-master.argocd.svc.cluster.local:6379 + jsonData: + client: standalone + - name: Argo Headless + type: redis-datasource + access: proxy + url: argocd-argocd-redis-bb-headless.argocd.svc.cluster.local:6379 + jsonData: + client: standalone + - name: Argo Replicas + type: redis-datasource + access: proxy + url: argocd-argocd-redis-bb-replicas.argocd.svc.cluster.local:6379 + jsonData: + client: standalone + {{- end }} + {{- if $authserviceRedisEnabled }} + - name: AuthService Master + type: redis-datasource + access: proxy + url: authservice-authservice-redis-bb-master.authservice.svc.cluster.local:6379 + jsonData: + client: standalone + - name: AuthService Headless + type: redis-datasource + access: proxy + url: authservice-authservice-redis-bb-headless.authservice.svc.cluster.local:6379 + jsonData: + client: standalone + - name: AuthService Replicas + type: redis-datasource + access: proxy + url: authservice-authservice-redis-bb-replicas.authservice.svc.cluster.local:6379 + jsonData: + client: standalone + {{- end }} + {{- if $gitlabRedis }} + - name: GitLab + type: redis-datasource + access: proxy + url: gitlab-redis-master.gitlab.svc.cluster.local:6379 + jsonData: + client: standalone + secureJsonData: + password: $GITLAB_REDIS_PASSWORD + {{- end }} + {{- if .Values.loki.enabled }} + - name: Loki + type: loki + {{- if eq .Values.loki.strategy "monolith" }} + url: http://logging-loki.logging.svc.cluster.local:3100 + {{- else }} + url: http://logging-loki-read.logging.svc.cluster.local:3100 + {{- end }} + access: proxy + editable: true + {{- end }} + {{- if and .Values.loki.enabled .Values.tempo.enabled }} + jsonData: + derivedFields: + - datasourceName: Tempo + matcherRegex: "traceID=(\\w+)" + name: TraceID + url: "$${__value.raw}" + datasourceUid: tempo + {{- end }} + {{- if .Values.tempo.enabled }} + - name: Tempo + type: tempo + access: proxy + orgId: 1 + uid: tempo + url: http://tempo-tempo.tempo.svc:3100 + isDefault: false + editable: true + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: 'prometheus' + {{- end }} + {{- if and .Values.loki.enabled .Values.tempo.enabled }} + jsonData: + httpMethod: GET + tracesToLogs: + datasourceUid: 'Loki' + tags: ['job', 'instance', 'pod', 'namespace'] + mappedTags: [{ key: 'service.name', value: 'service' }] + mapTagNamesEnabled: false + spanStartTimeShift: '1h' + spanEndTimeShift: '1h' + filterByTraceID: false + filterBySpanID: false + serviceMap: + datasourceUid: 'prometheus' + search: + hide: false + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: 'Loki' + {{- end }} +{{- end }} + +grafana.ini: + {{- if .Values.istio.enabled }} + server: + root_url: https://grafana.{{ $domainName }}/ + {{- end }} + + auth.generic_oauth: + enabled: {{ or .Values.grafana.sso.enabled (and (dig "grafana" "client_id" false .Values.monitoring.sso) .Values.monitoring.sso.enabled) }} + {{- if .Values.sso.name }} + name: {{ .Values.sso.name }} + {{- end }} + {{- if or (and .Values.grafana.sso.enabled .Values.grafana.sso.grafana.client_id) (and (dig "grafana" "client_id" false .Values.monitoring.sso) .Values.monitoring.sso.enabled) }} + client_id: $__file{/etc/secrets/auth_generic_oauth/client_id} + {{- end }} + {{- if or (and .Values.grafana.sso.enabled .Values.grafana.sso.grafana.client_secret) (and (dig "grafana" "client_secret" false .Values.monitoring.sso) .Values.monitoring.sso.enabled) }} + client_secret: $__file{/etc/secrets/auth_generic_oauth/client_secret} + {{- end }} + {{- if (and (dig "grafana" "client_id" false .Values.monitoring.sso) .Values.monitoring.sso.enabled) }} + scopes: {{ ( dig "grafana" "scopes" false .Values.monitoring.sso) | default "openid profile email" }} + auth_url: {{ default (include "sso.oidc.auth" .) (dig "grafana" "auth_url" false .Values.monitoring.sso) }} + token_url: {{ default (include "sso.oidc.token" .) (dig "grafana" "token_url" false .Values.monitoring.sso) }} + api_url: {{ default (include "sso.oidc.userinfo" .) (dig "grafana" "api_url" false .Values.monitoring.sso) }} + allow_sign_up: {{ (dig "grafana" "allow_sign_up" false .Values.monitoring.sso) | default "True" }} + role_attribute_path: {{ (dig "grafana" "role_attribute_path" false .Values.monitoring.sso) | default "Viewer" }} + {{- else if .Values.grafana.sso.enabled }} + scopes: {{ .Values.grafana.sso.grafana.scopes | default "openid profile email" }} + auth_url: {{ default (include "sso.oidc.auth" .) .Values.grafana.sso.grafana.auth_url }} + token_url: {{ default (include "sso.oidc.token" .) .Values.grafana.sso.grafana.token_url }} + api_url: {{ default (include "sso.oidc.userinfo" .) .Values.grafana.sso.grafana.api_url }} + allow_sign_up: {{ .Values.grafana.sso.grafana.allow_sign_up | default "True" }} + role_attribute_path: {{ .Values.grafana.sso.grafana.role_attribute_path | default "Viewer" }} + {{- end }} + {{- with .Values.grafana.sso.grafana }} + {{- list "allowed_domains" .allowed_domains | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_client_ca" .tls_client_ca | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_skip_verify_insecure" .tls_skip_verify_insecure | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_client_cert" .tls_client_cert | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_client_key" .tls_client_key | include "bigbang.addValueIfSet" | indent 6 }} + {{- end }} + {{- with .Values.monitoring.sso.grafana }} + {{- list "allowed_domains" .allowed_domains | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_client_ca" .tls_client_ca | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_skip_verify_insecure" .tls_skip_verify_insecure | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_client_cert" .tls_client_cert | include "bigbang.addValueIfSet" | indent 6 }} + {{- list "tls_client_key" .tls_client_key | include "bigbang.addValueIfSet" | indent 6 }} + {{- end }} + +{{- if and (or .Values.grafana.sso.grafana.client_id (dig "grafana" "client_id" false .Values.monitoring.sso)) (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso)) }} +extraSecretMounts: + {{- if or (and .Values.grafana.sso.enabled .Values.grafana.sso.grafana.client_id) (and (dig "grafana" "client_id" false .Values.monitoring.sso) .Values.monitoring.sso.enabled) }} + - name: auth-generic-oauth-secret + mountPath: /etc/secrets/auth_generic_oauth + secretName: grafana-sso + defaultMode: 0440 + readOnly: true + {{- end }} + {{- if and (or .Values.grafana.sso.enabled .Values.monitoring.sso.enabled) (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso)) }} + - name: "oidc-ca-certificate" + mountPath: "/etc/oidc/ca.pem" + secretName: "tls-ca-sso-grafana" + readOnly: true + subPath: "ca.pem" + {{- end }} +{{- end }} +{{- if .Values.monitoring.enabled }} +serviceMonitor: + enabled: true +{{- end }} +{{- if $istioInjection }} + scheme: https + tlsConfig: + caFile: /etc/prom-certs/root-cert.pem + certFile: /etc/prom-certs/cert-chain.pem + keyFile: /etc/prom-certs/key.pem + insecureSkipVerify: true # Prometheus does not support Istio security naming, thus skip verifying target pod certificate +{{- end }} +{{- end }} + +{{- /* This function merges defaults in lists from above into overlays */ -}} +{{- /* The end user will not have to replicate `prometheus.prometheusSpec.additionalScrapeConfigs` or `grafana.extraSecretMounts` from above when providing an overlay */ -}} +{{- /* There is a hidden flag `skipOverlayMerge` that can be added to `prometheus.prometheusSpec` or `grafana` overlays to ignore the defaults */ -}} +{{- define "bigbang.overlays.grafana" }} + + {{- $defaults := fromYaml (include "bigbang.defaults.grafana" .) }} + + {{- $overlays := dig "values" dict .Values.grafana }} + {{- range $grafanaConfig, $default := $defaults }} + {{- $overlay := (dig $grafanaConfig dict $overlays) }} + # Only continue if an overlay matches a default constriant and hidden "skipOverlayMerge" is not set + {{- if and $overlay (kindIs "map" $overlay) (not $overlay.skipOverlayMerge) }} + + # Add any default extraSecretMounts to overlay + {{- if and (dig "extraSecretMounts" list $default) (dig "extraSecretMounts" list $overlay) }} + {{ $_ := set $overlay "extraSecretMounts" (concat $default.extraSecretMounts $overlay.extraSecretMounts) }} + {{- end }} + + # Add any default additionalDataSources to overlay + {{- if and (dig "additionalDataSources" list $default) (dig "additionalDataSources" list $overlay) }} + {{ $_ := set $overlay "additionalDataSources" (concat $default.additionalDataSources $overlay.additionalDataSources) }} + {{- end }} + {{- end }} + {{- end }} +{{ toYaml $overlays }} +{{- end }} + diff --git a/chart/templates/monitoring/secret-ca.yaml b/chart/templates/monitoring/secret-ca.yaml index b2fc380983a3b946040d9cae4ae713cd72da236d..5c89323f3dbe35a9c6ce8b8fca39f2e5575df2ba 100644 --- a/chart/templates/monitoring/secret-ca.yaml +++ b/chart/templates/monitoring/secret-ca.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.monitoring.enabled .Values.monitoring.sso.enabled (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso)) }} +{{- if or (and .Values.monitoring.enabled .Values.monitoring.sso.enabled) (and .Values.grafana.enabled .Values.grafana.sso.enabled) (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso)) }} apiVersion: v1 kind: Secret metadata: @@ -7,4 +7,5 @@ metadata: type: Opaque data: ca.pem: {{ default (dig "certificateAuthority" "cert" "" .Values.sso) .Values.sso.certificate_authority | b64enc }} -{{- end }} \ No newline at end of file +{{- end }} + diff --git a/chart/templates/monitoring/secret-sso.yaml b/chart/templates/monitoring/secret-sso.yaml deleted file mode 100644 index ffc9daf1a8596d828886c7b0e8e420cd03a000ed..0000000000000000000000000000000000000000 --- a/chart/templates/monitoring/secret-sso.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if and .Values.monitoring.enabled .Values.monitoring.sso.enabled .Values.monitoring.sso.grafana.client_id }} -apiVersion: v1 -kind: Secret -metadata: - name: grafana-sso - namespace: monitoring -type: kubernetes.io/opaque -stringData: - {{- if .Values.monitoring.sso.grafana.client_id }} - client_id: {{ .Values.monitoring.sso.grafana.client_id }} - {{- end }} - {{- if .Values.monitoring.sso.grafana.client_secret }} - client_secret: {{ .Values.monitoring.sso.grafana.client_secret }} - {{- end }} -{{- end }} diff --git a/chart/templates/monitoring/values.yaml b/chart/templates/monitoring/values.yaml index 8dfcf931ceaba00ecac646bcf9114a56be08262d..61de5dc87ddeca38843c23d2ff20abef88e8e258 100644 --- a/chart/templates/monitoring/values.yaml +++ b/chart/templates/monitoring/values.yaml @@ -53,10 +53,6 @@ istio: {{- end }} gateways: - istio-system/{{ default "public" .Values.monitoring.ingress.gateway }} - grafana: - enabled: true - gateways: - - istio-system/{{ default "public" .Values.monitoring.ingress.gateway }} injection: {{ dig "istio" "injection" "enabled" .Values.monitoring }} alertmanager: @@ -184,200 +180,6 @@ global: sso: enabled: {{ .Values.monitoring.sso.enabled }} -grafana: - - {{- if $gitlabRedis }} - envFromSecret: grafana-env-secret - {{- end }} - - {{- if .Values.tempo.enabled }} - env: - GF_FEATURE_TOGGLES_ENABLE: "traceqlEditor tempoSearch tempoServiceGraph" - {{- end }} - - image: - pullPolicy: {{ .Values.imagePullPolicy }} - pullSecrets: - - private-registry - sidecar: - imagePullPolicy: {{ .Values.imagePullPolicy }} - - {{- if $istioInjection }} - podAnnotations: - {{ include "istioAnnotation" . }} - {{- if $gitlabRedis }} - checksum/gitlabRedisPassword: {{ sha256sum .Values.addons.gitlab.redis.password }} - {{- end }} - {{- end }} - - {{- if or .Values.loki.enabled .Values.tempo.enabled $gitlabRedis $authserviceRedisEnabled .Values.addons.argocd.enabled }} - additionalDataSources: - {{- if .Values.addons.argocd.enabled }} - - name: Argo Master - type: redis-datasource - access: proxy - url: argocd-argocd-redis-bb-master.argocd.svc.cluster.local:6379 - jsonData: - client: standalone - - name: Argo Headless - type: redis-datasource - access: proxy - url: argocd-argocd-redis-bb-headless.argocd.svc.cluster.local:6379 - jsonData: - client: standalone - - name: Argo Replicas - type: redis-datasource - access: proxy - url: argocd-argocd-redis-bb-replicas.argocd.svc.cluster.local:6379 - jsonData: - client: standalone - {{- end }} - {{- if $authserviceRedisEnabled }} - - name: AuthService Master - type: redis-datasource - access: proxy - url: authservice-authservice-redis-bb-master.authservice.svc.cluster.local:6379 - jsonData: - client: standalone - - name: AuthService Headless - type: redis-datasource - access: proxy - url: authservice-authservice-redis-bb-headless.authservice.svc.cluster.local:6379 - jsonData: - client: standalone - - name: AuthService Replicas - type: redis-datasource - access: proxy - url: authservice-authservice-redis-bb-replicas.authservice.svc.cluster.local:6379 - jsonData: - client: standalone - {{- end }} - {{- if $gitlabRedis }} - - name: GitLab - type: redis-datasource - access: proxy - url: gitlab-redis-master.gitlab.svc.cluster.local:6379 - jsonData: - client: standalone - secureJsonData: - password: $GITLAB_REDIS_PASSWORD - {{- end }} - {{- if .Values.loki.enabled }} - - name: Loki - type: loki - {{- if eq .Values.loki.strategy "monolith" }} - url: http://logging-loki.logging.svc.cluster.local:3100 - {{- else }} - url: http://logging-loki-read.logging.svc.cluster.local:3100 - {{- end }} - access: proxy - editable: true - {{- end }} - {{- if and .Values.loki.enabled .Values.tempo.enabled }} - jsonData: - derivedFields: - - datasourceName: Tempo - matcherRegex: "traceID=(\\w+)" - name: TraceID - url: "$${__value.raw}" - datasourceUid: tempo - {{- end }} - {{- if .Values.tempo.enabled }} - - name: Tempo - type: tempo - access: proxy - orgId: 1 - uid: tempo - url: http://tempo-tempo.tempo.svc:3100 - isDefault: false - editable: true - jsonData: - httpMethod: GET - serviceMap: - datasourceUid: 'prometheus' - {{- if .Values.loki.enabled }} - jsonData: - httpMethod: GET - tracesToLogs: - datasourceUid: 'Loki' - tags: ['job', 'instance', 'pod', 'namespace'] - mappedTags: [{ key: 'service.name', value: 'service' }] - mapTagNamesEnabled: false - spanStartTimeShift: '1h' - spanEndTimeShift: '1h' - filterByTraceID: false - filterBySpanID: false - serviceMap: - datasourceUid: 'prometheus' - search: - hide: false - nodeGraph: - enabled: true - lokiSearch: - datasourceUid: 'Loki' - {{- end }} - {{- end }} - {{- end }} - - grafana.ini: - {{- if .Values.istio.enabled }} - server: - root_url: https://grafana.{{ $domainName }}/ - {{- end }} - - auth.generic_oauth: - enabled: {{ .Values.monitoring.sso.enabled }} - {{- if .Values.sso.name }} - name: {{ .Values.sso.name }} - {{- end }} - {{- if and .Values.monitoring.sso.enabled .Values.monitoring.sso.grafana.client_id }} - client_id: $__file{/etc/secrets/auth_generic_oauth/client_id} - {{- end }} - {{- if and .Values.monitoring.sso.enabled .Values.monitoring.sso.grafana.client_secret }} - client_secret: $__file{/etc/secrets/auth_generic_oauth/client_secret} - {{- end }} - scopes: {{ .Values.monitoring.sso.grafana.scopes | default "openid profile email" }} - auth_url: {{ default (include "sso.oidc.auth" .) .Values.monitoring.sso.grafana.auth_url }} - token_url: {{ default (include "sso.oidc.token" .) .Values.monitoring.sso.grafana.token_url }} - api_url: {{ default (include "sso.oidc.userinfo" .) .Values.monitoring.sso.grafana.api_url }} - allow_sign_up: {{ .Values.monitoring.sso.grafana.allow_sign_up | default "True" }} - role_attribute_path: {{ .Values.monitoring.sso.grafana.role_attribute_path | default "Viewer" }} - {{- with .Values.monitoring.sso.grafana }} - {{- list "allowed_domains" .allowed_domains | include "bigbang.addValueIfSet" | indent 6 }} - {{- list "tls_client_ca" .tls_client_ca | include "bigbang.addValueIfSet" | indent 6 }} - {{- list "tls_skip_verify_insecure" .tls_skip_verify_insecure | include "bigbang.addValueIfSet" | indent 6 }} - {{- list "tls_client_cert" .tls_client_cert | include "bigbang.addValueIfSet" | indent 6 }} - {{- list "tls_client_key" .tls_client_key | include "bigbang.addValueIfSet" | indent 6 }} - {{- end }} - - {{- if and .Values.monitoring.sso.enabled (or .Values.monitoring.sso.grafana.client_id (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso))) }} - extraSecretMounts: - {{- if .Values.monitoring.sso.grafana.client_id }} - - name: auth-generic-oauth-secret - mountPath: /etc/secrets/auth_generic_oauth - secretName: grafana-sso - defaultMode: 0440 - readOnly: true - {{- end }} - {{- if (or .Values.sso.certificate_authority (dig "certificateAuthority" "cert" false .Values.sso)) }} - - name: "oidc-ca-certificate" - mountPath: "/etc/oidc/ca.pem" - secretName: "tls-ca-sso" - readOnly: true - subPath: "ca.pem" - {{- end }} - {{- end }} - - {{- if $istioInjection }} - serviceMonitor: - scheme: https - tlsConfig: - caFile: /etc/prom-certs/root-cert.pem - certFile: /etc/prom-certs/cert-chain.pem - keyFile: /etc/prom-certs/key.pem - insecureSkipVerify: true # Prometheus does not support Istio security naming, thus skip verifying target pod certificate - {{- end }} - prometheus-node-exporter: image: pullPolicy: {{ .Values.imagePullPolicy }} @@ -459,21 +261,6 @@ prometheusOperator: {{- end }} {{- end }} - {{- range $monitoringConfig, $default := $defaults }} - {{- $overlay := (dig $monitoringConfig dict $overlays) }} - # Only continue if an overlay matches a default constriant and hidden "skipOverlayMerge" is not set - {{- if and $overlay (kindIs "map" $overlay) (not $overlay.skipOverlayMerge) }} - - # Add any default extraSecretMounts to overlay - {{- if and (dig "extraSecretMounts" list $default) (dig "extraSecretMounts" list $overlay) }} - {{ $_ := set $overlay "extraSecretMounts" (concat $default.extraSecretMounts $overlay.extraSecretMounts) }} - {{- end }} - - # Add any default additionalDataSources to overlay - {{- if and (dig "additionalDataSources" list $default) (dig "additionalDataSources" list $overlay) }} - {{ $_ := set $overlay "additionalDataSources" (concat $default.additionalDataSources $overlay.additionalDataSources) }} - {{- end }} - {{- end }} - {{- end }} {{ toYaml $overlays }} {{- end }} + diff --git a/chart/values.schema.json b/chart/values.schema.json index 512f9e18a7d73b9aa56a2006de5947a93e42a602..fee043ea305329502e5a01919a380dbeb61ede47 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -31,6 +31,7 @@ "neuvector", "tempo", "monitoring", + "grafana", "twistlock", "addons" ], @@ -759,6 +760,83 @@ }, "additionalProperties": false }, + "grafana": { + "allOf": [ + { + "$ref": "#/$defs/basePackage" + } + ], + "properties": { + "enabled": true, + "sourceType": true, + "git": true, + "helmRepo": true, + "flux": true, + "values": true, + "postRenderers": true, + "istio": { + "$ref": "#/$defs/istio" + }, + "ingress": { + "$ref": "#/$defs/ingress" + }, + "sso": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "grafana": { + "type": "object", + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "scopes": { + "type": "string" + }, + "allow_sign_up": { + "type": "boolean" + }, + "role_attribute_path": { + "type": "string" + }, + "token_url": { + "type": "string" + }, + "auth_url": { + "type": "string" + }, + "api_url": { + "type": "string" + }, + "tls_client_ca": { + "type": "string" + }, + "tls_skip_verify_insecure": { + "type": "boolean" + }, + "tls_client_cert": { + "type": "string" + }, + "tls_client_key": { + "type": "string" + }, + "allowed_domains": { + "type": "string" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, "twistlock": { "allOf": [ { @@ -1768,3 +1846,4 @@ } } } + diff --git a/chart/values.yaml b/chart/values.yaml index 514bdf73cec995fa15f28dace82c14311cbd2bca..72ea2c960399a8bd5085169ecdbd5e896e8a4aba 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -838,11 +838,11 @@ monitoring: git: repo: https://repo1.dso.mil/big-bang/product/packages/monitoring.git path: "./chart" - tag: "45.27.2-bb.4" + tag: "47.1.0-bb.1" helmRepo: repoName: "registry1" chartName: "monitoring" - tag: "45.27.2-bb.4" + tag: "47.1.0-bb.1" # -- Flux reconciliation overrides specifically for the Monitoring Package flux: @@ -872,6 +872,43 @@ monitoring: # -- Alertmanager OIDC client secret client_secret: "" + # -- Values to passthrough to the monitoring chart: https://repo1.dso.mil/big-bang/product/packages/monitoring.git + values: {} + + # -- Post Renderers. See docs/postrenders.md + postRenderers: [] +# ---------------------------------------------------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------------------------------------------------- + +# Grafana +# +grafana: + # -- Toggle deployment of Grafana + enabled: true + + # -- Choose source type of "git" or "helmRepo" + sourceType: "git" + + git: + repo: https://repo1.dso.mil/big-bang/apps/sandbox/grafana.git + path: "./chart" + tag: "6.57.4-bb.0" + helmRepo: + repoName: "registry1" + chartName: "grafana" + tag: "6.57.4-bb.0" + + # -- Flux reconciliation overrides specifically for the Monitoring Package + flux: {} + + # -- Redirect the package ingress to a specific Istio Gateway (listed in `istio.gateways`). The default is "public". + ingress: + gateway: "" + + sso: + # -- Toggle SSO for grafana components on and off + enabled: false grafana: # -- Grafana OIDC client ID client_id: "" @@ -887,7 +924,7 @@ monitoring: role_attribute_path: "Viewer" # -- Other options available, see package Documentation. - # -- Values to passthrough to the monitoring chart: https://repo1.dso.mil/big-bang/product/packages/monitoring.git + # -- Values to passthrough to the grafana chart: https://repo1.dso.mil/big-bang/product/packages/grafana.git values: {} # -- Post Renderers. See docs/postrenders.md diff --git a/docs/assets/configs/example/dev-sso-values.yaml b/docs/assets/configs/example/dev-sso-values.yaml index e2b1ddb64cd5e3831f8f902ac3a03f61c5829449..4130727d01182ef61f7f6babfbbc8a1c2b9a7fe7 100644 --- a/docs/assets/configs/example/dev-sso-values.yaml +++ b/docs/assets/configs/example/dev-sso-values.yaml @@ -175,10 +175,14 @@ monitoring: client_id: platform1_a8604cc9-f5e9-4656-802d-d05624370245_bb8-prometheus alertmanager: client_id: platform1_a8604cc9-f5e9-4656-802d-d05624370245_bb8-alertmanager + +grafana: + sso: + enabled: true grafana: client_id: platform1_a8604cc9-f5e9-4656-802d-d05624370245_bb8-grafana scopes: "openid Grafana" - + twistlock: # SSO (SAML) requires a license and enabling the init job - see https://repo1.dso.mil/big-bang/apps/security-tools/twistlock/-/blob/main/docs/initialization.md sso: diff --git a/tests/test-values.yaml b/tests/test-values.yaml index d68d1792d6110d7617b6f456b3c8b975e8375495..f4213c100f4219c397364d3ea302e72221af2675 100644 --- a/tests/test-values.yaml +++ b/tests/test-values.yaml @@ -322,6 +322,13 @@ gatekeeper: kyverno: values: replicaCount: 3 + resources: + limits: + cpu: 768m + memory: 768Mi + requests: + cpu: 768m + memory: 768Mi bbtests: enabled: true @@ -720,15 +727,20 @@ promtail: monitoring: enabled: true + flux: + timeout: 20m + install: + disableOpenAPIValidation: true + crds: CreateReplace + upgrade: + disableOpenAPIValidation: true + crds: CreateReplace sso: enabled: false prometheus: client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_prometheus alertmanager: client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_alertmanager - grafana: - client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_grafana - scopes: "Grafana" values: prometheus: prometheusSpec: @@ -749,35 +761,6 @@ monitoring: cpu: 100m memory: 30Mi limits: {} - grafana: - testFramework: - enabled: false - dashboards: - default: - k8s-deployment: - gnetId: 741 - revision: 1 - datasource: Prometheus - downloadDashboards: - resources: - limits: - cpu: 20m - memory: 20Mi - requests: - cpu: 20m - memory: 20Mi - dashboardProviders: - dashboardproviders.yaml: - apiVersion: 1 - providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards bbtests: enabled: true cypress: @@ -788,6 +771,47 @@ monitoring: cypress_alertmanager_url: 'https://alertmanager.bigbang.dev' cypress_check_istio_dashboards: 'true' +grafana: + enabled: true + sso: + enabled: false + grafana: + client_id: dev_00eb8904-5b88-4c68-ad67-cec0d2e07aa6_grafana + scopes: "openid Grafana" + values: + dashboards: + default: + k8s-deployment: + gnetId: 741 + revision: 1 + datasource: Prometheus + downloadDashboards: + resources: + limits: + cpu: 20m + memory: 20Mi + requests: + cpu: 20m + memory: 20Mi + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards + bbtests: + enabled: true + cypress: + image: registry1.dso.mil/bigbang-ci/cypress-kubectl:8.3.1 + envs: + cypress_grafana_url: 'https://grafana.bigbang.dev' + neuvector: values: k3s: @@ -1663,4 +1687,5 @@ addons: values: replicas: 1 bbtests: - enabled: true \ No newline at end of file + enabled: true +