From 345c40d350730ffe4180b6084923a1b2dad6d07f Mon Sep 17 00:00:00 2001 From: renovate Date: Tue, 25 Aug 2020 00:06:20 +0000 Subject: [PATCH 01/11] Update apache/nifi Docker tag to v1.12.0 --- Dockerfile | 4 ++-- Jenkinsfile | 2 +- download.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9b9d5a1..373be3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_REGISTRY=nexus-docker-secure.levelup-dev.io ARG BASE_IMAGE=redhat/openjdk/openjdk11 ARG BASE_TAG=1.11 -FROM apache/nifi:1.11.4 as base +FROM apache/nifi:1.12.0 as base FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} @@ -10,7 +10,7 @@ LABEL org.opencontainers.image.title="NiFi" \ org.opencontainers.image.description="Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data." \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.url="https://nifi.apache.org/" \ - org.opencontainers.image.version="1.11.4" \ + org.opencontainers.image.version="1.12.0" \ maintainer="cht@dsop.io" ENV NIFI_VERSION=1.11.4 diff --git a/Jenkinsfile b/Jenkinsfile index ce419f9..92c152a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,2 +1,2 @@ @Library('DCCSCR@master') _ -dccscrPipeline(version: "1.11.4") +dccscrPipeline(version: "1.12.0") diff --git a/download.yaml b/download.yaml index 343c68f..417df78 100644 --- a/download.yaml +++ b/download.yaml @@ -1,6 +1,6 @@ resources: - - url: "docker://docker.io/apache/nifi@sha256:f63e3f2ab59630f217a3cf51a0e5dc98e1c99b49d3fb009fcf5ef0cd9f740213" - tag: "apache/nifi:1.11.4" + - url: "docker://docker.io/apache/nifi@sha256:289229d426eddc5cb45fcfcacc4cb553bbcf39de95f8d48f1c089ae7b394f1e0" + tag: "apache/nifi:1.12.0" - url: "https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm" filename: xmlstarlet.rpm validation: -- GitLab From 7c8b49573fa8fcbea5a42e0f570f89e6415db2b9 Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Thu, 1 Oct 2020 10:31:28 -0400 Subject: [PATCH 02/11] minor version upgrade --- Dockerfile | 4 ++-- Jenkinsfile | 2 +- download.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 373be3a..e540352 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_REGISTRY=nexus-docker-secure.levelup-dev.io ARG BASE_IMAGE=redhat/openjdk/openjdk11 ARG BASE_TAG=1.11 -FROM apache/nifi:1.12.0 as base +FROM apache/nifi:1.12.1 as base FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} @@ -10,7 +10,7 @@ LABEL org.opencontainers.image.title="NiFi" \ org.opencontainers.image.description="Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data." \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.url="https://nifi.apache.org/" \ - org.opencontainers.image.version="1.12.0" \ + org.opencontainers.image.version="1.12.1" \ maintainer="cht@dsop.io" ENV NIFI_VERSION=1.11.4 diff --git a/Jenkinsfile b/Jenkinsfile index 92c152a..7b192a9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,2 +1,2 @@ @Library('DCCSCR@master') _ -dccscrPipeline(version: "1.12.0") +dccscrPipeline(version: "1.12.1") diff --git a/download.yaml b/download.yaml index 417df78..e8011d2 100644 --- a/download.yaml +++ b/download.yaml @@ -1,11 +1,11 @@ resources: - url: "docker://docker.io/apache/nifi@sha256:289229d426eddc5cb45fcfcacc4cb553bbcf39de95f8d48f1c089ae7b394f1e0" - tag: "apache/nifi:1.12.0" + tag: "apache/nifi:1.12.1" - url: "https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm" filename: xmlstarlet.rpm validation: type: sha256 - value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0 + value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0 - url: "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" filename: jq-linux64 validation: -- GitLab From 3335a464a82414edeb49a072370ae25e4019e8b5 Mon Sep 17 00:00:00 2001 From: renovate Date: Fri, 2 Oct 2020 00:07:00 +0000 Subject: [PATCH 03/11] Update apache/nifi:1.12.1 Docker digest to bf7576a --- download.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download.yaml b/download.yaml index e8011d2..4670911 100644 --- a/download.yaml +++ b/download.yaml @@ -1,5 +1,5 @@ resources: - - url: "docker://docker.io/apache/nifi@sha256:289229d426eddc5cb45fcfcacc4cb553bbcf39de95f8d48f1c089ae7b394f1e0" + - url: "docker://docker.io/apache/nifi@sha256:bf7576ab7ad0bfe38c86be5baa47229d1644287984034dc9d5ff4801c5827115" tag: "apache/nifi:1.12.1" - url: "https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm" filename: xmlstarlet.rpm -- GitLab From c2a6dfdcd353d6ce11987dc1791c816f9c718f5e Mon Sep 17 00:00:00 2001 From: Mark Kenney Date: Thu, 8 Oct 2020 17:57:52 +0000 Subject: [PATCH 04/11] Update "ENV NIFI_VERSION" to "1.12.1" on line 16. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e540352..fd6daf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ LABEL org.opencontainers.image.title="NiFi" \ org.opencontainers.image.version="1.12.1" \ maintainer="cht@dsop.io" -ENV NIFI_VERSION=1.11.4 +ENV NIFI_VERSION=1.12.1 ENV NIFI_BASE_DIR=/opt/nifi ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-current ENV NIFI_VERSION_DIR=nifi-${NIFI_VERSION} \ -- GitLab From abd6c8aee489a73cb9b41b2a4c8912ddd9886f83 Mon Sep 17 00:00:00 2001 From: jeason Date: Wed, 14 Oct 2020 15:51:18 -0600 Subject: [PATCH 05/11] Project template: file templates --- .gitlab/CODEOWNERS | 6 +++ .gitlab/issue_templates/Access Request.md | 16 ++++++++ .../issue_templates/Application - Archive.md | 21 +++++++++++ .../issue_templates/Application - Initial.md | 32 ++++++++++++++++ .../issue_templates/Application - Update.md | 35 ++++++++++++++++++ .gitlab/issue_templates/Bug.md | 37 +++++++++++++++++++ .gitlab/issue_templates/Feature Request.md | 32 ++++++++++++++++ .../issue_templates/Leadership Question.md | 7 ++++ .gitlab/issue_templates/New Findings.md | 20 ++++++++++ .../issue_templates/Onboarding Question.md | 7 ++++ .gitlab/issue_templates/Pipeline Failure.md | 31 ++++++++++++++++ 11 files changed, 244 insertions(+) create mode 100644 .gitlab/CODEOWNERS create mode 100644 .gitlab/issue_templates/Access Request.md create mode 100644 .gitlab/issue_templates/Application - Archive.md create mode 100644 .gitlab/issue_templates/Application - Initial.md create mode 100644 .gitlab/issue_templates/Application - Update.md create mode 100644 .gitlab/issue_templates/Bug.md create mode 100644 .gitlab/issue_templates/Feature Request.md create mode 100644 .gitlab/issue_templates/Leadership Question.md create mode 100644 .gitlab/issue_templates/New Findings.md create mode 100644 .gitlab/issue_templates/Onboarding Question.md create mode 100644 .gitlab/issue_templates/Pipeline Failure.md diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS new file mode 100644 index 0000000..64a2c68 --- /dev/null +++ b/.gitlab/CODEOWNERS @@ -0,0 +1,6 @@ +[Pipelines] +.gitlab-ci.yml @ironbank-notifications/cht +.gitlab-ci.yaml @ironbank-notifications/cht + +[Gitlab Configuration Files] +.gitlab/* @ironbank-notifications/cht diff --git a/.gitlab/issue_templates/Access Request.md b/.gitlab/issue_templates/Access Request.md new file mode 100644 index 0000000..1a7b224 --- /dev/null +++ b/.gitlab/issue_templates/Access Request.md @@ -0,0 +1,16 @@ +## Summary + +The following individuals are requesting access to this project (one per line): +(List or tag all individuals here) + + +The access level should be: +- [ ] Developer access +- [ ] Remove access + + +## Definition of Done +- [ ] All accounts have been provided the necessary accesses + + +/label ~"Access" ~"To Do" \ No newline at end of file diff --git a/.gitlab/issue_templates/Application - Archive.md b/.gitlab/issue_templates/Application - Archive.md new file mode 100644 index 0000000..9f3b5fe --- /dev/null +++ b/.gitlab/issue_templates/Application - Archive.md @@ -0,0 +1,21 @@ +## Summary + +Requesting this application be archived due to one of the following reasons: +- [ ] Version is no longer supported by vendor +- [ ] Application is End-Of-Life +- [ ] License violation. +- [ ] Other. See below. + +## Detailed Description + +(Please provide a detailed description of why this application should be archived) + + +## Definition of Done +- [ ] Application has been reviewed for archival +- [ ] Project is officially marked as stale +- [ ] Iron Bank frontend no longer lists application as available or approved + + +/label ~"Container::Archive" +/cc @ironbank-notifications/archive \ No newline at end of file diff --git a/.gitlab/issue_templates/Application - Initial.md b/.gitlab/issue_templates/Application - Initial.md new file mode 100644 index 0000000..6594a05 --- /dev/null +++ b/.gitlab/issue_templates/Application - Initial.md @@ -0,0 +1,32 @@ +## Summary + +Requesting application to be hardened. This is only for initial hardening of a container. + + +## Version Information + +Current version: (State the current version of the application as you see it) + +Under support: (Is the updated version within the same major version of the application or is this a new major version?) + + +## Definition of Done +Hardening: +- [ ] Container builds successfully +- [ ] Greylist file has been created (requires a member from container hardening) +- [ ] Branch has been merged into `development` + +Justifications: +- [ ] All findings have been justified per the above documentation +- [ ] Justifications have been provided to the container hardening team + +Approval Process (container hardening team processes): +- [ ] Peer review from Container Hardening Team +- [ ] Findings Approver has reviewed and approved all justifications +- [ ] Approval request has been sent to Authorizing Official +- [ ] Approval request has been processed by Authorizing Official + + + +/label ~"Container::Initial" +/cc @ironbank-notifications/cht \ No newline at end of file diff --git a/.gitlab/issue_templates/Application - Update.md b/.gitlab/issue_templates/Application - Update.md new file mode 100644 index 0000000..caebb3e --- /dev/null +++ b/.gitlab/issue_templates/Application - Update.md @@ -0,0 +1,35 @@ +## Summary + +Requesting application be updated to a newer version. + + + +## Version Information + +Current version: (State the current version of the application as you see it) + +Updated version: (State the version you would like the application updated to) + +Under support: (Is the updated version within the same major version of the application or is this a new major version?) + + +## Definition of Done +Hardening: +- [ ] Container builds successfully +- [ ] Container version has been updated in greylist file +- [ ] Branch has been merged into `development` + +Justifications: +- [ ] All findings have been justified per the above documentation +- [ ] Justifications have been provided to the container hardening team + +Approval Process: +- [ ] Peer review from Container Hardening Team +- [ ] Findings Approver has reviewed and approved all justifications +- [ ] Approval request has been sent to Authorizing Official +- [ ] Approval request has been processed by Authorizing Official + + + +/label ~"Container::Update" +/cc @ironbank-notifications/updates \ No newline at end of file diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 0000000..1427a0c --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,37 @@ +## Summary + +(Summarize the bug encountered concisely) + + +## Steps to reproduce + +(How one can reproduce the issue - this is very important) + + +## What is the current bug behavior? + +(What actually happens) + + +## What is the expected correct behavior? + +(What you should see instead) + + +## Relevant logs and/or screenshots + +(Paste any relevant logs - please use code blocks (```) to format console output, +logs, and code as it's very hard to read otherwise.) + + +## Possible fixes + +(If you can, link to the line of code that might be responsible for the problem) + + +## Defintion of Done +- [ ] Bug has been identified and corrected within the container + + +/label ~Bug +/cc @ironbank-notifications/bug \ No newline at end of file diff --git a/.gitlab/issue_templates/Feature Request.md b/.gitlab/issue_templates/Feature Request.md new file mode 100644 index 0000000..a0e2f19 --- /dev/null +++ b/.gitlab/issue_templates/Feature Request.md @@ -0,0 +1,32 @@ +## Feature description + +(Detailed description of the feature being requested) + + +## Use cases + + +(Detailed description of the use case for this feature) + + +## Benefits + +(How does this benefit others) + + +## Requirements + +(Any requirements for this feature to be enabled?) + + +## Links / references + +(List of links or references that support this feature) + + +## Definition of Done +- [ ] Feature has been implemented + + +/label ~Feature +/cc @ironbank-notifications/feature \ No newline at end of file diff --git a/.gitlab/issue_templates/Leadership Question.md b/.gitlab/issue_templates/Leadership Question.md new file mode 100644 index 0000000..4674f82 --- /dev/null +++ b/.gitlab/issue_templates/Leadership Question.md @@ -0,0 +1,7 @@ +## Leadership question + +(Detailed description of the question you'd like to ask the leadership team) + + +/label ~"Question::Leadership" ~"To Do" +/cc @ironbank-notifications/leadership \ No newline at end of file diff --git a/.gitlab/issue_templates/New Findings.md b/.gitlab/issue_templates/New Findings.md new file mode 100644 index 0000000..068d029 --- /dev/null +++ b/.gitlab/issue_templates/New Findings.md @@ -0,0 +1,20 @@ +## Summary + +Container has new findings discovered during continuous monitoring. + + + +## Definition of Done +Justifications: +- [ ] All findings have been justified +- [ ] Justifications have been provided to the container hardening team + +Approval Process: +- [ ] Findings Approver has reviewed and approved all justifications +- [ ] Approval request has been sent to Authorizing Official +- [ ] Approval request has been processed by Authorizing Official + + + +/label ~"Container::New Findings" +/cc @ironbank-notifications/security \ No newline at end of file diff --git a/.gitlab/issue_templates/Onboarding Question.md b/.gitlab/issue_templates/Onboarding Question.md new file mode 100644 index 0000000..77dea11 --- /dev/null +++ b/.gitlab/issue_templates/Onboarding Question.md @@ -0,0 +1,7 @@ +## Onboarding question + +(Detailed description of the question you'd like to ask the onboarding team) + + +/label ~"Question::Onboarding" ~"To Do" +/cc @ironbank-notifications/onboarding \ No newline at end of file diff --git a/.gitlab/issue_templates/Pipeline Failure.md b/.gitlab/issue_templates/Pipeline Failure.md new file mode 100644 index 0000000..28b82a9 --- /dev/null +++ b/.gitlab/issue_templates/Pipeline Failure.md @@ -0,0 +1,31 @@ +## Summary + +(Summarize the pipeline issue encountered concisely) + + +## Link to failed pipeline + +(Link to the failed pipeline) + + +## What is the current bug behavior? + +(What actually happens) + + +## What is the expected correct behavior? + +(What you should see instead) + + +## Possible fixes + +(If you can, link to the line of code that might be responsible for the problem) + + +## Definition of Done +- [ ] Pipeline failure has been resolved + + +/label ~Pipeline +/cc @ironbank-notifications/pipelines \ No newline at end of file -- GitLab From a3a988a2d26761ef8b8950c9295fff5f9cbeab38 Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Fri, 13 Nov 2020 17:46:55 -0500 Subject: [PATCH 06/11] added scripts dir --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd6daf2..d888579 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -ARG BASE_REGISTRY=nexus-docker-secure.levelup-dev.io -ARG BASE_IMAGE=redhat/openjdk/openjdk11 +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=ironbank/redhat/openjdk/openjdk11 ARG BASE_TAG=1.11 FROM apache/nifi:1.12.1 as base @@ -60,4 +60,6 @@ WORKDIR ${NIFI_HOME} HEALTHCHECK --start-period=60s \ CMD /opt/nifi/nifi-current/bin/nifi.sh status +RUN mkdir -p ${NIFI_BASE_DIR}/scripts + ENTRYPOINT ["../scripts/start.sh"] -- GitLab From d3bdad18dc3b063aa2500436ef3115904e83e075 Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Mon, 16 Nov 2020 11:53:50 -0500 Subject: [PATCH 07/11] added scripts --- scripts/common.sh | 36 ++++++ scripts/secure.sh | 82 +++++++++++++ scripts/start.sh | 98 ++++++++++++++++ scripts/toolkit.sh | 130 +++++++++++++++++++++ scripts/update_cluster_state_management.sh | 31 +++++ scripts/update_login_providers.sh | 47 ++++++++ 6 files changed, 424 insertions(+) create mode 100644 scripts/common.sh create mode 100644 scripts/secure.sh create mode 100644 scripts/start.sh create mode 100644 scripts/toolkit.sh create mode 100644 scripts/update_cluster_state_management.sh create mode 100644 scripts/update_login_providers.sh diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000..f3ea435 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,36 @@ +#!/bin/sh -e +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 - value to search for +# 2 - value to replace +# 3 - file to perform replacement inline +prop_replace () { + target_file=${3:-${nifi_props_file}} + echo 'replacing target file ' ${target_file} + sed -i -e "s|^$1=.*$|$1=$2|" ${target_file} +} + +uncomment() { + target_file=${2} + echo "Uncommenting ${target_file}" + sed -i -e "s|^\#$1|$1|" ${target_file} +} + +# NIFI_HOME is defined by an ENV command in the backing Dockerfile +export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf +export nifi_props_file=${NIFI_HOME}/conf/nifi.properties +export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties +export hostname=$(hostname) diff --git a/scripts/secure.sh b/scripts/secure.sh new file mode 100644 index 0000000..46fa098 --- /dev/null +++ b/scripts/secure.sh @@ -0,0 +1,82 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts_dir='/opt/nifi/scripts' + +[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" + +# Perform idempotent changes of configuration to support secure environments +echo 'Configuring environment with SSL settings' + +: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."} +if [ ! -f "${KEYSTORE_PATH}" ]; then + echo "Keystore file specified (${KEYSTORE_PATH}) does not exist." + exit 1 +fi +: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."} +: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."} + +: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."} +if [ ! -f "${TRUSTSTORE_PATH}" ]; then + echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist." + exit 1 +fi +: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."} +: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."} + +prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}" +prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}" +prop_replace 'nifi.security.keystorePasswd' "${KEYSTORE_PASSWORD}" +prop_replace 'nifi.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" +prop_replace 'nifi.security.truststore' "${TRUSTSTORE_PATH}" +prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}" +prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}" + +prop_replace 'keystore' "${KEYSTORE_PATH}" ${nifi_toolkit_props_file} +prop_replace 'keystoreType' "${KEYSTORE_TYPE}" ${nifi_toolkit_props_file} +prop_replace 'keystorePasswd' "${KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file} +prop_replace 'keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file} +prop_replace 'truststore' "${TRUSTSTORE_PATH}" ${nifi_toolkit_props_file} +prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}" ${nifi_toolkit_props_file} +prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}" ${nifi_toolkit_props_file} + +# Disable HTTP and enable HTTPS +prop_replace 'nifi.web.http.port' '' +prop_replace 'nifi.web.http.host' '' +prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}" +prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.secure' 'true' + +# Setup nifi-toolkit +prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}:${NIFI_WEB_HTTPS_PORT:-8443}" ${nifi_toolkit_props_file} + +# Check if the user has specified a nifi.web.proxy.host setting and handle appropriately +if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping.' +else + prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}" +fi + +# Establish initial user and an associated admin identity +sed -i -e 's||'"${INITIAL_ADMIN_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml +sed -i -e 's||'"${INITIAL_ADMIN_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml + +if [ -n "${NODE_IDENTITY}" ]; then + sed -i -e 's||'"${NODE_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml +fi + +prop_replace 'proxiedEntity' "${INITIAL_ADMIN_IDENTITY}" ${nifi_toolkit_props_file} diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 0000000..e4bc79a --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,98 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts_dir='/opt/nifi/scripts' + +[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" + +# Override JVM memory settings +if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then + prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then + prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then + uncomment "java.arg.debug" ${nifi_bootstrap_file} +fi + +# Establish baseline properties +prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}" +prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" +prop_replace 'nifi.remote.input.secure' 'false' + +# Set nifi-toolkit properties files and baseUrl +"${scripts_dir}/toolkit.sh" +prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file} + +prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" +prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" +prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}" +prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" +prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}" +prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" +prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" +prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" +prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" +prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" +prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" + +# Set analytics properties +prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" +prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" +prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" +prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" +prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" +prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" + +. "${scripts_dir}/update_cluster_state_management.sh" + +# Check if we are secured or unsecured +case ${AUTH} in + tls) + echo 'Enabling Two-Way SSL user authentication' + . "${scripts_dir}/secure.sh" + ;; + ldap) + echo 'Enabling LDAP user authentication' + # Reference ldap-provider in properties + prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider' + + . "${scripts_dir}/secure.sh" + . "${scripts_dir}/update_login_providers.sh" + ;; + *) + if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.' + fi + ;; +esac + +# Continuously provide logs so that 'docker logs' can produce them +"${NIFI_HOME}/bin/nifi.sh" run & +nifi_pid="$!" +tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" & + +trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT; +trap ":" EXIT + +echo NiFi running with PID ${nifi_pid}. +wait ${nifi_pid} diff --git a/scripts/toolkit.sh b/scripts/toolkit.sh new file mode 100644 index 0000000..4da9ccf --- /dev/null +++ b/scripts/toolkit.sh @@ -0,0 +1,130 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts_dir='/opt/nifi/scripts' + +[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" + +# Override JVM memory settings +if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then + prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then + prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then + uncomment "java.arg.debug" ${nifi_bootstrap_file} +fi + +# Establish baseline properties +prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}" +prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" +prop_replace 'nifi.remote.input.secure' 'false' + +# Set nifi-toolkit properties files and baseUrl +"${scripts_dir}/toolkit.sh" +prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file} + +prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" +prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" +prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}" +prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" +prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}" +prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" +prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" +prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" +prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" +prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" +prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" + +# Set analytics properties +prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" +prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" +prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" +prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" +prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" +prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" + +. "${scripts_dir}/update_cluster_state_management.sh" + +# Check if we are secured or unsecured +case ${AUTH} in + tls) + echo 'Enabling Two-Way SSL user authentication' + . "${scripts_dir}/secure.sh" + ;; + ldap) + echo 'Enabling LDAP user authentication' + # Reference ldap-provider in properties + prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider' + + . "${scripts_dir}/secure.sh" + . "${scripts_dir}/update_login_providers.sh" + ;; + *) + if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.' + fi + ;; +esac + +# Continuously provide logs so that 'docker logs' can produce them +"${NIFI_HOME}/bin/nifi.sh" run & +nifi_pid="$!" +tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" & + +trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT; +trap ":" EXIT + +echo NiFi running with PID ${nifi_pid}. +wait ${nifi_pid} +nifi@ade64fede7e3:/opt/nifi/scripts$ cat toolkit.sh +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cat < ${nifi_toolkit_props_file} +baseUrl= +keystore= +keystoreType= +keystorePasswd= +keyPasswd= +truststore= +truststoreType= +truststorePasswd= +proxiedEntity= +EOT + +cat < ${HOME}/.nifi-cli.config +nifi.props=${nifi_toolkit_props_file} diff --git a/scripts/update_cluster_state_management.sh b/scripts/update_cluster_state_management.sh new file mode 100644 index 0000000..1975bf8 --- /dev/null +++ b/scripts/update_cluster_state_management.sh @@ -0,0 +1,31 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +state_providers_file=${NIFI_HOME}/conf/state-management.xml +property_xpath='/stateManagement/cluster-provider/property' + +edit_property() { + property_name=$1 + property_value=$2 + + if [ -n "${property_value}" ]; then + xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}" + fi +} + +edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}" +edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}" diff --git a/scripts/update_login_providers.sh b/scripts/update_login_providers.sh new file mode 100644 index 0000000..eda2888 --- /dev/null +++ b/scripts/update_login_providers.sh @@ -0,0 +1,47 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml +property_xpath='//loginIdentityProviders/provider/property' + +# Update a given property in the login-identity-providers file if a value is specified +edit_property() { + property_name=$1 + property_value=$2 + + if [ -n "${property_value}" ]; then + xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}" + fi +} + +# Remove comments to enable the ldap-provider +sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}" + +edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}" +edit_property 'Manager DN' "${LDAP_MANAGER_DN}" +edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}" +edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}" +edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}" +edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}" +edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}" +edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}" +edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}" +edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}" +edit_property 'Url' "${LDAP_URL}" +edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}" +edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}" +edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}" -- GitLab From c662f740955fdf68b6080fce41b34506bfc6b5cd Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Mon, 16 Nov 2020 12:41:00 -0500 Subject: [PATCH 08/11] copied scripts dir --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index d888579..372e09a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,10 +38,12 @@ RUN groupadd -g 1000 nifi && \ rm -f /xmlstarlet.rpm && \ chmod +x /usr/bin/jq && \ dnf clean all && \ - rm -rf /var/cache/dnf + rm -rf /var/cache/dnf && \ + mkdir -p ${NIFI_BASE_DIR}/scripts COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR} COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties +COPY scripts/ ${NIFI_BASE_DIR}/scripts/ VOLUME ${NIFI_LOG_DIR} \ ${NIFI_HOME}/conf \ @@ -60,6 +62,4 @@ WORKDIR ${NIFI_HOME} HEALTHCHECK --start-period=60s \ CMD /opt/nifi/nifi-current/bin/nifi.sh status -RUN mkdir -p ${NIFI_BASE_DIR}/scripts - ENTRYPOINT ["../scripts/start.sh"] -- GitLab From 88047588c5d67dbb13b9e164b62a7598f0dd770e Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Mon, 16 Nov 2020 13:48:32 -0500 Subject: [PATCH 09/11] changed permission on script dir --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 372e09a..a042db6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,8 @@ RUN groupadd -g 1000 nifi && \ chmod +x /usr/bin/jq && \ dnf clean all && \ rm -rf /var/cache/dnf && \ - mkdir -p ${NIFI_BASE_DIR}/scripts + mkdir -p ${NIFI_BASE_DIR}/scripts && \ + chmod +x ${NIFI_BASE_DIR}/scripts COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR} COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties -- GitLab From f778a8a5ffa3bccaa9eeebf4d7475368287982e7 Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Mon, 16 Nov 2020 14:24:25 -0500 Subject: [PATCH 10/11] changed permission on script dir --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a042db6..e3a2e79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN groupadd -g 1000 nifi && \ dnf clean all && \ rm -rf /var/cache/dnf && \ mkdir -p ${NIFI_BASE_DIR}/scripts && \ - chmod +x ${NIFI_BASE_DIR}/scripts + chmod +x ${NIFI_BASE_DIR}/scripts/*.sh COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR} COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties -- GitLab From bf30e7ec54ddbc0d6eeefa449208c55ed041baf1 Mon Sep 17 00:00:00 2001 From: "shen_vickie@bah.com" Date: Mon, 16 Nov 2020 14:51:11 -0500 Subject: [PATCH 11/11] minor update to Dockerfile --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e3a2e79..e083ff2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,13 +39,15 @@ RUN groupadd -g 1000 nifi && \ chmod +x /usr/bin/jq && \ dnf clean all && \ rm -rf /var/cache/dnf && \ - mkdir -p ${NIFI_BASE_DIR}/scripts && \ - chmod +x ${NIFI_BASE_DIR}/scripts/*.sh + mkdir -p ${NIFI_BASE_DIR}/scripts COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR} COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties COPY scripts/ ${NIFI_BASE_DIR}/scripts/ +RUN chmod +x ${NIFI_BASE_DIR}/scripts/*.sh + + VOLUME ${NIFI_LOG_DIR} \ ${NIFI_HOME}/conf \ ${NIFI_HOME}/database_repository \ -- GitLab