diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..64a2c68c3ababda8d526d6cd995f02cd36f837ab --- /dev/null +++ b/.gitlab/CODEOWNERS @@ -0,0 +1,6 @@ +[Pipelines] +.gitlab-ci.yml @ironbank-notifications/cht +.gitlab-ci.yaml @ironbank-notifications/cht + +[Gitlab Configuration Files] +.gitlab/* @ironbank-notifications/cht diff --git a/.gitlab/issue_templates/Access Request.md b/.gitlab/issue_templates/Access Request.md new file mode 100644 index 0000000000000000000000000000000000000000..1a7b224d6ccdad95fef69b5c8be1ce2b543f338e --- /dev/null +++ b/.gitlab/issue_templates/Access Request.md @@ -0,0 +1,16 @@ +## Summary + +The following individuals are requesting access to this project (one per line): +(List or tag all individuals here) + + +The access level should be: +- [ ] Developer access +- [ ] Remove access + + +## Definition of Done +- [ ] All accounts have been provided the necessary accesses + + +/label ~"Access" ~"To Do" \ No newline at end of file diff --git a/.gitlab/issue_templates/Application - Archive.md b/.gitlab/issue_templates/Application - Archive.md new file mode 100644 index 0000000000000000000000000000000000000000..9f3b5fe4d8d43ae9f82411a391b200d4b43f2668 --- /dev/null +++ b/.gitlab/issue_templates/Application - Archive.md @@ -0,0 +1,21 @@ +## Summary + +Requesting this application be archived due to one of the following reasons: +- [ ] Version is no longer supported by vendor +- [ ] Application is End-Of-Life +- [ ] License violation. +- [ ] Other. See below. + +## Detailed Description + +(Please provide a detailed description of why this application should be archived) + + +## Definition of Done +- [ ] Application has been reviewed for archival +- [ ] Project is officially marked as stale +- [ ] Iron Bank frontend no longer lists application as available or approved + + +/label ~"Container::Archive" +/cc @ironbank-notifications/archive \ No newline at end of file diff --git a/.gitlab/issue_templates/Application - Initial.md b/.gitlab/issue_templates/Application - Initial.md new file mode 100644 index 0000000000000000000000000000000000000000..6594a0580b941815c0c7c6264cdfc42e28231f57 --- /dev/null +++ b/.gitlab/issue_templates/Application - Initial.md @@ -0,0 +1,32 @@ +## Summary + +Requesting application to be hardened. This is only for initial hardening of a container. + + +## Version Information + +Current version: (State the current version of the application as you see it) + +Under support: (Is the updated version within the same major version of the application or is this a new major version?) + + +## Definition of Done +Hardening: +- [ ] Container builds successfully +- [ ] Greylist file has been created (requires a member from container hardening) +- [ ] Branch has been merged into `development` + +Justifications: +- [ ] All findings have been justified per the above documentation +- [ ] Justifications have been provided to the container hardening team + +Approval Process (container hardening team processes): +- [ ] Peer review from Container Hardening Team +- [ ] Findings Approver has reviewed and approved all justifications +- [ ] Approval request has been sent to Authorizing Official +- [ ] Approval request has been processed by Authorizing Official + + + +/label ~"Container::Initial" +/cc @ironbank-notifications/cht \ No newline at end of file diff --git a/.gitlab/issue_templates/Application - Update.md b/.gitlab/issue_templates/Application - Update.md new file mode 100644 index 0000000000000000000000000000000000000000..caebb3e9aab279c7f109ec0fbfa246b8add6d972 --- /dev/null +++ b/.gitlab/issue_templates/Application - Update.md @@ -0,0 +1,35 @@ +## Summary + +Requesting application be updated to a newer version. + + + +## Version Information + +Current version: (State the current version of the application as you see it) + +Updated version: (State the version you would like the application updated to) + +Under support: (Is the updated version within the same major version of the application or is this a new major version?) + + +## Definition of Done +Hardening: +- [ ] Container builds successfully +- [ ] Container version has been updated in greylist file +- [ ] Branch has been merged into `development` + +Justifications: +- [ ] All findings have been justified per the above documentation +- [ ] Justifications have been provided to the container hardening team + +Approval Process: +- [ ] Peer review from Container Hardening Team +- [ ] Findings Approver has reviewed and approved all justifications +- [ ] Approval request has been sent to Authorizing Official +- [ ] Approval request has been processed by Authorizing Official + + + +/label ~"Container::Update" +/cc @ironbank-notifications/updates \ No newline at end of file diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md new file mode 100644 index 0000000000000000000000000000000000000000..1427a0caed1833bccd3b1e5f8c5f6eafde05266c --- /dev/null +++ b/.gitlab/issue_templates/Bug.md @@ -0,0 +1,37 @@ +## Summary + +(Summarize the bug encountered concisely) + + +## Steps to reproduce + +(How one can reproduce the issue - this is very important) + + +## What is the current bug behavior? + +(What actually happens) + + +## What is the expected correct behavior? + +(What you should see instead) + + +## Relevant logs and/or screenshots + +(Paste any relevant logs - please use code blocks (```) to format console output, +logs, and code as it's very hard to read otherwise.) + + +## Possible fixes + +(If you can, link to the line of code that might be responsible for the problem) + + +## Defintion of Done +- [ ] Bug has been identified and corrected within the container + + +/label ~Bug +/cc @ironbank-notifications/bug \ No newline at end of file diff --git a/.gitlab/issue_templates/Feature Request.md b/.gitlab/issue_templates/Feature Request.md new file mode 100644 index 0000000000000000000000000000000000000000..a0e2f195dc66e4187264381c5e96e8aa96db8a09 --- /dev/null +++ b/.gitlab/issue_templates/Feature Request.md @@ -0,0 +1,32 @@ +## Feature description + +(Detailed description of the feature being requested) + + +## Use cases + + +(Detailed description of the use case for this feature) + + +## Benefits + +(How does this benefit others) + + +## Requirements + +(Any requirements for this feature to be enabled?) + + +## Links / references + +(List of links or references that support this feature) + + +## Definition of Done +- [ ] Feature has been implemented + + +/label ~Feature +/cc @ironbank-notifications/feature \ No newline at end of file diff --git a/.gitlab/issue_templates/Leadership Question.md b/.gitlab/issue_templates/Leadership Question.md new file mode 100644 index 0000000000000000000000000000000000000000..4674f82f930085f34f51b4ecbb4d396519f53192 --- /dev/null +++ b/.gitlab/issue_templates/Leadership Question.md @@ -0,0 +1,7 @@ +## Leadership question + +(Detailed description of the question you'd like to ask the leadership team) + + +/label ~"Question::Leadership" ~"To Do" +/cc @ironbank-notifications/leadership \ No newline at end of file diff --git a/.gitlab/issue_templates/New Findings.md b/.gitlab/issue_templates/New Findings.md new file mode 100644 index 0000000000000000000000000000000000000000..068d029d89cb62dd4d4da5e03924c608172d97d6 --- /dev/null +++ b/.gitlab/issue_templates/New Findings.md @@ -0,0 +1,20 @@ +## Summary + +Container has new findings discovered during continuous monitoring. + + + +## Definition of Done +Justifications: +- [ ] All findings have been justified +- [ ] Justifications have been provided to the container hardening team + +Approval Process: +- [ ] Findings Approver has reviewed and approved all justifications +- [ ] Approval request has been sent to Authorizing Official +- [ ] Approval request has been processed by Authorizing Official + + + +/label ~"Container::New Findings" +/cc @ironbank-notifications/security \ No newline at end of file diff --git a/.gitlab/issue_templates/Onboarding Question.md b/.gitlab/issue_templates/Onboarding Question.md new file mode 100644 index 0000000000000000000000000000000000000000..77dea11e56c87d3fb65a1cf2ce7901621058f970 --- /dev/null +++ b/.gitlab/issue_templates/Onboarding Question.md @@ -0,0 +1,7 @@ +## Onboarding question + +(Detailed description of the question you'd like to ask the onboarding team) + + +/label ~"Question::Onboarding" ~"To Do" +/cc @ironbank-notifications/onboarding \ No newline at end of file diff --git a/.gitlab/issue_templates/Pipeline Failure.md b/.gitlab/issue_templates/Pipeline Failure.md new file mode 100644 index 0000000000000000000000000000000000000000..28b82a9454358a542efaa4b9c1c99542e3487fd6 --- /dev/null +++ b/.gitlab/issue_templates/Pipeline Failure.md @@ -0,0 +1,31 @@ +## Summary + +(Summarize the pipeline issue encountered concisely) + + +## Link to failed pipeline + +(Link to the failed pipeline) + + +## What is the current bug behavior? + +(What actually happens) + + +## What is the expected correct behavior? + +(What you should see instead) + + +## Possible fixes + +(If you can, link to the line of code that might be responsible for the problem) + + +## Definition of Done +- [ ] Pipeline failure has been resolved + + +/label ~Pipeline +/cc @ironbank-notifications/pipelines \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9b9d5a192bfeeafee7b27bfd7ef51e3875121c08..e083ff2b9ce3933c2c6d5d88d63ee1b98689e845 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -ARG BASE_REGISTRY=nexus-docker-secure.levelup-dev.io -ARG BASE_IMAGE=redhat/openjdk/openjdk11 +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=ironbank/redhat/openjdk/openjdk11 ARG BASE_TAG=1.11 -FROM apache/nifi:1.11.4 as base +FROM apache/nifi:1.12.1 as base FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} @@ -10,10 +10,10 @@ LABEL org.opencontainers.image.title="NiFi" \ org.opencontainers.image.description="Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data." \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.url="https://nifi.apache.org/" \ - org.opencontainers.image.version="1.11.4" \ + org.opencontainers.image.version="1.12.1" \ maintainer="cht@dsop.io" -ENV NIFI_VERSION=1.11.4 +ENV NIFI_VERSION=1.12.1 ENV NIFI_BASE_DIR=/opt/nifi ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-current ENV NIFI_VERSION_DIR=nifi-${NIFI_VERSION} \ @@ -38,10 +38,15 @@ RUN groupadd -g 1000 nifi && \ rm -f /xmlstarlet.rpm && \ chmod +x /usr/bin/jq && \ dnf clean all && \ - rm -rf /var/cache/dnf + rm -rf /var/cache/dnf && \ + mkdir -p ${NIFI_BASE_DIR}/scripts COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR} COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties +COPY scripts/ ${NIFI_BASE_DIR}/scripts/ + +RUN chmod +x ${NIFI_BASE_DIR}/scripts/*.sh + VOLUME ${NIFI_LOG_DIR} \ ${NIFI_HOME}/conf \ diff --git a/Jenkinsfile b/Jenkinsfile index ce419f9c436eff22f2eec0be900ce73a37a446c5..7b192a90233857901980cbbf2d9a25a3db0d85d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,2 +1,2 @@ @Library('DCCSCR@master') _ -dccscrPipeline(version: "1.11.4") +dccscrPipeline(version: "1.12.1") diff --git a/download.yaml b/download.yaml index 343c68fca4868ddfdbc8cc486384815f1cb88428..46709111f7750e6b98bf7f00924ee0f2ef803e69 100644 --- a/download.yaml +++ b/download.yaml @@ -1,11 +1,11 @@ resources: - - url: "docker://docker.io/apache/nifi@sha256:f63e3f2ab59630f217a3cf51a0e5dc98e1c99b49d3fb009fcf5ef0cd9f740213" - tag: "apache/nifi:1.11.4" + - url: "docker://docker.io/apache/nifi@sha256:bf7576ab7ad0bfe38c86be5baa47229d1644287984034dc9d5ff4801c5827115" + tag: "apache/nifi:1.12.1" - url: "https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm" filename: xmlstarlet.rpm validation: type: sha256 - value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0 + value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0 - url: "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" filename: jq-linux64 validation: diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000000000000000000000000000000000000..f3ea43580edea1a71d938467a9ddcb7410172235 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,36 @@ +#!/bin/sh -e +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 - value to search for +# 2 - value to replace +# 3 - file to perform replacement inline +prop_replace () { + target_file=${3:-${nifi_props_file}} + echo 'replacing target file ' ${target_file} + sed -i -e "s|^$1=.*$|$1=$2|" ${target_file} +} + +uncomment() { + target_file=${2} + echo "Uncommenting ${target_file}" + sed -i -e "s|^\#$1|$1|" ${target_file} +} + +# NIFI_HOME is defined by an ENV command in the backing Dockerfile +export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf +export nifi_props_file=${NIFI_HOME}/conf/nifi.properties +export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties +export hostname=$(hostname) diff --git a/scripts/secure.sh b/scripts/secure.sh new file mode 100644 index 0000000000000000000000000000000000000000..46fa09885974c51e0d8a471c23685ed614e1f09f --- /dev/null +++ b/scripts/secure.sh @@ -0,0 +1,82 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts_dir='/opt/nifi/scripts' + +[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" + +# Perform idempotent changes of configuration to support secure environments +echo 'Configuring environment with SSL settings' + +: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."} +if [ ! -f "${KEYSTORE_PATH}" ]; then + echo "Keystore file specified (${KEYSTORE_PATH}) does not exist." + exit 1 +fi +: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."} +: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."} + +: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."} +if [ ! -f "${TRUSTSTORE_PATH}" ]; then + echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist." + exit 1 +fi +: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."} +: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."} + +prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}" +prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}" +prop_replace 'nifi.security.keystorePasswd' "${KEYSTORE_PASSWORD}" +prop_replace 'nifi.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" +prop_replace 'nifi.security.truststore' "${TRUSTSTORE_PATH}" +prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}" +prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}" + +prop_replace 'keystore' "${KEYSTORE_PATH}" ${nifi_toolkit_props_file} +prop_replace 'keystoreType' "${KEYSTORE_TYPE}" ${nifi_toolkit_props_file} +prop_replace 'keystorePasswd' "${KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file} +prop_replace 'keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file} +prop_replace 'truststore' "${TRUSTSTORE_PATH}" ${nifi_toolkit_props_file} +prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}" ${nifi_toolkit_props_file} +prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}" ${nifi_toolkit_props_file} + +# Disable HTTP and enable HTTPS +prop_replace 'nifi.web.http.port' '' +prop_replace 'nifi.web.http.host' '' +prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}" +prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.secure' 'true' + +# Setup nifi-toolkit +prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}:${NIFI_WEB_HTTPS_PORT:-8443}" ${nifi_toolkit_props_file} + +# Check if the user has specified a nifi.web.proxy.host setting and handle appropriately +if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping.' +else + prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}" +fi + +# Establish initial user and an associated admin identity +sed -i -e 's||'"${INITIAL_ADMIN_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml +sed -i -e 's||'"${INITIAL_ADMIN_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml + +if [ -n "${NODE_IDENTITY}" ]; then + sed -i -e 's||'"${NODE_IDENTITY}"'|' ${NIFI_HOME}/conf/authorizers.xml +fi + +prop_replace 'proxiedEntity' "${INITIAL_ADMIN_IDENTITY}" ${nifi_toolkit_props_file} diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..e4bc79a1dd5f91383f2f694bf186196549164bf1 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,98 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts_dir='/opt/nifi/scripts' + +[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" + +# Override JVM memory settings +if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then + prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then + prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then + uncomment "java.arg.debug" ${nifi_bootstrap_file} +fi + +# Establish baseline properties +prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}" +prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" +prop_replace 'nifi.remote.input.secure' 'false' + +# Set nifi-toolkit properties files and baseUrl +"${scripts_dir}/toolkit.sh" +prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file} + +prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" +prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" +prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}" +prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" +prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}" +prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" +prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" +prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" +prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" +prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" +prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" + +# Set analytics properties +prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" +prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" +prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" +prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" +prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" +prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" + +. "${scripts_dir}/update_cluster_state_management.sh" + +# Check if we are secured or unsecured +case ${AUTH} in + tls) + echo 'Enabling Two-Way SSL user authentication' + . "${scripts_dir}/secure.sh" + ;; + ldap) + echo 'Enabling LDAP user authentication' + # Reference ldap-provider in properties + prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider' + + . "${scripts_dir}/secure.sh" + . "${scripts_dir}/update_login_providers.sh" + ;; + *) + if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.' + fi + ;; +esac + +# Continuously provide logs so that 'docker logs' can produce them +"${NIFI_HOME}/bin/nifi.sh" run & +nifi_pid="$!" +tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" & + +trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT; +trap ":" EXIT + +echo NiFi running with PID ${nifi_pid}. +wait ${nifi_pid} diff --git a/scripts/toolkit.sh b/scripts/toolkit.sh new file mode 100644 index 0000000000000000000000000000000000000000..4da9ccfd14163a0b843acd7de67ff05a4c5c31dc --- /dev/null +++ b/scripts/toolkit.sh @@ -0,0 +1,130 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +scripts_dir='/opt/nifi/scripts' + +[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" + +# Override JVM memory settings +if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then + prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then + prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file} +fi + +if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then + uncomment "java.arg.debug" ${nifi_bootstrap_file} +fi + +# Establish baseline properties +prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}" +prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}" +prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" +prop_replace 'nifi.remote.input.secure' 'false' + +# Set nifi-toolkit properties files and baseUrl +"${scripts_dir}/toolkit.sh" +prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file} + +prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" +prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" +prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}" +prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" +prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}" +prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" +prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" +prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" +prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" +prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" +prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" + +# Set analytics properties +prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" +prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" +prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" +prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" +prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" +prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" + +. "${scripts_dir}/update_cluster_state_management.sh" + +# Check if we are secured or unsecured +case ${AUTH} in + tls) + echo 'Enabling Two-Way SSL user authentication' + . "${scripts_dir}/secure.sh" + ;; + ldap) + echo 'Enabling LDAP user authentication' + # Reference ldap-provider in properties + prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider' + + . "${scripts_dir}/secure.sh" + . "${scripts_dir}/update_login_providers.sh" + ;; + *) + if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.' + fi + ;; +esac + +# Continuously provide logs so that 'docker logs' can produce them +"${NIFI_HOME}/bin/nifi.sh" run & +nifi_pid="$!" +tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" & + +trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT; +trap ":" EXIT + +echo NiFi running with PID ${nifi_pid}. +wait ${nifi_pid} +nifi@ade64fede7e3:/opt/nifi/scripts$ cat toolkit.sh +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cat < ${nifi_toolkit_props_file} +baseUrl= +keystore= +keystoreType= +keystorePasswd= +keyPasswd= +truststore= +truststoreType= +truststorePasswd= +proxiedEntity= +EOT + +cat < ${HOME}/.nifi-cli.config +nifi.props=${nifi_toolkit_props_file} diff --git a/scripts/update_cluster_state_management.sh b/scripts/update_cluster_state_management.sh new file mode 100644 index 0000000000000000000000000000000000000000..1975bf87d2447af1768225a790286d7270c6a3e7 --- /dev/null +++ b/scripts/update_cluster_state_management.sh @@ -0,0 +1,31 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +state_providers_file=${NIFI_HOME}/conf/state-management.xml +property_xpath='/stateManagement/cluster-provider/property' + +edit_property() { + property_name=$1 + property_value=$2 + + if [ -n "${property_value}" ]; then + xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}" + fi +} + +edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}" +edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}" diff --git a/scripts/update_login_providers.sh b/scripts/update_login_providers.sh new file mode 100644 index 0000000000000000000000000000000000000000..eda2888b4fa9628bea425476011aa62dd3693fcb --- /dev/null +++ b/scripts/update_login_providers.sh @@ -0,0 +1,47 @@ +#!/bin/sh -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml +property_xpath='//loginIdentityProviders/provider/property' + +# Update a given property in the login-identity-providers file if a value is specified +edit_property() { + property_name=$1 + property_value=$2 + + if [ -n "${property_value}" ]; then + xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}" + fi +} + +# Remove comments to enable the ldap-provider +sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}" + +edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}" +edit_property 'Manager DN' "${LDAP_MANAGER_DN}" +edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}" +edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}" +edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}" +edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}" +edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}" +edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}" +edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}" +edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}" +edit_property 'Url' "${LDAP_URL}" +edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}" +edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}" +edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}"