included scripts for 1.12.1, removed rpm

Dockerfile

@@ -25,17 +25,13 @@ ENV NIFI_VERSION_DIR=nifi-${NIFI_VERSION} \
 
 USER root
 
-RUN dnf -y install jq
-
-
 RUN groupadd -g 1000 nifi && \
     useradd -r -u 1000 -m -g nifi nifi && \
     dnf update -y && \
-    dnf install -y libxslt procps-ng && \
+    dnf install -y libxslt procps-ng jq && \
     dnf clean all && \
     rm -rf /var/cache/dnf
 
-# https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm
 COPY xmlstarlet-1.6.1-11.el8.x86_64.rpm ${HOME}/xmlstarlet-1.6.1-11.el8.x86_64.rpm
 RUN yum -y localinstall --nogpgcheck ${HOME}/xmlstarlet-1.6.1-11.el8.x86_64.rpm && \
     rm -f ${HOME}/xmlstarlet-1.6.1-11.el8.x86_64.rpm

download.yaml

@@ -6,8 +6,4 @@ resources:
     validation:
       type: sha256
       value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0
-  - url: "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64"
-    filename: jq-linux64
-    validation:
-      type: sha256
-      value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44
+

scripts/common.sh

+#!/bin/sh -e
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# 1 - value to search for
+# 2 - value to replace
+# 3 - file to perform replacement inline
+prop_replace () {
+  target_file=${3:-${nifi_props_file}}
+  echo 'replacing target file ' ${target_file}
+  sed -i -e "s|^$1=.*$|$1=$2|"  ${target_file}
+}
+
+uncomment() {
+	target_file=${2}
+	echo "Uncommenting ${target_file}"
+	sed -i -e "s|^\#$1|$1|" ${target_file}
+}
+
+# NIFI_HOME is defined by an ENV command in the backing Dockerfile
+export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf
+export nifi_props_file=${NIFI_HOME}/conf/nifi.properties
+export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties
+export hostname=$(hostname)

scripts/secure.sh

+#!/bin/sh -e
+
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+scripts_dir='/opt/nifi/scripts'
+
+[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
+
+# Perform idempotent changes of configuration to support secure environments
+echo 'Configuring environment with SSL settings'
+
+: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}
+if [ ! -f "${KEYSTORE_PATH}" ]; then
+    echo "Keystore file specified (${KEYSTORE_PATH}) does not exist."
+    exit 1
+fi
+: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}
+: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}
+
+: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."}
+if [ ! -f "${TRUSTSTORE_PATH}" ]; then
+    echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist."
+    exit 1
+fi
+: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}
+: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."}
+
+prop_replace 'nifi.security.keystore'           "${KEYSTORE_PATH}"
+prop_replace 'nifi.security.keystoreType'       "${KEYSTORE_TYPE}"
+prop_replace 'nifi.security.keystorePasswd'     "${KEYSTORE_PASSWORD}"
+prop_replace 'nifi.security.keyPasswd'          "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}"
+prop_replace 'nifi.security.truststore'         "${TRUSTSTORE_PATH}"
+prop_replace 'nifi.security.truststoreType'     "${TRUSTSTORE_TYPE}"
+prop_replace 'nifi.security.truststorePasswd'   "${TRUSTSTORE_PASSWORD}"
+
+prop_replace 'keystore'           "${KEYSTORE_PATH}"                    ${nifi_toolkit_props_file}
+prop_replace 'keystoreType'       "${KEYSTORE_TYPE}"                    ${nifi_toolkit_props_file}
+prop_replace 'keystorePasswd'     "${KEYSTORE_PASSWORD}"                ${nifi_toolkit_props_file}
+prop_replace 'keyPasswd'          "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file}
+prop_replace 'truststore'         "${TRUSTSTORE_PATH}"                  ${nifi_toolkit_props_file}
+prop_replace 'truststoreType'     "${TRUSTSTORE_TYPE}"                  ${nifi_toolkit_props_file}
+prop_replace 'truststorePasswd'   "${TRUSTSTORE_PASSWORD}"              ${nifi_toolkit_props_file}
+
+# Disable HTTP and enable HTTPS
+prop_replace 'nifi.web.http.port'   ''
+prop_replace 'nifi.web.http.host'   ''
+prop_replace 'nifi.web.https.port'  "${NIFI_WEB_HTTPS_PORT:-8443}"
+prop_replace 'nifi.web.https.host'  "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}"
+prop_replace 'nifi.remote.input.secure' 'true'
+
+# Setup nifi-toolkit
+prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}:${NIFI_WEB_HTTPS_PORT:-8443}" ${nifi_toolkit_props_file}
+
+# Check if the user has specified a nifi.web.proxy.host setting and handle appropriately
+if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then
+    echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode.  The NiFi UI may be inaccessible if using port mapping.'
+else
+    prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}"
+fi
+
+# Establish initial user and an associated admin identity
+sed -i -e 's|<property name="Initial User Identity 1"></property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|'  ${NIFI_HOME}/conf/authorizers.xml
+sed -i -e 's|<property name="Initial Admin Identity"></property>|<property name="Initial Admin Identity">'"${INITIAL_ADMIN_IDENTITY}"'</property>|'  ${NIFI_HOME}/conf/authorizers.xml
+
+if [ -n "${NODE_IDENTITY}" ]; then
+    sed -i -e 's|<property name="Node Identity 1"></property>|<property name="Node Identity 1">'"${NODE_IDENTITY}"'</property>|'  ${NIFI_HOME}/conf/authorizers.xml
+fi
+
+prop_replace 'proxiedEntity' "${INITIAL_ADMIN_IDENTITY}" ${nifi_toolkit_props_file}

scripts/start.sh

+#!/bin/sh -e
+
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+scripts_dir='/opt/nifi/scripts'
+
+[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
+
+# Override JVM memory settings
+if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then
+    prop_replace 'java.arg.2'       "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file}
+fi
+
+if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then
+    prop_replace 'java.arg.3'       "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file}
+fi
+
+if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then
+    uncomment "java.arg.debug" ${nifi_bootstrap_file}
+fi
+
+# Establish baseline properties
+prop_replace 'nifi.web.http.port'               "${NIFI_WEB_HTTP_PORT:-8080}"
+prop_replace 'nifi.web.http.host'               "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}"
+prop_replace 'nifi.remote.input.host'           "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}"
+prop_replace 'nifi.remote.input.socket.port'    "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}"
+prop_replace 'nifi.remote.input.secure'         'false'
+
+# Set nifi-toolkit properties files and baseUrl
+"${scripts_dir}/toolkit.sh"
+prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file}
+
+prop_replace 'nifi.variable.registry.properties'    "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}"
+prop_replace 'nifi.cluster.is.node'                         "${NIFI_CLUSTER_IS_NODE:-false}"
+prop_replace 'nifi.cluster.node.address'                    "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}"
+prop_replace 'nifi.cluster.node.protocol.port'              "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}"
+prop_replace 'nifi.cluster.node.protocol.threads'           "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}"
+prop_replace 'nifi.cluster.node.protocol.max.threads'       "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}"
+prop_replace 'nifi.zookeeper.connect.string'                "${NIFI_ZK_CONNECT_STRING:-}"
+prop_replace 'nifi.zookeeper.root.node'                     "${NIFI_ZK_ROOT_NODE:-/nifi}"
+prop_replace 'nifi.cluster.flow.election.max.wait.time'     "${NIFI_ELECTION_MAX_WAIT:-5 mins}"
+prop_replace 'nifi.cluster.flow.election.max.candidates'    "${NIFI_ELECTION_MAX_CANDIDATES:-}"
+prop_replace 'nifi.web.proxy.context.path'                  "${NIFI_WEB_PROXY_CONTEXT_PATH:-}"
+
+# Set analytics properties
+prop_replace 'nifi.analytics.predict.enabled'                   "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}"
+prop_replace 'nifi.analytics.predict.interval'                  "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}"
+prop_replace 'nifi.analytics.query.interval'                    "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}"
+prop_replace 'nifi.analytics.connection.model.implementation'   "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}"
+prop_replace 'nifi.analytics.connection.model.score.name'       "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}"
+prop_replace 'nifi.analytics.connection.model.score.threshold'  "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}"
+
+. "${scripts_dir}/update_cluster_state_management.sh"
+
+# Check if we are secured or unsecured
+case ${AUTH} in
+    tls)
+        echo 'Enabling Two-Way SSL user authentication'
+        . "${scripts_dir}/secure.sh"
+        ;;
+    ldap)
+        echo 'Enabling LDAP user authentication'
+        # Reference ldap-provider in properties
+        prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider'
+
+        . "${scripts_dir}/secure.sh"
+        . "${scripts_dir}/update_login_providers.sh"
+        ;;
+    *)
+        if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then
+            echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode.  Will not update nifi.web.proxy.host.'
+        fi
+        ;;
+esac
+
+# Continuously provide logs so that 'docker logs' can    produce them
+"${NIFI_HOME}/bin/nifi.sh" run &
+nifi_pid="$!"
+tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" &
+
+trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT;
+trap ":" EXIT
+
+echo NiFi running with PID ${nifi_pid}.
+wait ${nifi_pid}

scripts/toolkit.sh

+#!/bin/sh -e
+
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+cat <<EOT > ${nifi_toolkit_props_file}
+baseUrl=
+keystore=
+keystoreType=
+keystorePasswd=
+keyPasswd=
+truststore=
+truststoreType=
+truststorePasswd=
+proxiedEntity=
+EOT
+
+cat <<EOT > ${HOME}/.nifi-cli.config
+nifi.props=${nifi_toolkit_props_file}
+EOT
\ No newline at end of file

scripts/update_cluster_state_management.sh

+#!/bin/sh -e
+
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+state_providers_file=${NIFI_HOME}/conf/state-management.xml
+property_xpath='/stateManagement/cluster-provider/property'
+
+edit_property() {
+  property_name=$1
+  property_value=$2
+
+  if [ -n "${property_value}" ]; then
+    xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}"
+  fi
+}
+
+edit_property 'Connect String'     "${NIFI_ZK_CONNECT_STRING}"
+edit_property "Root Node"                   "${NIFI_ZK_ROOT_NODE}"
\ No newline at end of file

scripts/update_login_providers.sh

+#!/bin/sh -e
+
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml
+property_xpath='//loginIdentityProviders/provider/property'
+
+# Update a given property in the login-identity-providers file if a value is specified
+edit_property() {
+  property_name=$1
+  property_value=$2
+
+  if [ -n "${property_value}" ]; then
+    xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}"
+  fi
+}
+
+# Remove comments to enable the ldap-provider
+sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}"
+
+edit_property 'Authentication Strategy'     "${LDAP_AUTHENTICATION_STRATEGY}"
+edit_property 'Manager DN'                  "${LDAP_MANAGER_DN}"
+edit_property 'Manager Password'            "${LDAP_MANAGER_PASSWORD}"
+edit_property 'TLS - Keystore'              "${LDAP_TLS_KEYSTORE}"
+edit_property 'TLS - Keystore Password'     "${LDAP_TLS_KEYSTORE_PASSWORD}"
+edit_property 'TLS - Keystore Type'         "${LDAP_TLS_KEYSTORE_TYPE}"
+edit_property 'TLS - Truststore'            "${LDAP_TLS_TRUSTSTORE}"
+edit_property 'TLS - Truststore Password'   "${LDAP_TLS_TRUSTSTORE_PASSWORD}"
+edit_property 'TLS - Truststore Type'       "${LDAP_TLS_TRUSTSTORE_TYPE}"
+edit_property 'TLS - Protocol'              "${LDAP_TLS_PROTOCOL}"
+edit_property 'Url'                         "${LDAP_URL}"
+edit_property 'User Search Base'            "${LDAP_USER_SEARCH_BASE}"
+edit_property 'User Search Filter'          "${LDAP_USER_SEARCH_FILTER}"
+edit_property 'Identity Strategy'           "${LDAP_IDENTITY_STRATEGY}"
\ No newline at end of file