UNCLASSIFIED

Skip to content

GitLab

Commit d3bdad18 authored by Vickie Shen's avatar Vickie Shen
Browse files

added scripts

parent a3a988a2
Pipeline #77291 passed with stages
in 26 minutes and 37 seconds
Show whitespace changes
Inline Side-by-side
Showing with 424 additions and 0 deletions
scripts/common.sh 0 → 100644
View file @ d3bdad18
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# 1 - value to search for
# 2 - value to replace
# 3 - file to perform replacement inline
prop_replace () {
target_file=${3:-${nifi_props_file}}
echo 'replacing target file ' ${target_file}
sed -i -e "s|^$1=.*$|$1=$2|" ${target_file}
}
uncomment() {
target_file=${2}
echo "Uncommenting ${target_file}"
sed -i -e "s|^\#$1|$1|" ${target_file}
}
# NIFI_HOME is defined by an ENV command in the backing Dockerfile
export nifi_bootstrap_file=${NIFI_HOME}/conf/bootstrap.conf
export nifi_props_file=${NIFI_HOME}/conf/nifi.properties
export nifi_toolkit_props_file=${HOME}/.nifi-cli.nifi.properties
export hostname=$(hostname)
scripts/secure.sh 0 → 100644
View file @ d3bdad18
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
scripts_dir='/opt/nifi/scripts'
[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
# Perform idempotent changes of configuration to support secure environments
echo 'Configuring environment with SSL settings'
: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}
if [ ! -f "${KEYSTORE_PATH}" ]; then
echo "Keystore file specified (${KEYSTORE_PATH}) does not exist."
exit 1
fi
: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}
: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}
: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."}
if [ ! -f "${TRUSTSTORE_PATH}" ]; then
echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist."
exit 1
fi
: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}
: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."}
prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}"
prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}"
prop_replace 'nifi.security.keystorePasswd' "${KEYSTORE_PASSWORD}"
prop_replace 'nifi.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}"
prop_replace 'nifi.security.truststore' "${TRUSTSTORE_PATH}"
prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}"
prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}"
prop_replace 'keystore' "${KEYSTORE_PATH}" ${nifi_toolkit_props_file}
prop_replace 'keystoreType' "${KEYSTORE_TYPE}" ${nifi_toolkit_props_file}
prop_replace 'keystorePasswd' "${KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file}
prop_replace 'keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" ${nifi_toolkit_props_file}
prop_replace 'truststore' "${TRUSTSTORE_PATH}" ${nifi_toolkit_props_file}
prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}" ${nifi_toolkit_props_file}
prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}" ${nifi_toolkit_props_file}
# Disable HTTP and enable HTTPS
prop_replace 'nifi.web.http.port' ''
prop_replace 'nifi.web.http.host' ''
prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}"
prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}"
prop_replace 'nifi.remote.input.secure' 'true'
# Setup nifi-toolkit
prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$HOSTNAME}:${NIFI_WEB_HTTPS_PORT:-8443}" ${nifi_toolkit_props_file}
# Check if the user has specified a nifi.web.proxy.host setting and handle appropriately
if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then
echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping.'
else
prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}"
fi
# Establish initial user and an associated admin identity
sed -i -e 's|<property name="Initial User Identity 1"></property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml
sed -i -e 's|<property name="Initial Admin Identity"></property>|<property name="Initial Admin Identity">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml
if [ -n "${NODE_IDENTITY}" ]; then
sed -i -e 's|<property name="Node Identity 1"></property>|<property name="Node Identity 1">'"${NODE_IDENTITY}"'</property>|' ${NIFI_HOME}/conf/authorizers.xml
fi
prop_replace 'proxiedEntity' "${INITIAL_ADMIN_IDENTITY}" ${nifi_toolkit_props_file}
scripts/start.sh 0 → 100644
View file @ d3bdad18
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
scripts_dir='/opt/nifi/scripts'
[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
# Override JVM memory settings
if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then
prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file}
fi
if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then
prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file}
fi
if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then
uncomment "java.arg.debug" ${nifi_bootstrap_file}
fi
# Establish baseline properties
prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}"
prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}"
prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}"
prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}"
prop_replace 'nifi.remote.input.secure' 'false'
# Set nifi-toolkit properties files and baseUrl
"${scripts_dir}/toolkit.sh"
prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file}
prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}"
prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}"
prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}"
prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}"
prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}"
prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}"
prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}"
prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}"
prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}"
prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}"
prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}"
# Set analytics properties
prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}"
prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}"
prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}"
prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}"
prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}"
prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}"
. "${scripts_dir}/update_cluster_state_management.sh"
# Check if we are secured or unsecured
case ${AUTH} in
tls)
echo 'Enabling Two-Way SSL user authentication'
. "${scripts_dir}/secure.sh"
;;
ldap)
echo 'Enabling LDAP user authentication'
# Reference ldap-provider in properties
prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider'
. "${scripts_dir}/secure.sh"
. "${scripts_dir}/update_login_providers.sh"
;;
*)
if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then
echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.'
fi
;;
esac
# Continuously provide logs so that 'docker logs' can produce them
"${NIFI_HOME}/bin/nifi.sh" run &
nifi_pid="$!"
tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" &
trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT;
trap ":" EXIT
echo NiFi running with PID ${nifi_pid}.
wait ${nifi_pid}
scripts/toolkit.sh 0 → 100644
View file @ d3bdad18
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
scripts_dir='/opt/nifi/scripts'
[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
# Override JVM memory settings
if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then
prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file}
fi
if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then
prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file}
fi
if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then
uncomment "java.arg.debug" ${nifi_bootstrap_file}
fi
# Establish baseline properties
prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}"
prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}"
prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}"
prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}"
prop_replace 'nifi.remote.input.secure' 'false'
# Set nifi-toolkit properties files and baseUrl
"${scripts_dir}/toolkit.sh"
prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file}
prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}"
prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}"
prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}"
prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}"
prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}"
prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}"
prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}"
prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}"
prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}"
prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}"
prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}"
# Set analytics properties
prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}"
prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}"
prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}"
prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}"
prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}"
prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}"
. "${scripts_dir}/update_cluster_state_management.sh"
# Check if we are secured or unsecured
case ${AUTH} in
tls)
echo 'Enabling Two-Way SSL user authentication'
. "${scripts_dir}/secure.sh"
;;
ldap)
echo 'Enabling LDAP user authentication'
# Reference ldap-provider in properties
prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider'
. "${scripts_dir}/secure.sh"
. "${scripts_dir}/update_login_providers.sh"
;;
*)
if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then
echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.'
fi
;;
esac
# Continuously provide logs so that 'docker logs' can produce them
"${NIFI_HOME}/bin/nifi.sh" run &
nifi_pid="$!"
tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" &
trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT;
trap ":" EXIT
echo NiFi running with PID ${nifi_pid}.
wait ${nifi_pid}
nifi@ade64fede7e3:/opt/nifi/scripts$ cat toolkit.sh
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
cat <<EOT > ${nifi_toolkit_props_file}
baseUrl=
keystore=
keystoreType=
keystorePasswd=
keyPasswd=
truststore=
truststoreType=
truststorePasswd=
proxiedEntity=
EOT
cat <<EOT > ${HOME}/.nifi-cli.config
nifi.props=${nifi_toolkit_props_file}
scripts/update_cluster_state_management.sh 0 → 100644
View file @ d3bdad18
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
state_providers_file=${NIFI_HOME}/conf/state-management.xml
property_xpath='/stateManagement/cluster-provider/property'
edit_property() {
property_name=$1
property_value=$2
if [ -n "${property_value}" ]; then
xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${state_providers_file}"
fi
}
edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}"
edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}"
scripts/update_login_providers.sh 0 → 100644
View file @ d3bdad18
#!/bin/sh -e
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
login_providers_file=${NIFI_HOME}/conf/login-identity-providers.xml
property_xpath='//loginIdentityProviders/provider/property'
# Update a given property in the login-identity-providers file if a value is specified
edit_property() {
property_name=$1
property_value=$2
if [ -n "${property_value}" ]; then
xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}"
fi
}
# Remove comments to enable the ldap-provider
sed -i '/To enable the ldap-provider remove/d' "${login_providers_file}"
edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}"
edit_property 'Manager DN' "${LDAP_MANAGER_DN}"
edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}"
edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}"
edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}"
edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}"
edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}"
edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}"
edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}"
edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}"
edit_property 'Url' "${LDAP_URL}"
edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}"
edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}"
edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

UNCLASSIFIED