diff --git a/Dockerfile b/Dockerfile index cac17a5b92a46b319acd3c0e4253daa444e33bbe..e037eefea6f1ba135bba5d648fd4ebf0eca5f65e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,6 @@ FROM apache/nifi:1.12.1 as base FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} - ENV NIFI_VERSION=1.12.1 ENV NIFI_BASE_DIR=/opt/nifi ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-current @@ -24,8 +23,6 @@ COPY xmlstarlet.rpm /xmlstarlet.rpm RUN groupadd -g 1000 nifi && \ useradd -r -u 1000 -m -g nifi nifi && \ - chown nifi:0 ${HOME} && \ - chmod g=u ${HOME} && \ dnf update -y && \ dnf install -y libxslt procps-ng && \ rpm -i --nosignature /xmlstarlet.rpm && \ @@ -37,10 +34,14 @@ RUN groupadd -g 1000 nifi && \ COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR} COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties -COPY scripts/ ${NIFI_BASE_DIR}/scripts/ +COPY --chown=nifi:nifi scripts/ ${NIFI_BASE_DIR}/scripts/ -RUN chmod +x ${NIFI_BASE_DIR}/scripts/*.sh +RUN chmod +x ${NIFI_BASE_DIR}/scripts/*.sh && \ + chmod -R go-rwx ${NIFI_BASE_DIR} && \ + echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh +USER ${USER} +WORKDIR ${NIFI_HOME} VOLUME ${NIFI_LOG_DIR} \ ${NIFI_HOME}/conf \ @@ -53,9 +54,6 @@ VOLUME ${NIFI_LOG_DIR} \ # Web HTTP(s) & Socket Site-to-Site Ports EXPOSE 8080 8443 10000 8000 -USER ${USER} -WORKDIR ${NIFI_HOME} - HEALTHCHECK --start-period=60s \ CMD /opt/nifi/nifi-current/bin/nifi.sh status diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 54e52eafd63f7d21f34d6a452f0cd6aba8242875..93e3bbd359caf02e0101b834045dee1bbb034d83 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -20,20 +20,20 @@ args: labels: org.opencontainers.image.title: "nifi" ## Human-readable description of the software packaged in the image - # org.opencontainers.image.description: "FIXME" + org.opencontainers.image.description: "NiFi is an easy to use, powerful, and reliable system to process and distribute data." ## License(s) under which contained software is distributed - # org.opencontainers.image.licenses: "FIXME" + org.opencontainers.image.licenses: "Apache-2.0" ## URL to find more information on the image - # org.opencontainers.image.url: "FIXME" + org.opencontainers.image.url: "https://nifi.apache.org" ## Name of the distributing entity, organization or individual - # org.opencontainers.image.vendor: "FIXME" + org.opencontainers.image.vendor: "Apache" org.opencontainers.image.version: "1.12.1" ## Keywords to help with search (ex. "cicd,gitops,golang") - # mil.dso.ironbank.image.keywords: "FIXME" + mil.dso.ironbank.image.keywords: "dataflow,processing" ## This value can be "opensource" or "commercial" - # mil.dso.ironbank.image.type: "FIXME" + mil.dso.ironbank.image.type: "opensource" ## Product the image belongs to for grouping multiple images - # mil.dso.ironbank.product.name: "FIXME" + mil.dso.ironbank.product.name: "NiFi" # List of resources to make available to the offline build context resources: @@ -51,15 +51,8 @@ resources: value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 # List of project maintainers -# FIXME: Fill in the following details for the current container owner in the whitelist -# FIXME: Include any other vendor information if applicable maintainers: -- email: "melissari_sean@bah.com" -# # The name of the current container owner -# name: "FIXME" -# # The gitlab username of the current container owner -# username: "FIXME" -# cht_member: true # FIXME: Uncomment if the maintainer is a member of CHT -# - name: "FIXME" -# username: "FIXME" -# email: "FIXME" +- name: "Sean Melissari" + username: "sean.melissari" + email: "melissari_sean@bah.com" + cht_member: true diff --git a/nifi.properties b/nifi.properties index be5a96887f8c6b612a0b09c56a527952a8a2a047..85a86ce0fcc517670d28ec2e596889bc9fd5921a 100644 --- a/nifi.properties +++ b/nifi.properties @@ -61,27 +61,22 @@ nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog nifi.flowfile.repository.directory=./flowfile_repository -nifi.flowfile.repository.partitions=256 -nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.checkpoint.interval=20 secs nifi.flowfile.repository.always.sync=false nifi.flowfile.repository.encryption.key.provider.implementation= nifi.flowfile.repository.encryption.key.provider.location= nifi.flowfile.repository.encryption.key.id= nifi.flowfile.repository.encryption.key= +nifi.flowfile.repository.retain.orphaned.flowfiles=true nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager nifi.queue.swap.threshold=20000 -nifi.swap.in.period=5 sec -nifi.swap.in.threads=1 -nifi.swap.out.period=5 sec -nifi.swap.out.threads=4 # Content Repository nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository nifi.content.claim.max.appendable.size=1 MB -nifi.content.claim.max.flow.files=100 nifi.content.repository.directory.default=./content_repository -nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.retention.period=7 days nifi.content.repository.archive.max.usage.percentage=50% nifi.content.repository.archive.enabled=true nifi.content.repository.always.sync=false @@ -93,7 +88,6 @@ nifi.content.repository.encryption.key= # Provenance Repository Properties nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository -nifi.provenance.repository.debug.frequency=1_000_000 nifi.provenance.repository.encryption.key.provider.implementation= nifi.provenance.repository.encryption.key.provider.location= nifi.provenance.repository.encryption.key.id= @@ -101,9 +95,9 @@ nifi.provenance.repository.encryption.key= # Persistent Provenance Repository Properties nifi.provenance.repository.directory.default=./provenance_repository -nifi.provenance.repository.max.storage.time=24 hours -nifi.provenance.repository.max.storage.size=1 GB -nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.max.storage.time=30 days +nifi.provenance.repository.max.storage.size=10 GB +nifi.provenance.repository.rollover.time=10 mins nifi.provenance.repository.rollover.size=100 MB nifi.provenance.repository.query.threads=2 nifi.provenance.repository.index.threads=2 @@ -132,16 +126,15 @@ nifi.components.status.repository.buffer.size=1440 nifi.components.status.snapshot.frequency=1 min # Site to Site properties -nifi.remote.input.host= +nifi.remote.input.host=f3a79f73876d nifi.remote.input.secure=false -nifi.remote.input.socket.port= +nifi.remote.input.socket.port=10000 nifi.remote.input.http.enabled=true nifi.remote.input.http.transaction.ttl=30 sec nifi.remote.contents.cache.expiration=30 secs # web properties # -nifi.web.war.directory=./lib -nifi.web.http.host= +nifi.web.http.host=f3a79f73876d nifi.web.http.port=8080 nifi.web.http.network.interface.default= nifi.web.https.host= @@ -152,6 +145,9 @@ nifi.web.jetty.threads=200 nifi.web.max.header.size=16 KB nifi.web.proxy.context.path=/ib-nifi/ nifi.web.proxy.host= +nifi.web.max.content.size= +nifi.web.max.requests.per.second=30000 +nifi.web.should.send.server.version=true # security properties # nifi.sensitive.props.key= @@ -168,6 +164,7 @@ nifi.security.truststore= nifi.security.truststoreType= nifi.security.truststorePasswd= nifi.security.user.authorizer=managed-authorizer +nifi.security.allow.anonymous.authentication=false nifi.security.user.login.identity.provider= nifi.security.ocsp.responder.url= nifi.security.ocsp.responder.certificate= @@ -210,11 +207,12 @@ nifi.security.user.knox.audiences= # cluster common properties (all nodes must have same values) # nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.heartbeat.missable.max=8 nifi.cluster.protocol.is.secure=false # cluster node properties (only configure for cluster nodes) # nifi.cluster.is.node=false -nifi.cluster.node.address= +nifi.cluster.node.address=f3a79f73876d nifi.cluster.node.protocol.port= nifi.cluster.node.protocol.threads=10 nifi.cluster.node.protocol.max.threads=50 @@ -229,14 +227,14 @@ nifi.cluster.flow.election.max.candidates= # cluster load balancing properties # nifi.cluster.load.balance.host= nifi.cluster.load.balance.port=6342 -nifi.cluster.load.balance.connections.per.node=4 +nifi.cluster.load.balance.connections.per.node=1 nifi.cluster.load.balance.max.thread.count=8 nifi.cluster.load.balance.comms.timeout=30 sec # zookeeper properties, used for cluster management # nifi.zookeeper.connect.string= -nifi.zookeeper.connect.timeout=3 secs -nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.connect.timeout=10 secs +nifi.zookeeper.session.timeout=10 secs nifi.zookeeper.root.node=/nifi # Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management diff --git a/scripts/common.sh b/scripts/common.sh old mode 100644 new mode 100755 diff --git a/scripts/secure.sh b/scripts/secure.sh old mode 100644 new mode 100755 diff --git a/scripts/start.sh b/scripts/start.sh old mode 100644 new mode 100755 diff --git a/scripts/toolkit.sh b/scripts/toolkit.sh old mode 100644 new mode 100755 index 4da9ccfd14163a0b843acd7de67ff05a4c5c31dc..b071efcb0db273d57a8c96ef7178967d9b3cf414 --- a/scripts/toolkit.sh +++ b/scripts/toolkit.sh @@ -15,105 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -scripts_dir='/opt/nifi/scripts' - -[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" - -# Override JVM memory settings -if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then - prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file} -fi - -if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then - prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file} -fi - -if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then - uncomment "java.arg.debug" ${nifi_bootstrap_file} -fi - -# Establish baseline properties -prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT:-8080}" -prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}" -prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}" -prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" -prop_replace 'nifi.remote.input.secure' 'false' - -# Set nifi-toolkit properties files and baseUrl -"${scripts_dir}/toolkit.sh" -prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file} - -prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" -prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" -prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}" -prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" -prop_replace 'nifi.cluster.node.protocol.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}" -prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" -prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" -prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" -prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" -prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" -prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" - -# Set analytics properties -prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" -prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" -prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" -prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" -prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" -prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" - -. "${scripts_dir}/update_cluster_state_management.sh" - -# Check if we are secured or unsecured -case ${AUTH} in - tls) - echo 'Enabling Two-Way SSL user authentication' - . "${scripts_dir}/secure.sh" - ;; - ldap) - echo 'Enabling LDAP user authentication' - # Reference ldap-provider in properties - prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider' - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_login_providers.sh" - ;; - *) - if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then - echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Will not update nifi.web.proxy.host.' - fi - ;; -esac - -# Continuously provide logs so that 'docker logs' can produce them -"${NIFI_HOME}/bin/nifi.sh" run & -nifi_pid="$!" -tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" & - -trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT; -trap ":" EXIT - -echo NiFi running with PID ${nifi_pid}. -wait ${nifi_pid} -nifi@ade64fede7e3:/opt/nifi/scripts$ cat toolkit.sh -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - cat < ${nifi_toolkit_props_file} baseUrl= keystore= @@ -128,3 +29,4 @@ EOT cat < ${HOME}/.nifi-cli.config nifi.props=${nifi_toolkit_props_file} +EOT diff --git a/scripts/update_cluster_state_management.sh b/scripts/update_cluster_state_management.sh old mode 100644 new mode 100755 diff --git a/scripts/update_login_providers.sh b/scripts/update_login_providers.sh old mode 100644 new mode 100755