Merge branch 'development' into 'master'

Development

See merge request !27

Dockerfile

@@ -6,13 +6,6 @@ FROM apache/nifi:1.12.1 as base
 
 FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
 
-LABEL org.opencontainers.image.title="NiFi" \
-      org.opencontainers.image.description="Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data." \
-      org.opencontainers.image.licenses="Apache-2.0" \
-      org.opencontainers.image.url="https://nifi.apache.org/" \
-      org.opencontainers.image.version="1.12.1" \
-      maintainer="cht@dsop.io"
-
 ENV NIFI_VERSION=1.12.1
 ENV NIFI_BASE_DIR=/opt/nifi
 ENV NIFI_HOME=${NIFI_BASE_DIR}/nifi-current
@@ -30,8 +23,6 @@ COPY xmlstarlet.rpm /xmlstarlet.rpm
 
 RUN groupadd -g 1000 nifi && \
     useradd -r -u 1000 -m -g nifi nifi && \
-    chown nifi:0 ${HOME} && \
-    chmod g=u ${HOME} && \
     dnf update -y && \
     dnf install -y libxslt procps-ng && \
     rpm -i --nosignature /xmlstarlet.rpm && \
@@ -43,10 +34,15 @@ RUN groupadd -g 1000 nifi && \
 
 COPY --chown=nifi:nifi --from=base ${NIFI_BASE_DIR} ${NIFI_BASE_DIR}
 COPY --chown=nifi:nifi nifi.properties ${NIFI_HOME}/conf/nifi.properties
-COPY scripts/ ${NIFI_BASE_DIR}/scripts/
+COPY --chown=nifi:nifi scripts/ ${NIFI_BASE_DIR}/scripts/
 
-RUN chmod +x ${NIFI_BASE_DIR}/scripts/*.sh
+RUN chown -R nifi:nifi ${NIFI_BASE_DIR} && \
+    chmod +x ${NIFI_BASE_DIR}/scripts/*.sh && \
+    chmod -R go-rwx ${NIFI_BASE_DIR} && \
+    echo "#!/bin/sh\n" > $NIFI_HOME/bin/nifi-env.sh
 
+USER ${USER}
+WORKDIR ${NIFI_HOME}
 
 VOLUME ${NIFI_LOG_DIR} \
        ${NIFI_HOME}/conf \
@@ -59,9 +55,6 @@ VOLUME ${NIFI_LOG_DIR} \
 # Web HTTP(s) & Socket Site-to-Site Ports
 EXPOSE 8080 8443 10000 8000
 
-USER ${USER}
-WORKDIR ${NIFI_HOME}
-
 HEALTHCHECK --start-period=60s \
   CMD /opt/nifi/nifi-current/bin/nifi.sh status
 

Jenkinsfile

-@Library('DCCSCR@master') _
-dccscrPipeline(version: "1.12.1")

download.yaml

-resources:
-  - url: "docker://docker.io/apache/nifi@sha256:bf7576ab7ad0bfe38c86be5baa47229d1644287984034dc9d5ff4801c5827115"
-    tag: "apache/nifi:1.12.1"
-  - url: "https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm"
-    filename: xmlstarlet.rpm
-    validation:
-      type: sha256
-      value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0
-  - url: "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64"
-    filename: jq-linux64
-    validation:
-      type: sha256
-      value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44

hardening_manifest.yaml

+---
+apiVersion: v1
+
+# The repository name in registry1, excluding /ironbank/
+name: "opensource/apache/nifi"
+
+# List of tags to push for the repository in registry1
+# The most specific version should be the first tag and will be shown
+# on ironbank.dsop.io
+tags:
+- "1.12.1"
+- "latest"
+
+# Build args passed to Dockerfile ARGs
+args:
+  BASE_IMAGE: "redhat/openjdk/openjdk11"
+  BASE_TAG: "1.11"
+
+# Docker image labels
+labels:
+  org.opencontainers.image.title: "nifi"
+  ## Human-readable description of the software packaged in the image
+  org.opencontainers.image.description: "NiFi is an easy to use, powerful, and reliable system to process and distribute data."
+  ## License(s) under which contained software is distributed
+  org.opencontainers.image.licenses: "Apache-2.0"
+  ## URL to find more information on the image
+  org.opencontainers.image.url: "https://nifi.apache.org"
+  ## Name of the distributing entity, organization or individual
+  org.opencontainers.image.vendor: "Apache"
+  org.opencontainers.image.version: "1.12.1"
+  ## Keywords to help with search (ex. "cicd,gitops,golang")
+  mil.dso.ironbank.image.keywords: "dataflow,processing"
+  ## This value can be "opensource" or "commercial"
+  mil.dso.ironbank.image.type: "opensource"
+  ## Product the image belongs to for grouping multiple images
+  mil.dso.ironbank.product.name: "NiFi"
+
+# List of resources to make available to the offline build context
+resources:
+- tag: apache/nifi:1.12.1
+  url: docker://docker.io/apache/nifi@sha256:bf7576ab7ad0bfe38c86be5baa47229d1644287984034dc9d5ff4801c5827115
+- filename: xmlstarlet.rpm
+  url: https://download-ib01.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/x/xmlstarlet-1.6.1-11.el8.x86_64.rpm
+  validation:
+    type: sha256
+    value: 1db13a198139dbbb4aedc079a2c5f74a01d036f4c816c9aab7f8ef7c10b019f0
+- filename: jq-linux64
+  url: https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
+  validation:
+    type: sha256
+    value: af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44
+
+# List of project maintainers
+maintainers:
+- name: "Sean Melissari"
+  username: "sean.melissari"
+  email: "melissari_sean@bah.com"
+  cht_member: true

nifi.properties

@@ -61,27 +61,22 @@ nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
 nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
 nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog
 nifi.flowfile.repository.directory=./flowfile_repository
-nifi.flowfile.repository.partitions=256
-nifi.flowfile.repository.checkpoint.interval=2 mins
+nifi.flowfile.repository.checkpoint.interval=20 secs
 nifi.flowfile.repository.always.sync=false
 nifi.flowfile.repository.encryption.key.provider.implementation=
 nifi.flowfile.repository.encryption.key.provider.location=
 nifi.flowfile.repository.encryption.key.id=
 nifi.flowfile.repository.encryption.key=
+nifi.flowfile.repository.retain.orphaned.flowfiles=true
 
 nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
 nifi.queue.swap.threshold=20000
-nifi.swap.in.period=5 sec
-nifi.swap.in.threads=1
-nifi.swap.out.period=5 sec
-nifi.swap.out.threads=4
 
 # Content Repository
 nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
 nifi.content.claim.max.appendable.size=1 MB
-nifi.content.claim.max.flow.files=100
 nifi.content.repository.directory.default=./content_repository
-nifi.content.repository.archive.max.retention.period=12 hours
+nifi.content.repository.archive.max.retention.period=7 days
 nifi.content.repository.archive.max.usage.percentage=50%
 nifi.content.repository.archive.enabled=true
 nifi.content.repository.always.sync=false
@@ -93,7 +88,6 @@ nifi.content.repository.encryption.key=
 
 # Provenance Repository Properties
 nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
-nifi.provenance.repository.debug.frequency=1_000_000
 nifi.provenance.repository.encryption.key.provider.implementation=
 nifi.provenance.repository.encryption.key.provider.location=
 nifi.provenance.repository.encryption.key.id=
@@ -101,9 +95,9 @@ nifi.provenance.repository.encryption.key=
 
 # Persistent Provenance Repository Properties
 nifi.provenance.repository.directory.default=./provenance_repository
-nifi.provenance.repository.max.storage.time=24 hours
-nifi.provenance.repository.max.storage.size=1 GB
-nifi.provenance.repository.rollover.time=30 secs
+nifi.provenance.repository.max.storage.time=30 days
+nifi.provenance.repository.max.storage.size=10 GB
+nifi.provenance.repository.rollover.time=10 mins
 nifi.provenance.repository.rollover.size=100 MB
 nifi.provenance.repository.query.threads=2
 nifi.provenance.repository.index.threads=2
@@ -132,16 +126,15 @@ nifi.components.status.repository.buffer.size=1440
 nifi.components.status.snapshot.frequency=1 min
 
 # Site to Site properties
-nifi.remote.input.host=
+nifi.remote.input.host=f3a79f73876d
 nifi.remote.input.secure=false
-nifi.remote.input.socket.port=
+nifi.remote.input.socket.port=10000
 nifi.remote.input.http.enabled=true
 nifi.remote.input.http.transaction.ttl=30 sec
 nifi.remote.contents.cache.expiration=30 secs
 
 # web properties #
-nifi.web.war.directory=./lib
-nifi.web.http.host=
+nifi.web.http.host=f3a79f73876d
 nifi.web.http.port=8080
 nifi.web.http.network.interface.default=
 nifi.web.https.host=
@@ -152,6 +145,9 @@ nifi.web.jetty.threads=200
 nifi.web.max.header.size=16 KB
 nifi.web.proxy.context.path=/ib-nifi/
 nifi.web.proxy.host=
+nifi.web.max.content.size=
+nifi.web.max.requests.per.second=30000
+nifi.web.should.send.server.version=true
 
 # security properties #
 nifi.sensitive.props.key=
@@ -168,6 +164,7 @@ nifi.security.truststore=
 nifi.security.truststoreType=
 nifi.security.truststorePasswd=
 nifi.security.user.authorizer=managed-authorizer
+nifi.security.allow.anonymous.authentication=false
 nifi.security.user.login.identity.provider=
 nifi.security.ocsp.responder.url=
 nifi.security.ocsp.responder.certificate=
@@ -210,11 +207,12 @@ nifi.security.user.knox.audiences=
 
 # cluster common properties (all nodes must have same values) #
 nifi.cluster.protocol.heartbeat.interval=5 sec
+nifi.cluster.protocol.heartbeat.missable.max=8
 nifi.cluster.protocol.is.secure=false
 
 # cluster node properties (only configure for cluster nodes) #
 nifi.cluster.is.node=false
-nifi.cluster.node.address=
+nifi.cluster.node.address=f3a79f73876d
 nifi.cluster.node.protocol.port=
 nifi.cluster.node.protocol.threads=10
 nifi.cluster.node.protocol.max.threads=50
@@ -229,14 +227,14 @@ nifi.cluster.flow.election.max.candidates=
 # cluster load balancing properties #
 nifi.cluster.load.balance.host=
 nifi.cluster.load.balance.port=6342
-nifi.cluster.load.balance.connections.per.node=4
+nifi.cluster.load.balance.connections.per.node=1
 nifi.cluster.load.balance.max.thread.count=8
 nifi.cluster.load.balance.comms.timeout=30 sec
 
 # zookeeper properties, used for cluster management #
 nifi.zookeeper.connect.string=
-nifi.zookeeper.connect.timeout=3 secs
-nifi.zookeeper.session.timeout=3 secs
+nifi.zookeeper.connect.timeout=10 secs
+nifi.zookeeper.session.timeout=10 secs
 nifi.zookeeper.root.node=/nifi
 
 # Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management

renovate.json

 {
-  "assignees": ["@sean.melissari"],
-  "baseBranches": ["development"],
+  "assignees": [
+    "@sean.melissari"
+  ],
+  "baseBranches": [
+    "development"
+  ],
   "regexManagers": [
     {
-      "fileMatch": ["^Dockerfile$"],
+      "fileMatch": [
+        "^Dockerfile$"
+      ],
       "matchStrings": [
         "version=\"(?<currentValue>.*?)\""
       ],
@@ -11,7 +17,9 @@
       "datasourceTemplate": "docker"
     },
     {
-      "fileMatch": ["^Dockerfile$"],
+      "fileMatch": [
+        "^Dockerfile$"
+      ],
       "matchStrings": [
         "VERSION=\"(?<currentValue>.*?)\""
       ],
@@ -19,9 +27,21 @@
       "datasourceTemplate": "docker"
     },
     {
-      "fileMatch": ["^Jenkinsfile$"],
+      "fileMatch": [
+        "^hardening_manifest.yaml$"
+      ],
+      "matchStrings": [
+        "org\\.opencontainers\\.image\\.version:\\s+\"(?<currentValue>.+?)\""
+      ],
+      "depNameTemplate": "apache/nifi",
+      "datasourceTemplate": "docker"
+    },
+    {
+      "fileMatch": [
+        "^hardening_manifest.yaml$"
+      ],
       "matchStrings": [
-        "version:\\s+\"(?<currentValue>.*?)\""
+        "tags:\\s+-\\s+\"(?<currentValue>.+?)\""
       ],
       "depNameTemplate": "apache/nifi",
       "datasourceTemplate": "docker"

scripts/toolkit.sh

@@ -15,105 +15,6 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 
-scripts_dir='/opt/nifi/scripts'
-
-[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh"
-
-# Override JVM memory settings
-if [ ! -z "${NIFI_JVM_HEAP_INIT}" ]; then
-    prop_replace 'java.arg.2'       "-Xms${NIFI_JVM_HEAP_INIT}" ${nifi_bootstrap_file}
-fi
-
-if [ ! -z "${NIFI_JVM_HEAP_MAX}" ]; then
-    prop_replace 'java.arg.3'       "-Xmx${NIFI_JVM_HEAP_MAX}" ${nifi_bootstrap_file}
-fi
-
-if [ ! -z "${NIFI_JVM_DEBUGGER}" ]; then
-    uncomment "java.arg.debug" ${nifi_bootstrap_file}
-fi
-
-# Establish baseline properties
-prop_replace 'nifi.web.http.port'               "${NIFI_WEB_HTTP_PORT:-8080}"
-prop_replace 'nifi.web.http.host'               "${NIFI_WEB_HTTP_HOST:-$HOSTNAME}"
-prop_replace 'nifi.remote.input.host'           "${NIFI_REMOTE_INPUT_HOST:-$HOSTNAME}"
-prop_replace 'nifi.remote.input.socket.port'    "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}"
-prop_replace 'nifi.remote.input.secure'         'false'
-
-# Set nifi-toolkit properties files and baseUrl
-"${scripts_dir}/toolkit.sh"
-prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$HOSTNAME}:${NIFI_WEB_HTTP_PORT:-8080}" ${nifi_toolkit_props_file}
-
-prop_replace 'nifi.variable.registry.properties'    "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}"
-prop_replace 'nifi.cluster.is.node'                         "${NIFI_CLUSTER_IS_NODE:-false}"
-prop_replace 'nifi.cluster.node.address'                    "${NIFI_CLUSTER_ADDRESS:-$HOSTNAME}"
-prop_replace 'nifi.cluster.node.protocol.port'              "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}"
-prop_replace 'nifi.cluster.node.protocol.threads'           "${NIFI_CLUSTER_NODE_PROTOCOL_THREADS:-10}"
-prop_replace 'nifi.cluster.node.protocol.max.threads'       "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}"
-prop_replace 'nifi.zookeeper.connect.string'                "${NIFI_ZK_CONNECT_STRING:-}"
-prop_replace 'nifi.zookeeper.root.node'                     "${NIFI_ZK_ROOT_NODE:-/nifi}"
-prop_replace 'nifi.cluster.flow.election.max.wait.time'     "${NIFI_ELECTION_MAX_WAIT:-5 mins}"
-prop_replace 'nifi.cluster.flow.election.max.candidates'    "${NIFI_ELECTION_MAX_CANDIDATES:-}"
-prop_replace 'nifi.web.proxy.context.path'                  "${NIFI_WEB_PROXY_CONTEXT_PATH:-}"
-
-# Set analytics properties
-prop_replace 'nifi.analytics.predict.enabled'                   "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}"
-prop_replace 'nifi.analytics.predict.interval'                  "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}"
-prop_replace 'nifi.analytics.query.interval'                    "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}"
-prop_replace 'nifi.analytics.connection.model.implementation'   "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}"
-prop_replace 'nifi.analytics.connection.model.score.name'       "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}"
-prop_replace 'nifi.analytics.connection.model.score.threshold'  "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}"
-
-. "${scripts_dir}/update_cluster_state_management.sh"
-
-# Check if we are secured or unsecured
-case ${AUTH} in
-    tls)
-        echo 'Enabling Two-Way SSL user authentication'
-        . "${scripts_dir}/secure.sh"
-        ;;
-    ldap)
-        echo 'Enabling LDAP user authentication'
-        # Reference ldap-provider in properties
-        prop_replace 'nifi.security.user.login.identity.provider' 'ldap-provider'
-
-        . "${scripts_dir}/secure.sh"
-        . "${scripts_dir}/update_login_providers.sh"
-        ;;
-    *)
-        if [ ! -z "${NIFI_WEB_PROXY_HOST}" ]; then
-            echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode.  Will not update nifi.web.proxy.host.'
-        fi
-        ;;
-esac
-
-# Continuously provide logs so that 'docker logs' can    produce them
-"${NIFI_HOME}/bin/nifi.sh" run &
-nifi_pid="$!"
-tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" &
-
-trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT;
-trap ":" EXIT
-
-echo NiFi running with PID ${nifi_pid}.
-wait ${nifi_pid}
-nifi@ade64fede7e3:/opt/nifi/scripts$ cat toolkit.sh
-#!/bin/sh -e
-
-#    Licensed to the Apache Software Foundation (ASF) under one or more
-#    contributor license agreements.  See the NOTICE file distributed with
-#    this work for additional information regarding copyright ownership.
-#    The ASF licenses this file to You under the Apache License, Version 2.0
-#    (the "License"); you may not use this file except in compliance with
-#    the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
 cat <<EOT > ${nifi_toolkit_props_file}
 baseUrl=
 keystore=
@@ -128,3 +29,4 @@ EOT
 
 cat <<EOT > ${HOME}/.nifi-cli.config
 nifi.props=${nifi_toolkit_props_file}
+EOT