Merge branch 'hardening_manifest' into '5.5.2'

Migrate to hardening_manifest.yaml

See merge request !2

.gitignore

+*.rpm
+*.rpm.dependencies
+*.rpm.sha256sum
+*.whl*
+*.gz*
+pip-20.2.2-py2.py3-none-any.whl
+Python-3.6.12.tgz
+archive.key
+build-container.sh
+python-libs_filelist.txt
+.env

Dockerfile

@@ -5,11 +5,6 @@ ARG BASE_TAG=5.5.2
 FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
 USER root
 
-ARG IMAGE_VERSION='5.5.2'
-ENV IMAGE_VERSION=${IMAGE_VERSION}
-ARG DOCKER_UPSTREAM_TAG
-ENV CONFLUENT_VERSION=${DOCKER_UPSTREAM_TAG}
-
 ENV CONNECT_LOG4J_DIR=/mnt/log
 ENV CONNECT_SECRETS_DIR=/mnt/apikeys
 ENV CONNECT_CONFIG_DIR=/mnt/config
@@ -18,24 +13,12 @@ ENV COMPONENT=connect
 ENV DOCKER_SCRIPT_DIR=/opt/caas/bin
 ENV COMPONENT_SCRIPT_DIR=/opt/confluent/etc
 
-LABEL io.k8s.display-name="Confluent Kafka Connect" \
-      io.k8s.description="Confluent Kafka Connect Image for Confluent Operator" \
-      io.openshift.tags="confluent operator,confluent platform,event-streaming,kafka-connect" \
-      name="Kafka Connect" \
-      maintainer="Confluent Platform <operator@confluent.io>" \
-      vendor="Confluent, Inc" \
-      version="$IMAGE_VERSION" \
-      release="$IMAGE_VERSION" \
-      summary="Kafka Connect Image for Confluent Operator" \
-      description="Kafka Connect is a framework for connecting Kafka with external systems such as databases, key-value stores, search indexes, and file systems"
-
 WORKDIR /opt
 
 ## Copy dependencies
-##
-COPY opt/caas/lib/ /opt/caas/lib/
-COPY opt/plugins/ /usr/share/java/cc-base/
-COPY include/opt/caas /opt/caas
+ADD operator-5.5.2-caas_lib.tar.gz /opt/caas/lib/
+ADD operator-5.5.2-plugins.tar.gz /usr/share/java/cc-base/
+ADD cp-server-connect-operator-5.5.2_caas.tar.gz /opt/caas/
 
 RUN yum install -y openssl findutils && yum clean all && mkdir -p "${CONNECT_LOG4J_DIR}" "${CONNECT_SECRETS_DIR}" "${CONNECT_CONFIG_DIR}" "${COMPONENT_SCRIPT_DIR}/${COMPONENT}" "${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jars" \
     && chmod -R ag+w "${CONNECT_LOG4J_DIR}" "${CONNECT_SECRETS_DIR}" "${CONNECT_CONFIG_DIR}" "/opt"

Jenkinsfile

-@Library('DCCSCR@master') _
-dccscrPipeline( version: "5.5.2" )
-

hardening_manifest.yaml

+---
+apiVersion: v1
+
+# The repository name in registry1, excluding /ironbank/
+name: "confluent/confluent-component-operator/cp-server-connect-operator-5.5.x"
+
+# List of tags to push for the repository in registry1
+# The most specific version should be the first tag and will be shown
+# on ironbank.dsop.io
+tags:
+- "5.5.2"
+- "5.5.x"
+- "5.5"
+  
+# Build args passed to Dockerfile ARGs
+args:
+  BASE_IMAGE: "redhat/openjdk/openjdk11"
+  BASE_TAG: "1.11"
+
+# Docker image labels
+labels:
+  org.opencontainers.image.title: "cp-server-connect-operator-5.5.x"
+  org.opencontainers.image.description: "Confluent Operator image for cp-server-connect."
+  org.opencontainers.image.licenses: "CONFLUENT ENTERPRISE LICENSE"
+  org.opencontainers.image.url: "https://docs.confluent.io/5.5.2/installation/operator/index.html"
+  org.opencontainers.image.vendor: "Confluent"
+  org.opencontainers.image.version: "5.5.2"
+  mil.dso.ironbank.image.keywords: "confluent,kafka,zookeeper,operator"
+  mil.dso.ironbank.image.type: "commercial"
+  mil.dso.ironbank.product.name: "Confluent Platform"
+
+# List of resources to make available to the offline build context
+resources:
+- filename: operator-5.5.2-plugins.tar.gz
+  url: https://ironbank-files.s3.amazonaws.com/operator-5.5.2-plugins.tar.gz
+  validation:
+    type: sha256
+    value: 37adbede99aeaf765a9cc16ecdf0b0616f564ddd2cd7364540b186f222edc54d
+- filename: operator-5.5.2-caas_lib.tar.gz
+  url: https://ironbank-files.s3.amazonaws.com/operator-5.5.2-caas_lib.tar.gz
+  validation:
+    type: sha256
+    value: 11112ceb5f4ad55956b4cd631be598acaec7fe754baec52d454ab4bd6222769e  
+- filename: cp-server-connect-operator-5.5.2_caas.tar.gz
+  url: https://ironbank-files.s3.amazonaws.com/cp-server-connect-operator-5.5.2_caas.tar.gz
+  validation: 
+    type: sha256
+    value: 874f5662daa9a49e1d2c7f0a70b13595e8370392c3f0d8afc3b872414d4f24cc
+  
+# List of project maintainers
+maintainers:
+- name: "Scott Stroud"
+  username: "scottstroud"
+  email: "confluent-fed@confluent.io"
+  cht_member: false  
+- name: "Preston McGowan"
+  username: "preston.mcgowan"
+  email: "confluent-fed@confluent.io"
+  cht_member: false  
+  
\ No newline at end of file

include/opt/caas/bin/configure

-#!/usr/bin/env bash
-#
-# Copyright 2018 Confluent Inc.
-#
-
-set -o errexit
-
-dub ensure CONNECT_SECRETS_DIR
-dub ensure CONNECT_LOG4J_DIR
-dub ensure CONNECT_CONFIG_DIR
-dub ensure CAAS_POD_ID
-
-if [ -e /mnt/secrets/jksPassword.txt ]; then
-	export KEYSTOREPASSWORD=$(echo $(cat /mnt/secrets/jksPassword.txt | cut -d '='  -f2-) | sed 's/ *$//g')
-else
-	export KEYSTOREPASSWORD=mystorepassword
-fi
-# Fix for https://issues.apache.org/jira/browse/KAFKA-3988
-if [[ $CONNECT_INTERNAL_KEY_CONVERTER == "org.apache.kafka.connect.json.JsonConverter" ]] || [[ $CONNECT_INTERNAL_VALUE_CONVERTER == "org.apache.kafka.connect.json.JsonConverter" ]]
-then
-  export CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE=false
-  export CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE=false
-fi
-
-if [[ $CONNECT_KEY_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
-then
-  dub ensure CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL
-fi
-
-if [[ $CONNECT_VALUE_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
-then
-  dub ensure CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL
-fi
-
-create_keystore() {
-	keystore_path=$1
-	extra_certfile=${2:-""}
-	if [ -e /mnt/sslcerts/fullchain.pem ] && [ -e /mnt/sslcerts/privkey.pem ]; then
-		args="-export
-			-in /mnt/sslcerts/fullchain.pem
-			-inkey /mnt/sslcerts/privkey.pem
-			-out /tmp/pkcs.p12
-			-name ${COMPONENT}
-			-passout pass:mykeypassword"
-
-		if [ ! -z ${extra_certfile} ] && [ -e ${extra_certfile} ]; then
-			args="${args} -certfile ${extra_certfile}"
-		fi
-		openssl pkcs12 ${args}
-
-    keytool -importkeystore \
-      -deststorepass ${KEYSTOREPASSWORD} \
-      -destkeypass ${KEYSTOREPASSWORD} \
-      -destkeystore ${keystore_path} \
-      -deststoretype pkcs12 \
-      -srckeystore /tmp/pkcs.p12 \
-      -srcstoretype PKCS12 \
-      -srcstorepass mykeypassword 
-
-    fi
-}
-create_keystore /tmp/keystore.jks
-##
-## Put all CAs provided as part of caCerts.pem file to truststore
-##
-if [ -e /mnt/sslcerts/cacerts.pem ]; then
-mkdir -p /tmp/trustCAs
-cat /mnt/sslcerts/cacerts.pem | awk 'split_after==1{n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1} {print > ("/tmp/trustCAs/ca" n ".pem")}'
-for file in /tmp/trustCAs/*; do
-  fileName="${file##*/}"
-  keytool -import \
-          -trustcacerts \
-          -alias "$fileName" \
-          -file "$file" \
-          -keystore /tmp/truststore.jks \
-          -deststorepass ${KEYSTOREPASSWORD} \
-          -noprompt
-done
-fi
-
-cat ${CONNECT_CONFIG_DIR}/shared/${COMPONENT}.properties > ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/${COMPONENT}.properties
-cat ${CONNECT_CONFIG_DIR}/${COMPONENT}.properties >> ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/${COMPONENT}.properties
-cat ${CONNECT_CONFIG_DIR}/shared/log4j.properties >  ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/log4j.properties
-cat ${CONNECT_CONFIG_DIR}/log4j.properties >> ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/log4j.properties
-cat ${CONNECT_CONFIG_DIR}/shared/admin.properties > ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/admin.properties
-cat ${CONNECT_CONFIG_DIR}/shared/jvm.config > ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jvm.config
-cat ${CONNECT_CONFIG_DIR}/jvm.config >> ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jvm.config
-if [ -e ${CONNECT_CONFIG_DIR}/shared/jolokia.config ]; then
-  create_keystore /tmp/jolokia-keystore.jks /mnt/sslcerts/cacerts.pem
-  cat ${CONNECT_CONFIG_DIR}/shared/jolokia.config > ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jolokia.config
-fi
-cat ${CONNECT_CONFIG_DIR}/shared/disk-usage-agent.properties >  ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/disk-usage-agent.properties
-cp /opt/caas/templates/jmx-exporter.yaml.j2 ${COMPONENT_SCRIPT_DIR}/jmx-exporter.yaml
-find ${COMPONENT_SCRIPT_DIR} -type f | xargs sed -i  "s/<<keystorepassword>>/${KEYSTOREPASSWORD}/g"

include/opt/caas/bin/ensure

-#!/usr/bin/env bash
-#
-# Copyright 2018 Confluent Inc.
-#
-
-set -o errexit
-
-echo "===> Check if Kafka is healthy ..."
-
-export KAFKA_CONNECT_BOOTSTRAP_SERVERS=$(grep "^bootstrap.servers" "${COMPONENT_SCRIPT_DIR}"/"${COMPONENT}"/"${COMPONENT}".properties | head -1 | cut -d '=' -f 2)
-echo ${KAFKA_CONNECT_BOOTSTRAP_SERVERS}
-cub kafka-ready \
-    "${KAFKA_CONNECT_CUB_KAFKA_MIN_BROKERS:-1}" \
-    "${KAFKA_CONNECT_CUB_KAFKA_TIMEOUT:-40}" \
-    -b "${KAFKA_CONNECT_BOOTSTRAP_SERVERS}" \
-    --config ${COMPONENT_SCRIPT_DIR}/"${COMPONENT}"/admin.properties

include/opt/caas/bin/launch

-#!/usr/bin/env bash
-#
-# Copyright 2018 Confluent Inc.
-#
-set -o errexit
-
-#echo "===> Adding disk usage agent to the java command ... " 
-#export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${COMPONENT_SCRIPT_DIR}/${COMPONENT}/disk-usage-agent.properties"
-
-echo "===> Adding jolokia agent to the java command ... " 
-export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777}
-export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"}
-if [ -e "${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jolokia.config" ]; then
-export JOLOKIA_EXTRA_ARGS=",$(cat ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jolokia.config | xargs)"
-fi
-export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}"
-
-echo "===> Adding jmx exporter to the java command ... " 
-export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778}
-export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"}
-export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:${COMPONENT_SCRIPT_DIR}/jmx-exporter.yaml"
-
-echo "===> Adding log4j config ... "
-export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${COMPONENT_SCRIPT_DIR}/${COMPONENT}/log4j.properties"
-
-echo "===> Adding JVM config to the java command ..."
-export EXTRA_ARGS="$(cat ${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jvm.config | xargs) ${EXTRA_ARGS}"
-# These ensure that the "if" sections for heap sizing, GC tuning, and JMX opts in kafka launch script do not trigger.
-export KAFKA_HEAP_OPTS=' '
-export KAFKA_JVM_PERFORMANCE_OPTS=' '
-export JMX_PORT=' '
-# Add external jars to the classpath
-# And this also makes sure that the CLASSPATH does not start with ":/etc/..."
-# because this causes the plugin scanner to scan the entire disk.
-export CLASSPATH="${COMPONENT_SCRIPT_DIR}/${COMPONENT}/jars/*"
-exec connect-distributed ${COMPONENT_SCRIPT_DIR}/"${COMPONENT}"/"${COMPONENT}".properties
\ No newline at end of file

include/opt/caas/bin/run

-#!/usr/bin/env bash
-#
-# Copyright 2018 Confluent Inc.
-#
-set -o nounset \
-    -o errexit
-
-echo "===> User"
-id
-
-echo "===> Configuring ..."
-${DOCKER_SCRIPT_DIR}/configure
-
-echo "===> Running preflight checks ... "
-${DOCKER_SCRIPT_DIR}/ensure
-
-echo "===> Launching ... "
-exec ${DOCKER_SCRIPT_DIR}/launch
\ No newline at end of file

include/opt/caas/templates/jmx-exporter.yaml.j2

-lowercaseOutputLabelNames: false
-lowercaseOutputName: true

licenses/licenses/confluent.txt

-Copyright 2020 Confluent, Inc.