Merge branch 'development' into 'master'

Development

See merge request !21

scripts/confluentinc/zookeeper/bin/configure

+#!/usr/bin/env bash
+#
+# Copyright 2020 Confluent Inc.
+#
+set -o nounset
+set -o errexit
+
+if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then
+ mkdir -p "${CP_COMPONENT_SCRIPT_DIR}"
+fi
+
+# load configuration
+cat "${ZOOKEEPER_CONFIG_DIR}/myid" > "${ZOOKEEPER_DATA_DIR}/myid"
+cat "${ZOOKEEPER_CONFIG_DIR}"/shared/zookeeper.properties > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties"
+cat "${ZOOKEEPER_CONFIG_DIR}"/zookeeper.properties >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties"
+cat "${ZOOKEEPER_CONFIG_DIR}"/shared/log4j.properties >  "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties"
+cat "${ZOOKEEPER_CONFIG_DIR}"/log4j.properties >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties"
+cat "${ZOOKEEPER_CONFIG_DIR}"/shared/disk-usage-agent.properties >  "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties"
+cat "${ZOOKEEPER_CONFIG_DIR}"/shared/jvm.config > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config"
+cat "${ZOOKEEPER_CONFIG_DIR}"/jvm.config >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config"
+if [ -e "${ZOOKEEPER_CONFIG_DIR}/shared/jolokia.config" ]; then
+  cat "${ZOOKEEPER_CONFIG_DIR}"/shared/jolokia.config > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config"
+fi
+

scripts/confluentinc/zookeeper/bin/launch

+#!/usr/bin/env bash
+#
+# Copyright 2018 Confluent Inc.
+#
+set -o errexit
+
+# Zookeeper myid
+echo "===> Printing ${ZOOKEEPER_DATA_DIR}/myid: $(cat "${ZOOKEEPER_DATA_DIR}/myid")"
+echo "===> Configure log4j"
+export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties"
+##
+## This condition is only required to work with older images
+## TODO, this code needs to move removed as the metrics configurion
+## is moved to the Operator code.
+##
+if [ -d /opt/confluent/etc ]; then
+
+  echo "===> Configure disk usage agent"
+  export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties"
+
+  echo "===> Configure jolokia agent"
+  export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777}
+  export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"}
+  if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then
+    export JOLOKIA_EXTRA_ARGS=",$(cat  < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")"
+  fi
+
+  if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then
+     export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}"
+  else
+     export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}"
+  fi
+
+  echo "===> Configure prometheus JMX agent"
+  export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778}
+  export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"}
+  export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml"
+  ## requires for now but have to remove in the future iterations
+  ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of
+  ## new stack deployment
+  echo "===> Configure JVM config..."
+  export EXTRA_ARGS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') $EXTRA_ARGS"
+else
+  echo "===> Configure JVM config..."
+  export EXTRA_ARGS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${EXTRA_ARGS}"
+fi
+
+# These ensure that the "if" sections for heap sizing and GC tuning in kafka launch script does not trigger.
+export KAFKA_HEAP_OPTS=' '
+export KAFKA_JVM_PERFORMANCE_OPTS=' '
+echo "===> Launching ${CP_COMPONENT_NAME} ..."
+
+exec zookeeper-server-start "${CP_COMPONENT_SCRIPT_DIR}/zookeeper.properties"

scripts/confluentinc/zookeeper/bin/run

+#!/usr/bin/env bash
+#
+# Copyright 2020 Confluent Inc.
+#
+set -o nounset
+set -o errexit
+
+export CP_COMPONENT_NAME=zookeeper
+export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME}
+export ZOOKEEPER_CONFIG_DIR=/mnt/config
+export ZOOKEEPER_DATA_DIR=/mnt/data/data
+
+echo "===> User"
+id
+
+export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME}
+echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin"
+
+"${OPERATOR_SCRIPT_DIR}"/bin/configure
+"${OPERATOR_SCRIPT_DIR}"/bin/launch

scripts/parser.py

@@ -13,9 +13,14 @@ def create_file(file_name, data):
     '''
     with open(file_name, 'w') as out_file:
         for key, value in data.items():
-            if isinstance(value, int) or isinstance(value, float) :
+            if isinstance(value, int) or isinstance(value, float):
                 value = str(value)
-            out_file.write(key+"="+value+"\n")
+            if isinstance(value, bool):
+                value = str(value).lower()
+            if len(value) == 0:
+                out_file.write(key + "\n")
+            else:
+                out_file.write(key+"="+value+"\n")
 
 def read_file(file_name):
     '''

scripts/startup.sh

 #!/usr/bin/env sh
 
 set -ox errexit
-if [[ -z ${POD_NAME} ]]; then
+if [[ -z "${POD_NAME}" ]]; then
     echo "Environment variable POD_NAME not found..."
     env
     exit
 fi
-pod_id=${POD_NAME}
-id=${pod_id##*-};
-pod_name=${POD_NAME%-*}
-cat /mnt/config/pod/${pod_name}/template.jsonnet | base64 -d > /opt/template.jsonnet
-/usr/local/bin/jsonnet /opt/template.jsonnet --ext-str id=${id} -o /opt/${pod_name}.json
-cat /opt/${pod_name}.json
-/opt/parser.py ${pod_name} /opt/${pod_name}.json /mnt/config
+
+pod_name=${POD_NAME}
+id=${pod_name##*-};
+component_name=${POD_NAME%-*}
+
+if [ -e "/mnt/config/pod/${component_name}/template.jsonnet" ]; then
+  cat < /mnt/config/pod/"${component_name}"/template.jsonnet | base64 -d > /opt/template.jsonnet
+  /usr/local/bin/jsonnet /opt/template.jsonnet --ext-str id="${id}" -o /opt/"${component_name}".json
+  cat /opt/"${component_name}".json
+  /opt/parser.py "${component_name}" "/opt/${component_name}.json" /mnt/config
+elif [ -e "/mnt/config/init/template.jsonnet" ]; then
+  # new version of operator 2.0
+  cat < /mnt/config/init/template.jsonnet | base64 -d > /mnt/config/template.jsonnet
+  cat < /mnt/config/init/data.json | base64 -d > /mnt/config/data.json
+  /opt/config.py "${pod_name}" /mnt/config/data.json /mnt/config/template.jsonnet /mnt/config/"${component_name}".json
+  cat /mnt/config/"${component_name}".json
+  /opt/parser.py "${component_name}" /mnt/config/"${component_name}".json /mnt/config
+  # write to both locations to support image overrides
+  mkdir -p /mnt/config/pod/"${pod_name}"
+  /opt/parser.py "${component_name}" /mnt/config/"${component_name}".json /mnt/config/pod/"${pod_name}"
+  # clean up excess output files
+  rm /mnt/config/template.jsonnet /mnt/config/data.json /mnt/config/"${component_name}".json
+else
+  echo "template.jsonnet not found in the path"
+  exit 1
+fi
+
+# load scripts accordingly
+if [ -n "${OPERATOR_CP_TYPE}" ]; then
+echo "load script for CP type: ${OPERATOR_CP_TYPE}"
+  if [ "${OPERATOR_CP_TYPE}" = "zookeeper" ]; then
+    cp -r /opt/confluentinc/zookeeper/ /mnt/config/
+  elif [ "${OPERATOR_CP_TYPE}" = "controlcenter" ]; then
+    cp -r /opt/confluentinc/controlcenter/ /mnt/config/
+  elif [ "${OPERATOR_CP_TYPE}" = "kafka" ]; then
+    cp -r /opt/confluentinc/kafka/ /mnt/config/
+  elif [ "${OPERATOR_CP_TYPE}" = "replicator" ]; then
+    cp -r /opt/confluentinc/replicator/ /mnt/config/
+  elif [ "${OPERATOR_CP_TYPE}" = "connect" ]; then
+    cp -r /opt/confluentinc/connect/ /mnt/config/
+  elif [ "${OPERATOR_CP_TYPE}" = "ksqldb" ]; then
+    cp -r /opt/confluentinc/ksqldb/ /mnt/config/
+  elif [ "${OPERATOR_CP_TYPE}" = "schemaregistry" ]; then
+    cp -r /opt/confluentinc/schemaregistry/ /mnt/config/
+  fi
+fi