diff --git a/Dockerfile b/Dockerfile index 04a79df2bda1c6df12b69dedac6b44002838b590..5e1cd641e97acda7b72b210e856d09492c38e3f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,9 +27,10 @@ USER root ######################################## ## Install Python -RUN dnf update -y && dnf install -y python38.x86_64 python38-pip-wheel.noarch \ +RUN dnf update -y && dnf install -y python38.x86_64 python38-pip-wheel.noarch openssl tar procps iputils hostname \ && ln -s /usr/bin/python3 /usr/bin/python \ && ln -s /usr/bin/pip3 /usr/bin/pip \ + && alias python=python3 \ && dnf clean all \ && rm -rf /var/cache/dnf diff --git a/scripts/config.py b/scripts/config.py new file mode 100755 index 0000000000000000000000000000000000000000..ffe8fcc01a6b91c672dfcfd44b1b4db71fe982af --- /dev/null +++ b/scripts/config.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +import json +import sys +import os +import _jsonnet +import requests + +# Rack awareness logic +node_labels = [] +rack_labels = [] +env_var = 'RACK_NODE_LABELS' +token_path = '/var/run/secrets/kubernetes.io/serviceaccount/token' +if env_var in os.environ: + rack_labels = json.loads(os.environ[env_var]) + service_host = os.environ['KUBERNETES_SERVICE_HOST'] + tcp_port = os.environ['KUBERNETES_PORT_443_TCP_PORT'] + hostname = os.environ['HOSTNAME'] + namespace = os.environ['POD_NAMESPACE'] + + with open(token_path) as f: + kube_token = f.read() + headers = {'Authorization': 'Bearer %s' % kube_token} + + # get label value for rack awareness node label + api_url = 'https://%s:%s/api/v1/' % (service_host, tcp_port) + + r = requests.get(api_url + 'namespaces/%s/pods/%s' % (namespace, hostname), headers=headers, verify=False) + if r.status_code != 200: + sys.exit(r.json()) + node_name = r.json()['spec']['nodeName'] + + r = requests.get(api_url + 'nodes/%s' % node_name, headers=headers, verify=False) + if r.status_code != 200: + sys.exit(r.json()) + + node_labels = r.json()['metadata']['labels'] + for label in rack_labels: + if label not in node_labels: + sys.exit('node label: %s not found in pod node metadata' % label) + +pod_name = sys.argv[1] +data_file = sys.argv[2] +jsonnet_file = sys.argv[3] +output_file = sys.argv[4] + +with open(data_file) as df: + data = json.load(df) + +with open(jsonnet_file, "r") as jf: + jsonnet_str = jf.read() + +json_str = _jsonnet.evaluate_snippet( + "snippet", + jsonnet_str, + ext_vars={ + 'podName': pod_name, + 'data': json.dumps(data), + 'nodeLabels': json.dumps(node_labels), + 'rackLabels': json.dumps(rack_labels) + }, +) + +json_obj = json.loads(json_str) +with open(output_file, 'w') as of: + json.dump(json_obj, of) diff --git a/scripts/confluentinc/connect/bin/configure b/scripts/confluentinc/connect/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..fc33f25bca0df50becd69c1d8eb9ed5fb1cb7341 --- /dev/null +++ b/scripts/confluentinc/connect/bin/configure @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +cat "${CONNECT_CONFIG_DIR}/shared/${CP_COMPONENT_NAME}.properties" > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${CONNECT_CONFIG_DIR}/${CP_COMPONENT_NAME}.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${CONNECT_CONFIG_DIR}/shared/log4j.properties" > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${CONNECT_CONFIG_DIR}/log4j.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${CONNECT_CONFIG_DIR}/shared/disk-usage-agent.properties" > "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" +cat "${CONNECT_CONFIG_DIR}/shared/jvm.config" > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${CONNECT_CONFIG_DIR}/jvm.config" >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${CONNECT_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${CONNECT_CONFIG_DIR}/shared/jolokia.config" > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi diff --git a/scripts/confluentinc/connect/bin/launch b/scripts/confluentinc/connect/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..57415b110b0a7b509c48f2fdde50c3e7fc9fd160 --- /dev/null +++ b/scripts/confluentinc/connect/bin/launch @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o errexit + +echo "===> Adding log4j config ... " +export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" + +## +## This condition is only required to work with older images +## TODO, this code needs to move removed as the metrics configurion +## is moved to the Operator code. +## +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure disk usage agent" + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + EXTRA_ARGS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') $EXTRA_ARGS" + export EXTRA_ARGS +else + echo "===> Configure JVM config..." + EXTRA_ARGS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${EXTRA_ARGS}" + export EXTRA_ARGS +fi + +export KAFKA_HEAP_OPTS=' ' +export KAFKA_JVM_PERFORMANCE_OPTS=' ' +export JMX_PORT=' ' +exec connect-distributed "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" \ No newline at end of file diff --git a/scripts/confluentinc/connect/bin/run b/scripts/confluentinc/connect/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..7393b017248989beda39e22ec509923f26c2a343 --- /dev/null +++ b/scripts/confluentinc/connect/bin/run @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=connect +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export CONNECT_CONFIG_DIR=/mnt/config + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch \ No newline at end of file diff --git a/scripts/confluentinc/controlcenter/bin/configure b/scripts/confluentinc/controlcenter/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..f3c844aa6b22712d55f425066e06f2d73aa4052a --- /dev/null +++ b/scripts/confluentinc/controlcenter/bin/configure @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +cat "${CONTROL_CENTER_CONFIG_DIR}/shared/${CP_COMPONENT_NAME}.properties" > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${CONTROL_CENTER_CONFIG_DIR}/${CP_COMPONENT_NAME}.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${CONTROL_CENTER_CONFIG_DIR}/shared/log4j.properties" > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${CONTROL_CENTER_CONFIG_DIR}/log4j.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${CONTROL_CENTER_CONFIG_DIR}/shared/disk-usage-agent.properties" > "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" +cat "${CONTROL_CENTER_CONFIG_DIR}/shared/jvm.config" > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${CONTROL_CENTER_CONFIG_DIR}/jvm.config" >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${CONTROL_CENTER_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${CONTROL_CENTER_CONFIG_DIR}/shared/jolokia.config" > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi diff --git a/scripts/confluentinc/controlcenter/bin/launch b/scripts/confluentinc/controlcenter/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..d9d4deb5a9a65d1bf3d408e188f4789515b4736c --- /dev/null +++ b/scripts/confluentinc/controlcenter/bin/launch @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o errexit + +echo "===> Adding log4j config ... " +export CONTROL_CENTER_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +## +## This condition is only required to work with older images +## TODO, this code needs to move removed as the metrics configurion +## is moved to the Operator code. +## +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure disk usage agent" + export CONTROL_CENTER_OPTS="${CONTROL_CENTER_OPTS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export CONTROL_CENTER_OPTS="${CONTROL_CENTER_OPTS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export CONTROL_CENTER_OPTS="${CONTROL_CENTER_OPTS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export CONTROL_CENTER_OPTS="${CONTROL_CENTER_OPTS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + CONTROL_CENTER_OPTS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') $CONTROL_CENTER_OPTS" + export CONTROL_CENTER_OPTS +else + echo "===> Configure JVM config..." + CONTROL_CENTER_OPTS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${CONTROL_CENTER_OPTS}" + export CONTROL_CENTER_OPTS +fi + +export CONTROL_CENTER_HEAP_OPTS=' ' +export CONTROL_CENTER_JVM_PERFORMANCE_OPTS=' ' +export CONTROL_CENTER_JMX_OPTS=' ' +export JMX_PORT=' ' +exec control-center-start "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" \ No newline at end of file diff --git a/scripts/confluentinc/controlcenter/bin/run b/scripts/confluentinc/controlcenter/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..cd7f3032a39d948263fe990902a66c8c4900ee68 --- /dev/null +++ b/scripts/confluentinc/controlcenter/bin/run @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=controlcenter +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export CONTROL_CENTER_CONFIG_DIR=/mnt/config + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch \ No newline at end of file diff --git a/scripts/confluentinc/kafka/bin/configure b/scripts/confluentinc/kafka/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..b26fd2cb52d64bcfce21ba1a8b5136d9707d1a49 --- /dev/null +++ b/scripts/confluentinc/kafka/bin/configure @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +cat "${KAFKA_CONFIG_DIR}/shared/${CP_COMPONENT_NAME}.properties" > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${KAFKA_CONFIG_DIR}/${CP_COMPONENT_NAME}.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${KAFKA_CONFIG_DIR}/shared/log4j.properties" > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${KAFKA_CONFIG_DIR}/log4j.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${KAFKA_CONFIG_DIR}/shared/disk-usage-agent.properties" > "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" +cat "${KAFKA_CONFIG_DIR}/shared/jvm.config" > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${KAFKA_CONFIG_DIR}/jvm.config" >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${KAFKA_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${KAFKA_CONFIG_DIR}/shared/jolokia.config" > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi diff --git a/scripts/confluentinc/kafka/bin/launch b/scripts/confluentinc/kafka/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..04250dc1276c0048d64b761730c6ade8aab375b0 --- /dev/null +++ b/scripts/confluentinc/kafka/bin/launch @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o errexit + +echo "===> Configure log4j" +export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +## +## This condition is only required to work with older images +## TODO, this code needs to move removed as the metrics configurion +## is moved to the Operator code. +## +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure disk usage agent" + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + EXTRA_ARGS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') ${EXTRA_ARGS}" + export EXTRA_ARGS +else + echo "===> Configure JVM config..." + EXTRA_ARGS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${EXTRA_ARGS}" + export EXTRA_ARGS +fi + +export KAFKA_HEAP_OPTS=' ' +export KAFKA_JVM_PERFORMANCE_OPTS=' ' +export JMX_PORT=' ' + +echo "===> Launching ${CP_COMPONENT_NAME} ... " +# shellcheck disable=SC2086 +exec kafka-run-class ${EXTRA_ARGS} kafka.Kafka "${CP_COMPONENT_SCRIPT_DIR}/kafka.properties" diff --git a/scripts/confluentinc/kafka/bin/run b/scripts/confluentinc/kafka/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..c482383ff655bde3ea1911b35318d9b423fecbda --- /dev/null +++ b/scripts/confluentinc/kafka/bin/run @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=kafka +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export KAFKA_CONFIG_DIR=/mnt/config + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch + diff --git a/scripts/confluentinc/ksqldb/bin/configure b/scripts/confluentinc/ksqldb/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..2e48690106e0915ccc2ffc880fc002e95fb50951 --- /dev/null +++ b/scripts/confluentinc/ksqldb/bin/configure @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# + +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +cat "${KSQL_CONFIG_DIR}/shared/${CP_COMPONENT_NAME}.properties" > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${KSQL_CONFIG_DIR}/${CP_COMPONENT_NAME}.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${KSQL_CONFIG_DIR}/shared/log4j.properties" > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${KSQL_CONFIG_DIR}/log4j.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${KSQL_CONFIG_DIR}/shared/disk-usage-agent.properties" > "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" +cat "${KSQL_CONFIG_DIR}/shared/jvm.config" > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${KSQL_CONFIG_DIR}/jvm.config" >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${KSQL_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${KSQL_CONFIG_DIR}/shared/jolokia.config" > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi +# backward compatibility +if [ -e "${KSQL_CONFIG_DIR}/shared/server_jaas.conf" ]; then + cat "${KSQL_CONFIG_DIR}/shared/server_jaas.conf > ${CP_COMPONENT_SCRIPT_DIR}/server_jaas.conf" +fi diff --git a/scripts/confluentinc/ksqldb/bin/launch b/scripts/confluentinc/ksqldb/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..ab1ca01105c47512b7efefa222e48fb1282a4592 --- /dev/null +++ b/scripts/confluentinc/ksqldb/bin/launch @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# Copyright 2019 Confluent Inc. +# +set -o errexit + +echo "===> Adding log4j config ... " +export KSQL_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +## +## This condition is only required to work with older images +## TODO, this code needs to move removed as the metrics configurion +## is moved to the Operator code. +## +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure disk usage agent" + export KSQL_OPTS="${KSQL_OPTS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export KSQL_OPTS="${KSQL_OPTS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export KSQL_OPTS="${KSQL_OPTS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export KSQL_OPTS="${KSQL_OPTS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + KSQL_OPTS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') $KSQL_OPTS" + export KSQL_OPTS + # for basic authentication, only for backward compabilitiy + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/server_jaas.conf" ]; then + export KSQL_OPTS="${KSQL_OPTS} -Djava.security.auth.login.config=${CP_COMPONENT_SCRIPT_DIR}/server_jaas.conf" + fi + echo "===> Adding KSQL healthcheck Jars in classPath ... " + export KSQL_CLASSPATH=${KSQL_CLASSPATH}:"/opt/confluent/ksql/libs/*" +else + echo "===> Configure JVM config..." + KSQL_OPTS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${KSQL_OPTS}" + export KSQL_OPTS +fi + +# These ensure that the "if" sections for heap sizing, GC tuning, and JMX opts in SR launch script do not trigger. +export KSQL_HEAP_OPTS=' ' +export KSQL_JMX_OPTS=' ' +export KSQL_JVM_PERFORMANCE_OPTS=' ' +export JMX_PORT=' ' +## not in-us, just a place-holder +export LOG_DIR=/mnt/log + +echo "===> Launching ${CP_COMPONENT_NAME} ... " +exec ksql-run-class io.confluent.ksql.rest.server.KsqlServerMain "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" diff --git a/scripts/confluentinc/ksqldb/bin/run b/scripts/confluentinc/ksqldb/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..4dd1f699aa7f3abdfd343a87d995d172f8c4ed91 --- /dev/null +++ b/scripts/confluentinc/ksqldb/bin/run @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=ksqldb +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export KSQL_CONFIG_DIR=/mnt/config + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch \ No newline at end of file diff --git a/scripts/confluentinc/replicator/bin/configure b/scripts/confluentinc/replicator/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..fc33f25bca0df50becd69c1d8eb9ed5fb1cb7341 --- /dev/null +++ b/scripts/confluentinc/replicator/bin/configure @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +cat "${CONNECT_CONFIG_DIR}/shared/${CP_COMPONENT_NAME}.properties" > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${CONNECT_CONFIG_DIR}/${CP_COMPONENT_NAME}.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${CONNECT_CONFIG_DIR}/shared/log4j.properties" > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${CONNECT_CONFIG_DIR}/log4j.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${CONNECT_CONFIG_DIR}/shared/disk-usage-agent.properties" > "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" +cat "${CONNECT_CONFIG_DIR}/shared/jvm.config" > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${CONNECT_CONFIG_DIR}/jvm.config" >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${CONNECT_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${CONNECT_CONFIG_DIR}/shared/jolokia.config" > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi diff --git a/scripts/confluentinc/replicator/bin/launch b/scripts/confluentinc/replicator/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..57415b110b0a7b509c48f2fdde50c3e7fc9fd160 --- /dev/null +++ b/scripts/confluentinc/replicator/bin/launch @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o errexit + +echo "===> Adding log4j config ... " +export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" + +## +## This condition is only required to work with older images +## TODO, this code needs to move removed as the metrics configurion +## is moved to the Operator code. +## +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure disk usage agent" + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + EXTRA_ARGS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') $EXTRA_ARGS" + export EXTRA_ARGS +else + echo "===> Configure JVM config..." + EXTRA_ARGS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${EXTRA_ARGS}" + export EXTRA_ARGS +fi + +export KAFKA_HEAP_OPTS=' ' +export KAFKA_JVM_PERFORMANCE_OPTS=' ' +export JMX_PORT=' ' +exec connect-distributed "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" \ No newline at end of file diff --git a/scripts/confluentinc/replicator/bin/run b/scripts/confluentinc/replicator/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..be125474a383ba69dfa33693304d8302ab401010 --- /dev/null +++ b/scripts/confluentinc/replicator/bin/run @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=replicator +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export CONNECT_CONFIG_DIR=/mnt/config + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch \ No newline at end of file diff --git a/scripts/confluentinc/schemaregistry/bin/configure b/scripts/confluentinc/schemaregistry/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..f353d0c94d09c655b6a498e5046444734674d7ca --- /dev/null +++ b/scripts/confluentinc/schemaregistry/bin/configure @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +cat "${SR_CONFIG_DIR}/shared/${CP_COMPONENT_NAME}.properties" > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${SR_CONFIG_DIR}/${CP_COMPONENT_NAME}.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${SR_CONFIG_DIR}/shared/log4j.properties" > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${SR_CONFIG_DIR}/log4j.properties" >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${SR_CONFIG_DIR}/shared/jvm.config" > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${SR_CONFIG_DIR}/jvm.config" >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${SR_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${SR_CONFIG_DIR}/shared/jolokia.config" > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi diff --git a/scripts/confluentinc/schemaregistry/bin/launch b/scripts/confluentinc/schemaregistry/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..e55a73146a48f390c0767c9844703af8f171802e --- /dev/null +++ b/scripts/confluentinc/schemaregistry/bin/launch @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Copyright 2018 Confluent Inc. +# +set -o errexit + +echo "===> Adding log4j config ... " +export SCHEMA_REGISTRY_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export SCHEMA_REGISTRY_OPTS="${SCHEMA_REGISTRY_OPTS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export SCHEMA_REGISTRY_OPTS="${SCHEMA_REGISTRY_OPTS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export SCHEMA_REGISTRY_OPTS="${SCHEMA_REGISTRY_OPTS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + SCHEMA_REGISTRY_OPTS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') ${SCHEMA_REGISTRY_OPTS}" + export SCHEMA_REGISTRY_OPTS +else + echo "===> Configure JVM config..." + SCHEMA_REGISTRY_OPTS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${SCHEMA_REGISTRY_OPTS}" + export SCHEMA_REGISTRY_OPTS +fi + +# These ensure that the "if" sections for heap sizing, GC tuning, and JMX opts in SR launch script do not trigger. +export SCHEMA_REGISTRY_HEAP_OPTS=' ' +export SCHEMA_REGISTRY_JMX_OPTS=' ' +export SCHEMA_REGISTRY_JVM_PERFORMANCE_OPTS=' ' +export JMX_PORT=' ' + +exec schema-registry-start "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" diff --git a/scripts/confluentinc/schemaregistry/bin/run b/scripts/confluentinc/schemaregistry/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..c744593f429f8781e38d7ae18e2681cde66cd20f --- /dev/null +++ b/scripts/confluentinc/schemaregistry/bin/run @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=schemaregistry +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export SR_CONFIG_DIR=/mnt/config + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch \ No newline at end of file diff --git a/scripts/confluentinc/zookeeper/bin/configure b/scripts/confluentinc/zookeeper/bin/configure new file mode 100755 index 0000000000000000000000000000000000000000..921b8829283815143b625d9b39d3bd5e500e5572 --- /dev/null +++ b/scripts/confluentinc/zookeeper/bin/configure @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +if [ ! -d "${CP_COMPONENT_SCRIPT_DIR}" ]; then + mkdir -p "${CP_COMPONENT_SCRIPT_DIR}" +fi + +# load configuration +cat "${ZOOKEEPER_CONFIG_DIR}/myid" > "${ZOOKEEPER_DATA_DIR}/myid" +cat "${ZOOKEEPER_CONFIG_DIR}"/shared/zookeeper.properties > "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${ZOOKEEPER_CONFIG_DIR}"/zookeeper.properties >> "${CP_COMPONENT_SCRIPT_DIR}/${CP_COMPONENT_NAME}.properties" +cat "${ZOOKEEPER_CONFIG_DIR}"/shared/log4j.properties > "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${ZOOKEEPER_CONFIG_DIR}"/log4j.properties >> "${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +cat "${ZOOKEEPER_CONFIG_DIR}"/shared/disk-usage-agent.properties > "${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" +cat "${ZOOKEEPER_CONFIG_DIR}"/shared/jvm.config > "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +cat "${ZOOKEEPER_CONFIG_DIR}"/jvm.config >> "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" +if [ -e "${ZOOKEEPER_CONFIG_DIR}/shared/jolokia.config" ]; then + cat "${ZOOKEEPER_CONFIG_DIR}"/shared/jolokia.config > "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" +fi + diff --git a/scripts/confluentinc/zookeeper/bin/launch b/scripts/confluentinc/zookeeper/bin/launch new file mode 100755 index 0000000000000000000000000000000000000000..e6dbceafc2b7a0a463d2e6b2bfb22eb841ca1340 --- /dev/null +++ b/scripts/confluentinc/zookeeper/bin/launch @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# Copyright 2018 Confluent Inc. +# +set -o errexit + +# Zookeeper myid +echo "===> Printing ${ZOOKEEPER_DATA_DIR}/myid: $(cat "${ZOOKEEPER_DATA_DIR}/myid")" +echo "===> Configure log4j" +export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${CP_COMPONENT_SCRIPT_DIR}/log4j.properties" +## +## This condition is only required to work with older images +## TODO, this code needs to move removed as the metrics configurion +## is moved to the Operator code. +## +if [ -d /opt/confluent/etc ]; then + + echo "===> Configure disk usage agent" + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/usr/share/java/cc-base/disk-usage-agent-${CONFLUENT_VERSION}.jar=${CP_COMPONENT_SCRIPT_DIR}/disk-usage-agent.properties" + + echo "===> Configure jolokia agent" + export JOLOKIA_AGENT_PORT=${JOLOKIA_AGENT_PORT:-7777} + export JOLOKIA_AGENT_HOST=${JOLOKIA_AGENT_HOST:-"0.0.0.0"} + if [ -e "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config" ]; then + export JOLOKIA_EXTRA_ARGS=",$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jolokia.config")" + fi + + if [ -e "/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar" ]; then + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.6.2-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + else + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jolokia/jolokia-jvm-1.3.6-agent.jar=port=${JOLOKIA_AGENT_PORT},host=${JOLOKIA_AGENT_HOST}${JOLOKIA_EXTRA_ARGS}" + fi + + echo "===> Configure prometheus JMX agent" + export JMX_EXPORTER_AGENT_PORT=${JMX_EXPORTER_AGENT_PORT:-7778} + export JMX_EXPORTER_AGENT_HOST=${JMX_EXPORTER_AGENT_HOST:-"0.0.0.0"} + export EXTRA_ARGS="${EXTRA_ARGS} -javaagent:/opt/caas/lib/jmx_prometheus_javaagent-0.14.0.jar=${JMX_EXPORTER_AGENT_PORT}:/mnt/config/shared/jmx-exporter.yaml" + ## requires for now but have to remove in the future iterations + ## we need to remove javaagent from the "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" as Operator addes as part of + ## new stack deployment + echo "===> Configure JVM config..." + export EXTRA_ARGS="$(sed '/-javaagent*/d' "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | sed '/^$/d' | tr '\n' ' ') $EXTRA_ARGS" +else + echo "===> Configure JVM config..." + export EXTRA_ARGS="$(cat < "${CP_COMPONENT_SCRIPT_DIR}/jvm.config" | tr '\n' ' ') ${EXTRA_ARGS}" +fi + +# These ensure that the "if" sections for heap sizing and GC tuning in kafka launch script does not trigger. +export KAFKA_HEAP_OPTS=' ' +export KAFKA_JVM_PERFORMANCE_OPTS=' ' +echo "===> Launching ${CP_COMPONENT_NAME} ..." + +exec zookeeper-server-start "${CP_COMPONENT_SCRIPT_DIR}/zookeeper.properties" diff --git a/scripts/confluentinc/zookeeper/bin/run b/scripts/confluentinc/zookeeper/bin/run new file mode 100755 index 0000000000000000000000000000000000000000..7b1411fbd7120ac5072b2213135225f6b2cf6104 --- /dev/null +++ b/scripts/confluentinc/zookeeper/bin/run @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright 2020 Confluent Inc. +# +set -o nounset +set -o errexit + +export CP_COMPONENT_NAME=zookeeper +export CP_COMPONENT_SCRIPT_DIR=/opt/confluentinc/etc/${CP_COMPONENT_NAME} +export ZOOKEEPER_CONFIG_DIR=/mnt/config +export ZOOKEEPER_DATA_DIR=/mnt/data/data + +echo "===> User" +id + +export OPERATOR_SCRIPT_DIR=/mnt/config/${CP_COMPONENT_NAME} +echo "===> Load ${CP_COMPONENT_NAME} operator scripts from path ${OPERATOR_SCRIPT_DIR}/bin" + +"${OPERATOR_SCRIPT_DIR}"/bin/configure +"${OPERATOR_SCRIPT_DIR}"/bin/launch diff --git a/scripts/parser.py b/scripts/parser.py index 06e3d1c15564db9f738c16c16f43c44fadf364d3..0d5040260b5cedf257860fda221541ddd5d796b2 100644 --- a/scripts/parser.py +++ b/scripts/parser.py @@ -13,9 +13,14 @@ def create_file(file_name, data): ''' with open(file_name, 'w') as out_file: for key, value in data.items(): - if isinstance(value, int) or isinstance(value, float) : + if isinstance(value, int) or isinstance(value, float): value = str(value) - out_file.write(key+"="+value+"\n") + if isinstance(value, bool): + value = str(value).lower() + if len(value) == 0: + out_file.write(key + "\n") + else: + out_file.write(key+"="+value+"\n") def read_file(file_name): ''' diff --git a/scripts/startup.sh b/scripts/startup.sh index e0ad840fef818328b3b5971aea53c97ab7e232a5..72bdd28a7404808b4de3959668cf990a3c0defd3 100644 --- a/scripts/startup.sh +++ b/scripts/startup.sh @@ -1,15 +1,54 @@ #!/usr/bin/env sh set -ox errexit -if [[ -z ${POD_NAME} ]]; then +if [[ -z "${POD_NAME}" ]]; then echo "Environment variable POD_NAME not found..." env exit fi -pod_id=${POD_NAME} -id=${pod_id##*-}; -pod_name=${POD_NAME%-*} -cat /mnt/config/pod/${pod_name}/template.jsonnet | base64 -d > /opt/template.jsonnet -/usr/local/bin/jsonnet /opt/template.jsonnet --ext-str id=${id} -o /opt/${pod_name}.json -cat /opt/${pod_name}.json -/opt/parser.py ${pod_name} /opt/${pod_name}.json /mnt/config + +pod_name=${POD_NAME} +id=${pod_name##*-}; +component_name=${POD_NAME%-*} + +if [ -e "/mnt/config/pod/${component_name}/template.jsonnet" ]; then + cat < /mnt/config/pod/"${component_name}"/template.jsonnet | base64 -d > /opt/template.jsonnet + /usr/local/bin/jsonnet /opt/template.jsonnet --ext-str id="${id}" -o /opt/"${component_name}".json + cat /opt/"${component_name}".json + /opt/parser.py "${component_name}" "/opt/${component_name}.json" /mnt/config +elif [ -e "/mnt/config/init/template.jsonnet" ]; then + # new version of operator 2.0 + cat < /mnt/config/init/template.jsonnet | base64 -d > /mnt/config/template.jsonnet + cat < /mnt/config/init/data.json | base64 -d > /mnt/config/data.json + /opt/config.py "${pod_name}" /mnt/config/data.json /mnt/config/template.jsonnet /mnt/config/"${component_name}".json + cat /mnt/config/"${component_name}".json + /opt/parser.py "${component_name}" /mnt/config/"${component_name}".json /mnt/config + # write to both locations to support image overrides + mkdir -p /mnt/config/pod/"${pod_name}" + /opt/parser.py "${component_name}" /mnt/config/"${component_name}".json /mnt/config/pod/"${pod_name}" + # clean up excess output files + rm /mnt/config/template.jsonnet /mnt/config/data.json /mnt/config/"${component_name}".json +else + echo "template.jsonnet not found in the path" + exit 1 +fi + +# load scripts accordingly +if [ -n "${OPERATOR_CP_TYPE}" ]; then +echo "load script for CP type: ${OPERATOR_CP_TYPE}" + if [ "${OPERATOR_CP_TYPE}" = "zookeeper" ]; then + cp -r /opt/confluentinc/zookeeper/ /mnt/config/ + elif [ "${OPERATOR_CP_TYPE}" = "controlcenter" ]; then + cp -r /opt/confluentinc/controlcenter/ /mnt/config/ + elif [ "${OPERATOR_CP_TYPE}" = "kafka" ]; then + cp -r /opt/confluentinc/kafka/ /mnt/config/ + elif [ "${OPERATOR_CP_TYPE}" = "replicator" ]; then + cp -r /opt/confluentinc/replicator/ /mnt/config/ + elif [ "${OPERATOR_CP_TYPE}" = "connect" ]; then + cp -r /opt/confluentinc/connect/ /mnt/config/ + elif [ "${OPERATOR_CP_TYPE}" = "ksqldb" ]; then + cp -r /opt/confluentinc/ksqldb/ /mnt/config/ + elif [ "${OPERATOR_CP_TYPE}" = "schemaregistry" ]; then + cp -r /opt/confluentinc/schemaregistry/ /mnt/config/ + fi +fi