UNCLASSIFIED

Skip to content

GitLab

Commit 3d752484 authored by Austin Denton's avatar Austin Denton
Browse files

Update worker

parent 8f3b02ce
Pipeline #213925 passed with stages
in 33 minutes and 57 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 229 additions and 83 deletions
Dockerfile
View file @ 3d752484
...@@ -16,26 +16,38 @@ ENV BITNAMI_PKG_EXTRA_DIRS="/opt/bitnami/airflow/dags" \ ...@@ -16,26 +16,38 @@ ENV BITNAMI_PKG_EXTRA_DIRS="/opt/bitnami/airflow/dags" \
COPY --from=base ${BITNAMI_HOME} ${BITNAMI_HOME} COPY --from=base ${BITNAMI_HOME} ${BITNAMI_HOME}
COPY --from=base ${BITNAMI_DIR} ${BITNAMI_DIR} COPY --from=base ${BITNAMI_DIR} ${BITNAMI_DIR}
COPY --from=base \
/lib/x86_64-linux-gnu/libbz2.so.1.0 \
/usr/lib64/
COPY prebuildfs / COPY prebuildfs /
RUN dnf update -y --nodocs && \ RUN dnf update -y --nodocs && \
dnf install -y curl tar gzip ca-certificates libxml2 \ dnf install -y curl tar gzip ca-certificates libxml2 procps glibc-langpack-en && \
procps glibc-locale-source glibc-langpack-en && \
dnf clean all && \ dnf clean all && \
rm -rf /var/cache/dnf && \ rm -rf /var/cache/dnf && \
localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ mkdir -p /local/wheels && \
chmod g+rwX /opt/bitnami chmod g+rwX /opt/bitnami
COPY rootfs / COPY rootfs /
RUN /opt/bitnami/scripts/airflow/postunpack.sh COPY *.whl *.tar.gz /local/wheels/
COPY --from=base \ RUN /opt/bitnami/scripts/airflow-worker/postunpack.sh && \
/lib/x86_64-linux-gnu/libbz2.so.1.0 \ source /opt/bitnami/airflow/venv/bin/activate && \
/usr/lib64/ # Remove for security advisory GHSA-x9p2-fxq6-2m5f GHSA-4f9m-pxwh-68hg GHSA-388g-jwpg-x6j4
rm -rf ${BITNAMI_HOME}/airflow/venv/lib/python3.8/site-packages/swagger_ui_bundle/vendor/swagger-ui-2.2.10 && \
# Update to thrift-0.14 per CVE-2020-13949
tar xfz /local/wheels/thrift-0.14.1.tar.gz -C /local/wheels && \
pip install --no-index --no-deps /local/wheels/thrift-0.14.1/lib/py && \
rm -rf /local/wheels/thrift-0.14.1* && \
for f in $(ls -l /local/wheels | awk '{print $9}' |sed '/^$/d'); do pip install --no-index --no-deps /local/wheels/$f; done && \
find /opt/bitnami/airflow/venv/lib/python3.8/site-packages -name "*.pem" -o -name "*.key" | egrep ".*test.*/.*\.pem|.*test.*/.*\.key" | xargs rm -f && \
rm -rf /local/*
ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ ENV AIRFLOW_HOME="/opt/bitnami/airflow" \
BITNAMI_APP_NAME="airflow" \ BITNAMI_APP_NAME="airflow-worker" \
BITNAMI_IMAGE_VERSION="2.0.1-debian-10-r53" \ BITNAMI_IMAGE_VERSION="2.0.1-debian-10-r54" \
C_FORCE_ROOT="True" \
LANG="en_US.UTF-8" \ LANG="en_US.UTF-8" \
LANGUAGE="en_US:en" \ LANGUAGE="en_US:en" \
LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \ LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \
...@@ -45,8 +57,8 @@ ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ ...@@ -45,8 +57,8 @@ ENV AIRFLOW_HOME="/opt/bitnami/airflow" \
NSS_WRAPPER_PASSWD="/opt/bitnami/airflow/nss_passwd" \ NSS_WRAPPER_PASSWD="/opt/bitnami/airflow/nss_passwd" \
PATH="/opt/bitnami/common/bin:/opt/bitnami/python/bin:/opt/bitnami/postgresql/bin:/opt/bitnami/airflow/venv/bin:$PATH" PATH="/opt/bitnami/common/bin:/opt/bitnami/python/bin:/opt/bitnami/postgresql/bin:/opt/bitnami/airflow/venv/bin:$PATH"
EXPOSE 8080 EXPOSE 8793
USER 1001 USER 1001
ENTRYPOINT [ "/opt/bitnami/scripts/airflow/entrypoint.sh" ] ENTRYPOINT [ "/opt/bitnami/scripts/airflow-worker/entrypoint.sh" ]
CMD [ "/opt/bitnami/scripts/airflow/run.sh" ] CMD [ "/opt/bitnami/scripts/airflow-worker/run.sh" ]
hardening_manifest.yaml
View file @ 3d752484
...@@ -38,6 +38,36 @@ labels: ...@@ -38,6 +38,36 @@ labels:
resources: resources:
- tag: bitnami/airflow-worker:2.0.1-debian-10-r53 - tag: bitnami/airflow-worker:2.0.1-debian-10-r53
url: docker://docker.io/bitnami/airflow-worker@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae389 url: docker://docker.io/bitnami/airflow-worker@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae389
- filename: thrift-0.14.1.tar.gz
url: https://mirror.jframeworks.com/apache/thrift/0.14.1/thrift-0.14.1.tar.gz
validation:
type: sha256
value: 13da5e1cd9c8a3bb89778c0337cc57eb0c29b08f3090b41cf6ab78594b410ca5
- filename: pylint-2.7.2-py3-none-any.whl
url: https://files.pythonhosted.org/packages/b3/66/af8f80d4fa77dcd4cba9e56e136522838920a2eaf6794b784e1f377f84d9/pylint-2.7.2-py3-none-any.whl
validation:
type: sha256
value: d09b0b07ba06bcdff463958f53f23df25e740ecd81895f7d2699ec04bbd8dc3b
- filename: networkx-2.5.1-py3-none-any.whl
url: https://files.pythonhosted.org/packages/f3/b7/c7f488101c0bb5e4178f3cde416004280fd40262433496830de8a8c21613/networkx-2.5.1-py3-none-any.whl
validation:
type: sha256
value: 0635858ed7e989f4c574c2328380b452df892ae85084144c73d8cd819f0c4e06
- filename: decorator-4.4.2-py2.py3-none-any.whl
url: https://files.pythonhosted.org/packages/ed/1b/72a1821152d07cf1d8b6fce298aeb06a7eb90f4d6d41acec9861e7cc6df0/decorator-4.4.2-py2.py3-none-any.whl
validation:
type: sha256
value: 41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760
- filename: aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_x86_64.whl
url: https://files.pythonhosted.org/packages/a6/76/f18138b0ff84fcd939667a2efc2e1b49c871299f9091f84c06bb4c350c01/aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_x86_64.whl
validation:
type: sha256
value: 79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013
- filename: azure_storage_blob-12.8.0-py2.py3-none-any.whl
url: https://files.pythonhosted.org/packages/09/14/4ca417a9c92b0fb93516575dd7be9b058bf13d531dcc21239b5f8f216a69/azure_storage_blob-12.8.0-py2.py3-none-any.whl
validation:
type: sha256
value: 46999df6e2cde8773739f7c3bd1eb5846d4b7dc1ef6e2161f3b6d1d0f21726ba
# List of project maintainers # List of project maintainers
maintainers: maintainers:
......
prebuildfs/opt/bitnami/.bitnami_components.json
View file @ 3d752484
{ {
"airflow": { "airflow-worker": {
"arch": "amd64", "arch": "amd64",
"digest": "d837c8af9305cfcbed7dd0493336ba0e38d7a3aa211192a8f05d117a7b7734ab", "digest": "cf754bdc8e231d408275c1971d9a732576e346e0cd4af1ea426739bf3e62538d",
"distro": "debian-10", "distro": "debian-10",
"type": "NAMI", "type": "NAMI",
"version": "2.0.1-4" "version": "2.0.1-2"
}, },
"gosu": { "gosu": {
"arch": "amd64", "arch": "amd64",
......
prebuildfs/opt/bitnami/scripts/liblog.sh
View file @ 3d752484
...@@ -106,5 +106,7 @@ indent() { ...@@ -106,5 +106,7 @@ indent() {
for ((i = 0; i < num; i++)); do for ((i = 0; i < num; i++)); do
indent_unit="${indent_unit}${char}" indent_unit="${indent_unit}${char}"
done done
# shellcheck disable=SC2001
# Complex regex, see https://github.com/koalaman/shellcheck/wiki/SC2001#exceptions
echo "$string" | sed "s/^/${indent_unit}/" echo "$string" | sed "s/^/${indent_unit}/"
} }
prebuildfs/opt/bitnami/scripts/libversion.sh
View file @ 3d752484
...@@ -9,7 +9,7 @@ ...@@ -9,7 +9,7 @@
# Functions # Functions
######################## ########################
# Gets semantic version # Gets semantic version
# Arguments: # Arguments:
# $1 - version: string to extract major.minor.patch # $1 - version: string to extract major.minor.patch
# $2 - section: 1 to extract major, 2 to extract minor, 3 to extract patch # $2 - section: 1 to extract major, 2 to extract minor, 3 to extract patch
...@@ -38,7 +38,7 @@ get_sematic_version () { ...@@ -38,7 +38,7 @@ get_sematic_version () {
done done
local number_regex='^[0-9]+$' local number_regex='^[0-9]+$'
if [[ "$section" =~ $number_regex ]] && (( $section > 0 )) && (( $section <= 3 )); then if [[ "$section" =~ $number_regex ]] && (( section > 0 )) && (( section <= 3 )); then
echo "${version_sections[$section]}" echo "${version_sections[$section]}"
return return
else else
......
rootfs/opt/bitnami/scripts/airflow-env.sh → rootfs/opt/bitnami/scripts/airflow-worker-env.sh
View file @ 3d752484
#!/bin/bash #!/bin/bash
# #
# Environment configuration for airflow # Environment configuration for airflow-worker
# The values for all environment variables will be set in the below order of precedence # The values for all environment variables will be set in the below order of precedence
# 1. Custom environment variables defined below after Bitnami defaults # 1. Custom environment variables defined below after Bitnami defaults
...@@ -15,27 +15,17 @@ export BITNAMI_ROOT_DIR="/opt/bitnami" ...@@ -15,27 +15,17 @@ export BITNAMI_ROOT_DIR="/opt/bitnami"
export BITNAMI_VOLUME_DIR="/bitnami" export BITNAMI_VOLUME_DIR="/bitnami"
# Logging configuration # Logging configuration
export MODULE="${MODULE:-airflow}" export MODULE="${MODULE:-airflow-worker}"
export BITNAMI_DEBUG="${BITNAMI_DEBUG:-false}" export BITNAMI_DEBUG="${BITNAMI_DEBUG:-false}"
# By setting an environment variable matching *_FILE to a file path, the prefixed environment # By setting an environment variable matching *_FILE to a file path, the prefixed environment
# variable will be overridden with the value specified in that file # variable will be overridden with the value specified in that file
airflow_env_vars=( airflow_worker_env_vars=(
AIRFLOW_USERNAME
AIRFLOW_PASSWORD
AIRFLOW_FIRSTNAME
AIRFLOW_LASTNAME
AIRFLOW_EMAIL
AIRFLOW_EXECUTOR AIRFLOW_EXECUTOR
AIRFLOW_FERNET_KEY AIRFLOW_FERNET_KEY
AIRFLOW_WEBSERVER_HOST AIRFLOW_WEBSERVER_HOST
AIRFLOW_WEBSERVER_PORT_NUMBER AIRFLOW_WEBSERVER_PORT_NUMBER
AIRFLOW_LOAD_EXAMPLES
AIRFLOW_BASE_URL
AIRFLOW_HOSTNAME_CALLABLE AIRFLOW_HOSTNAME_CALLABLE
AIRFLOW_POOL_NAME
AIRFLOW_POOL_SIZE
AIRFLOW_POOL_DESC
AIRFLOW_DATABASE_HOST AIRFLOW_DATABASE_HOST
AIRFLOW_DATABASE_PORT_NUMBER AIRFLOW_DATABASE_PORT_NUMBER
AIRFLOW_DATABASE_NAME AIRFLOW_DATABASE_NAME
...@@ -47,18 +37,8 @@ airflow_env_vars=( ...@@ -47,18 +37,8 @@ airflow_env_vars=(
REDIS_PORT_NUMBER REDIS_PORT_NUMBER
REDIS_USER REDIS_USER
REDIS_PASSWORD REDIS_PASSWORD
AIRFLOW_LDAP_ENABLE
AIRFLOW_LDAP_URI
AIRFLOW_LDAP_SEARCH
AIRFLOW_LDAP_BIND_USER
AIRFLOW_LDAP_BIND_PASSWORD
AIRFLOW_LDAP_UID_FIELD
AIRFLOW_LDAP_USE_TLS
AIRFLOW_LDAP_ALLOW_SELF_SIGNED
AIRFLOW_LDAP_TLS_CA_CERTIFICATE
AIRFLOW_USER_REGISTRATION_ROLE
) )
for env_var in "${airflow_env_vars[@]}"; do for env_var in "${airflow_worker_env_vars[@]}"; do
file_env_var="${env_var}_FILE" file_env_var="${env_var}_FILE"
if [[ -n "${!file_env_var:-}" ]]; then if [[ -n "${!file_env_var:-}" ]]; then
if [[ -r "${!file_env_var:-}" ]]; then if [[ -r "${!file_env_var:-}" ]]; then
...@@ -69,7 +49,7 @@ for env_var in "${airflow_env_vars[@]}"; do ...@@ -69,7 +49,7 @@ for env_var in "${airflow_env_vars[@]}"; do
fi fi
fi fi
done done
unset airflow_env_vars unset airflow_worker_env_vars
# Airflow paths # Airflow paths
export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow" export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow"
...@@ -78,11 +58,10 @@ export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin" ...@@ -78,11 +58,10 @@ export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin"
export AIRFLOW_VOLUME_DIR="/bitnami/airflow" export AIRFLOW_VOLUME_DIR="/bitnami/airflow"
export AIRFLOW_DATA_DIR="${AIRFLOW_BASE_DIR}/data" export AIRFLOW_DATA_DIR="${AIRFLOW_BASE_DIR}/data"
export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs" export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs"
export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-webserver.log" export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-worker.log"
export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg" export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg"
export AIRFLOW_WEBSERVER_CONF_FILE="${AIRFLOW_BASE_DIR}/webserver_config.py"
export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp" export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp"
export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-webserver.pid" export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-worker.pid"
export AIRFLOW_DATA_TO_PERSIST="$AIRFLOW_DATA_DIR" export AIRFLOW_DATA_TO_PERSIST="$AIRFLOW_DATA_DIR"
export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags" export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags"
...@@ -90,24 +69,12 @@ export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags" ...@@ -90,24 +69,12 @@ export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags"
export AIRFLOW_DAEMON_USER="airflow" export AIRFLOW_DAEMON_USER="airflow"
export AIRFLOW_DAEMON_GROUP="airflow" export AIRFLOW_DAEMON_GROUP="airflow"
# User configuration
export AIRFLOW_USERNAME="${AIRFLOW_USERNAME:-user}"
export AIRFLOW_PASSWORD="${AIRFLOW_PASSWORD:-bitnami}"
export AIRFLOW_FIRSTNAME="${AIRFLOW_FIRSTNAME:-Firstname}"
export AIRFLOW_LASTNAME="${AIRFLOW_LASTNAME:-Lastname}"
export AIRFLOW_EMAIL="${AIRFLOW_EMAIL:-user@example.com}"
# Airflow configuration # Airflow configuration
export AIRFLOW_EXECUTOR="${AIRFLOW_EXECUTOR:-SequentialExecutor}" export AIRFLOW_EXECUTOR="${AIRFLOW_EXECUTOR:-SequentialExecutor}"
export AIRFLOW_FERNET_KEY="${AIRFLOW_FERNET_KEY:-}" export AIRFLOW_FERNET_KEY="${AIRFLOW_FERNET_KEY:-}"
export AIRFLOW_WEBSERVER_HOST="${AIRFLOW_WEBSERVER_HOST:-127.0.0.1}" export AIRFLOW_WEBSERVER_HOST="${AIRFLOW_WEBSERVER_HOST:-127.0.0.1}"
export AIRFLOW_WEBSERVER_PORT_NUMBER="${AIRFLOW_WEBSERVER_PORT_NUMBER:-8080}" export AIRFLOW_WEBSERVER_PORT_NUMBER="${AIRFLOW_WEBSERVER_PORT_NUMBER:-8080}"
export AIRFLOW_LOAD_EXAMPLES="${AIRFLOW_LOAD_EXAMPLES:-yes}"
export AIRFLOW_BASE_URL="${AIRFLOW_BASE_URL:-}"
export AIRFLOW_HOSTNAME_CALLABLE="${AIRFLOW_HOSTNAME_CALLABLE:-}" export AIRFLOW_HOSTNAME_CALLABLE="${AIRFLOW_HOSTNAME_CALLABLE:-}"
export AIRFLOW_POOL_NAME="${AIRFLOW_POOL_NAME:-}"
export AIRFLOW_POOL_SIZE="${AIRFLOW_POOL_SIZE:-}"
export AIRFLOW_POOL_DESC="${AIRFLOW_POOL_DESC:-}"
# Airflow database configuration # Airflow database configuration
export AIRFLOW_DATABASE_HOST="${AIRFLOW_DATABASE_HOST:-postgresql}" export AIRFLOW_DATABASE_HOST="${AIRFLOW_DATABASE_HOST:-postgresql}"
...@@ -122,16 +89,4 @@ export REDIS_PORT_NUMBER="${REDIS_PORT_NUMBER:-6379}" ...@@ -122,16 +89,4 @@ export REDIS_PORT_NUMBER="${REDIS_PORT_NUMBER:-6379}"
export REDIS_USER="${REDIS_USER:-}" export REDIS_USER="${REDIS_USER:-}"
export REDIS_PASSWORD="${REDIS_PASSWORD:-}" export REDIS_PASSWORD="${REDIS_PASSWORD:-}"
# Airflow LDAP configuration
export AIRFLOW_LDAP_ENABLE="${AIRFLOW_LDAP_ENABLE:-no}"
export AIRFLOW_LDAP_URI="${AIRFLOW_LDAP_URI:-}"
export AIRFLOW_LDAP_SEARCH="${AIRFLOW_LDAP_SEARCH:-}"
export AIRFLOW_LDAP_BIND_USER="${AIRFLOW_LDAP_BIND_USER:-}"
export AIRFLOW_LDAP_BIND_PASSWORD="${AIRFLOW_LDAP_BIND_PASSWORD:-}"
export AIRFLOW_LDAP_UID_FIELD="${AIRFLOW_LDAP_UID_FIELD:-}"
export AIRFLOW_LDAP_USE_TLS="${AIRFLOW_LDAP_USE_TLS:-False}"
export AIRFLOW_LDAP_ALLOW_SELF_SIGNED="${AIRFLOW_LDAP_ALLOW_SELF_SIGNED:-True}"
export AIRFLOW_LDAP_TLS_CA_CERTIFICATE="${AIRFLOW_LDAP_TLS_CA_CERTIFICATE:-}"
export AIRFLOW_USER_REGISTRATION_ROLE="${AIRFLOW_USER_REGISTRATION_ROLE:-Public}"
# Custom environment variables may be defined below # Custom environment variables may be defined below
rootfs/opt/bitnami/scripts/airflow/entrypoint.sh → rootfs/opt/bitnami/scripts/airflow-worker/entrypoint.sh
View file @ 3d752484
...@@ -8,17 +8,17 @@ set -o pipefail ...@@ -8,17 +8,17 @@ set -o pipefail
# set -o xtrace # Uncomment this line for debugging purpose # set -o xtrace # Uncomment this line for debugging purpose
# Load Airflow environment variables # Load Airflow environment variables
. /opt/bitnami/scripts/airflow-env.sh . /opt/bitnami/scripts/airflow-worker-env.sh
# Load libraries # Load libraries
. /opt/bitnami/scripts/libbitnami.sh . /opt/bitnami/scripts/libbitnami.sh
. /opt/bitnami/scripts/libairflow.sh . /opt/bitnami/scripts/libairflowworker.sh
print_welcome_page print_welcome_page
if [[ "$*" = *"/opt/bitnami/scripts/airflow/run.sh"* || "$*" = *"/run.sh"* ]]; then if [[ "$*" = *"/opt/bitnami/scripts/airflow-worker/run.sh"* || "$*" = *"/run.sh"* ]]; then
info "** Starting Airflow setup **" info "** Starting Airflow setup **"
/opt/bitnami/scripts/airflow/setup.sh /opt/bitnami/scripts/airflow-worker/setup.sh
info "** Airflow setup finished! **" info "** Airflow setup finished! **"
fi fi
......
rootfs/opt/bitnami/scripts/airflow/postunpack.sh → rootfs/opt/bitnami/scripts/airflow-worker/postunpack.sh
View file @ 3d752484
...@@ -8,10 +8,10 @@ set -o pipefail ...@@ -8,10 +8,10 @@ set -o pipefail
# set -o xtrace # Uncomment this line for debugging purpose # set -o xtrace # Uncomment this line for debugging purpose
# Load Airflow environment variables # Load Airflow environment variables
. /opt/bitnami/scripts/airflow-env.sh . /opt/bitnami/scripts/airflow-worker-env.sh
# Load libraries # Load libraries
. /opt/bitnami/scripts/libairflow.sh . /opt/bitnami/scripts/libairflowworker.sh
. /opt/bitnami/scripts/libfs.sh . /opt/bitnami/scripts/libfs.sh
. /opt/bitnami/scripts/libos.sh . /opt/bitnami/scripts/libos.sh
......
rootfs/opt/bitnami/scripts/airflow/run.sh → rootfs/opt/bitnami/scripts/airflow-worker/run.sh
View file @ 3d752484
...@@ -8,17 +8,17 @@ set -o pipefail ...@@ -8,17 +8,17 @@ set -o pipefail
# set -o xtrace # Uncomment this line for debugging purposes # set -o xtrace # Uncomment this line for debugging purposes
# Load Airflow environment variables # Load Airflow environment variables
. /opt/bitnami/scripts/airflow-env.sh . /opt/bitnami/scripts/airflow-worker-env.sh
# Load libraries # Load libraries
. /opt/bitnami/scripts/libos.sh . /opt/bitnami/scripts/libos.sh
. /opt/bitnami/scripts/libairflow.sh . /opt/bitnami/scripts/libairflowworker.sh
args=("--pid" "$AIRFLOW_PID_FILE" "$@") args=("--pid" "$AIRFLOW_PID_FILE" "$@")
info "** Starting Airflow **" info "** Starting Airflow **"
if am_i_root; then if am_i_root; then
exec gosu "$AIRFLOW_DAEMON_USER" "${AIRFLOW_BIN_DIR}/airflow" "webserver" "${args[@]}" exec gosu "$AIRFLOW_DAEMON_USER" "${AIRFLOW_BIN_DIR}/airflow" "celery" "worker" "${args[@]}"
else else
exec "${AIRFLOW_BIN_DIR}/airflow" "webserver" "${args[@]}" exec "${AIRFLOW_BIN_DIR}/airflow" "celery" "worker" "${args[@]}"
fi fi
rootfs/opt/bitnami/scripts/airflow/setup.sh → rootfs/opt/bitnami/scripts/airflow-worker/setup.sh
View file @ 3d752484
...@@ -8,16 +8,16 @@ set -o pipefail ...@@ -8,16 +8,16 @@ set -o pipefail
# set -o xtrace # Uncomment this line for debugging purposes # set -o xtrace # Uncomment this line for debugging purposes
# Load Airflow environment variables # Load Airflow environment variables
. /opt/bitnami/scripts/airflow-env.sh . /opt/bitnami/scripts/airflow-worker-env.sh
# Load libraries # Load libraries
. /opt/bitnami/scripts/libos.sh . /opt/bitnami/scripts/libos.sh
. /opt/bitnami/scripts/libfs.sh . /opt/bitnami/scripts/libfs.sh
. /opt/bitnami/scripts/libairflow.sh . /opt/bitnami/scripts/libairflowworker.sh
# Ensure Airflow environment variables settings are valid # Ensure Airflow environment variables settings are valid
airflow_validate airflow_worker_validate
# Ensure Airflow daemon user exists when running as root # Ensure Airflow daemon user exists when running as root
am_i_root && ensure_user_exists "$AIRFLOW_DAEMON_USER" --group "$AIRFLOW_DAEMON_GROUP" am_i_root && ensure_user_exists "$AIRFLOW_DAEMON_USER" --group "$AIRFLOW_DAEMON_GROUP"
# Ensure Airflow is initialized # Ensure Airflow is initialized
airflow_initialize airflow_worker_initialize
rootfs/opt/bitnami/scripts/libairflowworker.sh 0 → 100644
View file @ 3d752484
#!/bin/bash
# Bitnami Airflow library
# shellcheck disable=SC1091
# shellcheck disable=SC2153
# Load Generic Libraries
. /opt/bitnami/scripts/libfile.sh
. /opt/bitnami/scripts/liblog.sh
. /opt/bitnami/scripts/libnet.sh
. /opt/bitnami/scripts/libos.sh
. /opt/bitnami/scripts/libservice.sh
. /opt/bitnami/scripts/libvalidations.sh
. /opt/bitnami/scripts/libpersistence.sh
# Load airflow library
. /opt/bitnami/scripts/libairflow.sh
########################
# Validate Airflow Scheduler inputs
# Globals:
# AIRFLOW_*
# Arguments:
# None
# Returns:
# None
#########################
airflow_worker_validate() {
# Check postgresql host
[[ -z "$AIRFLOW_WEBSERVER_HOST" ]] && print_validation_error "Missing AIRFLOW_WEBSERVER_HOST"
[[ -z "$AIRFLOW_WEBSERVER_PORT_NUMBER" ]] && print_validation_error "Missing AIRFLOW_WEBSERVER_PORT_NUMBER"
# Check postgresql host
[[ -z "$AIRFLOW_DATABASE_HOST" ]] && print_validation_error "Missing AIRFLOW_DATABASE_HOST"
# Avoid fail because of the above check
true
}
########################
# Ensure Airflow Scheduler is initialized
# Globals:
# AIRFLOW_*
# Arguments:
# None
# Returns:
# None
#########################
airflow_worker_initialize() {
# Change permissions if running as root
for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DATA_DIR"; do
ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
done
# The configuration file is not persisted. If it is not provided, generate it based on env vars
if [[ ! -f "$AIRFLOW_CONF_FILE" ]]; then
info "No injected configuration file found. Creating default config file"
airflow_worker_generate_config
else
info "Configuration file found, loading configuration"
fi
# Check if Airflow has already been initialized and persisted in a previous run
local -r app_name="airflow"
if ! is_app_initialized "$app_name"; then
airflow_wait_for_postgresql "$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER"
info "Persisting Airflow installation"
persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
else
# Check database connection
airflow_wait_for_postgresql "$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER"
# Restore persisted data
info "Restoring persisted Airflow installation"
restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST"
# Change the permissions after restoring the persisted data in case we are root
for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do
ensure_dir_exists "$dir"
am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir"
done
true # Avoid return false when I am not root
fi
# Wait for airflow webserver to be available
info "Waiting for Airflow Webserser to be up"
airflow_worker_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER"
[[ "$AIRFLOW_EXECUTOR" == "CeleryExecutor" ]] && wait-for-port --host "$REDIS_HOST" "$REDIS_PORT_NUMBER"
# Avoid to fail when the executor is not celery
true
}
########################
# Generate Airflow Scheduler conf file
# Globals:
# AIRFLOW_*
# Arguments:
# None
# Returns:
# None
#########################
airflow_worker_generate_config() {
# Generate Airflow default files
airflow_execute_command "version" "version"
# Configure Airflow Hostname
[[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE"
# Configure Airflow database
airflow_configure_database
# Configure the Webserver port
airflow_conf_set "webserver" "web_server_port" "$AIRFLOW_WEBSERVER_PORT_NUMBER"
# Setup fernet key
[[ -n "$AIRFLOW_FERNET_KEY" ]] && airflow_conf_set "core" "fernet_key" "$AIRFLOW_FERNET_KEY"
# Configure Airflow executor
airflow_conf_set "core" "executor" "$AIRFLOW_EXECUTOR"
[[ "$AIRFLOW_EXECUTOR" == "CeleryExecutor" ]] && airflow_configure_celery_executor
true # Avoid the function to fail due to the check above
}
########################
# Wait Ariflow webserver
# Globals:
# AIRFLOW_*
# Arguments:
# None
# Returns:
# None
#########################
airflow_worker_wait_for_webserver() {
local -r webserver_host="${1:?missing database host}"
local -r webserver_port="${2:?missing database port}"
check_webserver_connection() {
wait-for-port --host "$webserver_host" "$webserver_port"
}
if ! retry_while "check_webserver_connection"; then
error "Could not connect to the Airflow webserver"
return 1
fi
}
rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh
View file @ 3d752484
...@@ -33,6 +33,7 @@ if [[ "$WITH_ALL_LOCALES" =~ ^(yes|true|1)$ ]]; then ...@@ -33,6 +33,7 @@ if [[ "$WITH_ALL_LOCALES" =~ ^(yes|true|1)$ ]]; then
echo "Enabling all locales" echo "Enabling all locales"
cp "$SUPPORTED_LOCALES_FILE" "$LOCALES_FILE" cp "$SUPPORTED_LOCALES_FILE" "$LOCALES_FILE"
else else
# shellcheck disable=SC2001
LOCALES_TO_ADD="$(sed 's/[,;]\s*/\n/g' <<< "$EXTRA_LOCALES")" LOCALES_TO_ADD="$(sed 's/[,;]\s*/\n/g' <<< "$EXTRA_LOCALES")"
while [[ -n "$LOCALES_TO_ADD" ]] && read -r locale; do while [[ -n "$LOCALES_TO_ADD" ]] && read -r locale; do
echo "Enabling locale ${locale}" echo "Enabling locale ${locale}"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

UNCLASSIFIED