From 8f0fa320a632bcf8b50c3b2f92a8ebef1388f543 Mon Sep 17 00:00:00 2001 From: Austin Denton Date: Wed, 7 Apr 2021 10:19:38 -0600 Subject: [PATCH 1/5] Update to R53 to not use nami tool --- Dockerfile | 52 ++ LICENSE | 13 + README.md | 482 +++++++++++++++++- hardening_manifest.yaml | 47 ++ .../opt/bitnami/.bitnami_components.json | 44 ++ prebuildfs/opt/bitnami/licenses/licenses.txt | 3 + prebuildfs/opt/bitnami/scripts/libbitnami.sh | 51 ++ .../opt/bitnami/scripts/libcomponent.sh | 65 +++ prebuildfs/opt/bitnami/scripts/libfile.sh | 80 +++ prebuildfs/opt/bitnami/scripts/libfs.sh | 183 +++++++ prebuildfs/opt/bitnami/scripts/libhook.sh | 16 + prebuildfs/opt/bitnami/scripts/liblog.sh | 110 ++++ prebuildfs/opt/bitnami/scripts/libnet.sh | 142 ++++++ prebuildfs/opt/bitnami/scripts/libos.sh | 350 +++++++++++++ .../opt/bitnami/scripts/libpersistence.sh | 122 +++++ prebuildfs/opt/bitnami/scripts/libservice.sh | 235 +++++++++ .../opt/bitnami/scripts/libvalidations.sh | 248 +++++++++ prebuildfs/opt/bitnami/scripts/libversion.sh | 49 ++ .../opt/bitnami/scripts/libwebserver.sh | 421 +++++++++++++++ prebuildfs/usr/sbin/install_packages | 24 + rootfs/opt/bitnami/scripts/airflow-env.sh | 137 +++++ .../opt/bitnami/scripts/airflow/entrypoint.sh | 26 + .../opt/bitnami/scripts/airflow/postunpack.sh | 28 + rootfs/opt/bitnami/scripts/airflow/run.sh | 24 + rootfs/opt/bitnami/scripts/airflow/setup.sh | 23 + rootfs/opt/bitnami/scripts/libairflow.sh | 432 ++++++++++++++++ .../scripts/locales/add-extra-locales.sh | 43 ++ 27 files changed, 3448 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 hardening_manifest.yaml create mode 100644 prebuildfs/opt/bitnami/.bitnami_components.json create mode 100644 prebuildfs/opt/bitnami/licenses/licenses.txt create mode 100644 prebuildfs/opt/bitnami/scripts/libbitnami.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libcomponent.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libfile.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libfs.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libhook.sh create mode 100644 prebuildfs/opt/bitnami/scripts/liblog.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libnet.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libos.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libpersistence.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libservice.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libvalidations.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libversion.sh create mode 100644 prebuildfs/opt/bitnami/scripts/libwebserver.sh create mode 100755 prebuildfs/usr/sbin/install_packages create mode 100644 rootfs/opt/bitnami/scripts/airflow-env.sh create mode 100755 rootfs/opt/bitnami/scripts/airflow/entrypoint.sh create mode 100755 rootfs/opt/bitnami/scripts/airflow/postunpack.sh create mode 100755 rootfs/opt/bitnami/scripts/airflow/run.sh create mode 100755 rootfs/opt/bitnami/scripts/airflow/setup.sh create mode 100644 rootfs/opt/bitnami/scripts/libairflow.sh create mode 100755 rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d4d9b76 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 +ARG BASE_TAG=8.3 + +FROM bitnami/airflow-worker:2.0.1-debian-10-r53 as base +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} +ARG BITNAMI_HOME=/opt/bitnami +ARG BITNAMI_DIR=/bitnami + + +ENV BITNAMI_PKG_EXTRA_DIRS="/opt/bitnami/airflow/dags" \ + HOME="/" \ + OS_ARCH="amd64" \ + OS_FLAVOUR="debian-10" \ + OS_NAME="linux" + +COPY --from=base ${BITNAMI_HOME} ${BITNAMI_HOME} +COPY --from=base ${BITNAMI_DIR} ${BITNAMI_DIR} +COPY prebuildfs / + +RUN dnf update -y --nodocs && \ + dnf install -y curl tar gzip ca-certificates libxml2 \ + procps glibc-locale-source glibc-langpack-en && \ + dnf clean all && \ + rm -rf /var/cache/dnf && \ + localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ + chmod g+rwX /opt/bitnami + +COPY rootfs / +RUN /opt/bitnami/scripts/airflow/postunpack.sh + +COPY --from=base \ + /lib/x86_64-linux-gnu/libbz2.so.1.0 \ + /usr/lib64/ + +ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ + BITNAMI_APP_NAME="airflow" \ + BITNAMI_IMAGE_VERSION="2.0.1-debian-10-r53" \ + LANG="en_US.UTF-8" \ + LANGUAGE="en_US:en" \ + LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \ + LIBNSS_WRAPPER_PATH="/opt/bitnami/common/lib/libnss_wrapper.so" \ + LNAME="airflow" \ + NSS_WRAPPER_GROUP="/opt/bitnami/airflow/nss_group" \ + NSS_WRAPPER_PASSWD="/opt/bitnami/airflow/nss_passwd" \ + PATH="/opt/bitnami/common/bin:/opt/bitnami/python/bin:/opt/bitnami/postgresql/bin:/opt/bitnami/airflow/venv/bin:$PATH" + +EXPOSE 8080 + +USER 1001 +ENTRYPOINT [ "/opt/bitnami/scripts/airflow/entrypoint.sh" ] +CMD [ "/opt/bitnami/scripts/airflow/run.sh" ] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c850fde --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2015-2021 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 5dc6fa6..ac2bf29 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,481 @@ -# +# What is Apache Airflow? -Project template for all Iron Bank container repositories. \ No newline at end of file +> Airflow is a platform to programmatically author, schedule and monitor workflows. + +https://airflow.apache.org/ + +# TL;DR + +## Docker Compose + +```console +$ curl -LO https://raw.githubusercontent.com/bitnami/bitnami-docker-airflow/master/docker-compose.yml +$ docker-compose up +``` + +You can find the default credentials and available configuration options in the [Environment Variables](#environment-variables) section. + +# Why use Bitnami Images? + +* Bitnami closely tracks upstream source changes and promptly publishes new versions of this image using our automated systems. +* With Bitnami images the latest bug fixes and features are available as soon as possible. +* Bitnami containers, virtual machines and cloud images use the same components and configuration approach - making it easy to switch between formats based on your project needs. +* All our images are based on [minideb](https://github.com/bitnami/minideb) a minimalist Debian based container image which gives you a small base container image and the familiarity of a leading Linux distribution. +* All Bitnami images available in Docker Hub are signed with [Docker Content Trust (DCT)](https://docs.docker.com/engine/security/trust/content_trust/). You can use `DOCKER_CONTENT_TRUST=1` to verify the integrity of the images. +* Bitnami container images are released daily with the latest distribution packages available. + + +> This [CVE scan report](https://quay.io/repository/bitnami/airflow?tab=tags) contains a security report with all open CVEs. To get the list of actionable security issues, find the "latest" tag, click the vulnerability report link under the corresponding "Security scan" field and then select the "Only show fixable" filter on the next page. + + +# Supported tags and respective `Dockerfile` links + +Learn more about the Bitnami tagging policy and the difference between rolling tags and immutable tags [in our documentation page](https://docs.bitnami.com/tutorials/understand-rolling-tags-containers/). + + +* [`2`, `2-debian-10`, `2.0.1`, `2.0.1-debian-10-r51`, `latest` (2/debian-10/Dockerfile)](https://github.com/bitnami/bitnami-docker-airflow/blob/2.0.1-debian-10-r51/2/debian-10/Dockerfile) +* [`1`, `1-debian-10`, `1.10.15`, `1.10.15-debian-10-r17` (1/debian-10/Dockerfile)](https://github.com/bitnami/bitnami-docker-airflow/blob/1.10.15-debian-10-r17/1/debian-10/Dockerfile) + +Subscribe to project updates by watching the [bitnami/airflow GitHub repo](https://github.com/bitnami/bitnami-docker-airflow). + +# Prerequisites + +To run this application you need [Docker Engine](https://www.docker.com/products/docker-engine) >= `1.10.0`. [Docker Compose](https://www.docker.com/products/docker-compose) is recommended with a version `1.6.0` or later. + +# How to use this image + +Airflow requires access to a PostgreSQL database to store information. We will use our very own [PostgreSQL image](https://www.github.com/bitnami/bitnami-docker-postgresql) for the database requirements. Additionally, if you pretend to use the `CeleryExecutor`, you will also need an [Airflow Scheduler](https://www.github.com/bitnami/bitnami-docker-airflow-scheduler), one or more [Airflow Workers](https://www.github.com/bitnami/bitnami-docker-airflow-worker) and a [Redis(TM) server](https://www.github.com/bitnami/bitnami-docker-redis). + +## Using Docker Compose + +The main folder of this repository contains a functional [`docker-compose.yml`](https://github.com/bitnami/bitnami-docker-airflow/blob/master/docker-compose.yml) file. Run the application using it as shown below: + +```console +$ curl -sSL https://raw.githubusercontent.com/bitnami/bitnami-docker-airflow/master/docker-compose.yml > docker-compose.yml +$ docker-compose up -d +``` + +## Using the Docker Command Line + +If you want to run the application manually instead of using `docker-compose`, these are the basic steps you need to run: + +1. Create a network + + ```console + $ docker network create airflow-tier + ``` + +2. Create a volume for PostgreSQL persistence and create a PostgreSQL container + + ```console + $ docker volume create --name postgresql_data + $ docker run -d --name postgresql \ + -e POSTGRESQL_USERNAME=bn_airflow \ + -e POSTGRESQL_PASSWORD=bitnami1 \ + -e POSTGRESQL_DATABASE=bitnami_airflow \ + --net airflow-tier \ + --volume postgresql_data:/bitnami/postgresql \ + bitnami/postgresql:latest + ``` + +3. Create a volume for Redis(TM) persistence and create a Redis(TM) container + + ```console + $ docker volume create --name redis_data + $ docker run -d --name redis \ + -e ALLOW_EMPTY_PASSWORD=yes \ + --net airflow-tier \ + --volume redis_data:/bitnami \ + bitnami/redis:latest + ``` + +4. Create volumes for Airflow persistence and launch the container + + ```console + $ docker volume create --name airflow_data + $ docker run -d --name airflow -p 8080:8080 \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + -e AIRFLOW_LOAD_EXAMPLES=yes \ + -e AIRFLOW_PASSWORD=bitnami123 \ + -e AIRFLOW_USERNAME=user \ + -e AIRFLOW_EMAIL=user@example.com \ + --net airflow-tier \ + --volume airflow_data:/bitnami \ + bitnami/airflow:latest + ``` + +5. Create volumes for Airflow Scheduler persistence and launch the container + + ```console + $ docker volume create --name airflow_scheduler_data + $ docker run -d --name airflow-scheduler \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + -e AIRFLOW_LOAD_EXAMPLES=yes \ + --net airflow-tier \ + --volume airflow_scheduler_data:/bitnami \ + bitnami/airflow-scheduler:latest + ``` + +6. Create volumes for Airflow Worker persistence and launch the container + + ```console + $ docker volume create --name airflow_worker_data + $ docker run -d --name airflow-worker \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + --net airflow-tier \ + --volume airflow_worker_data:/bitnami \ + bitnami/airflow-worker:latest + ``` + +Access your application at http://your-ip:8080 + +## Persisting your application + +If you remove the container all your data and configurations will be lost, and the next time you run the image the database will be reinitialized. To avoid this loss of data, you should mount a volume that will persist even after the container is removed. + +For persistence you should mount a volume at the `/bitnami` path. Additionally you should mount volumes for persistence of [PostgreSQL data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) and [Redis(TM) data](https://github.com/bitnami/bitnami-docker-mariadb#persisting-your-database) + +The above examples define docker volumes namely `postgresql_data`, `redis_data`, `airflow_data`, `airflow_scheduler_data` and `airflow_worker_data`. The Airflow application state will persist as long as these volumes are not removed. + +To avoid inadvertent removal of these volumes you can [mount host directories as data volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). Alternatively you can make use of volume plugins to host the volume data. + +### Mount host directories as data volumes with Docker Compose + +The following `docker-compose.yml` template demonstrates the use of host directories as data volumes. + +```yaml +version: '2' +services: + postgresql: + image: 'bitnami/postgresql:latest' + environment: + - POSTGRESQL_DATABASE=bitnami_airflow + - POSTGRESQL_USERNAME=bn_airflow + - POSTGRESQL_PASSWORD=bitnami1 + volumes: + - /path/to/airflow-persistence:/bitnami/postgresql + redis: + image: 'bitnami/redis:latest' + environment: + - ALLOW_EMPTY_PASSWORD=yes + volumes: + - /path/to/airflow-persistence:/bitnami + airflow-worker: + image: bitnami/airflow-worker:latest + environment: + - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW_EXECUTOR=CeleryExecutor + - AIRFLOW_DATABASE_NAME=bitnami_airflow + - AIRFLOW_DATABASE_USERNAME=bn_airflow + - AIRFLOW_DATABASE_PASSWORD=bitnami1 + - AIRFLOW_LOAD_EXAMPLES=yes + volumes: + - /path/to/airflow-persistence:/bitnami + airflow-scheduler: + image: bitnami/airflow-scheduler:latest + environment: + - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW_EXECUTOR=CeleryExecutor + - AIRFLOW_DATABASE_NAME=bitnami_airflow + - AIRFLOW_DATABASE_USERNAME=bn_airflow + - AIRFLOW_DATABASE_PASSWORD=bitnami1 + - AIRFLOW_LOAD_EXAMPLES=yes + volumes: + - /path/to/airflow-persistence:/bitnami + airflow: + image: bitnami/airflow:latest + environment: + - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW_EXECUTOR=CeleryExecutor + - AIRFLOW_DATABASE_NAME=bitnami_airflow + - AIRFLOW_DATABASE_USERNAME=bn_airflow + - AIRFLOW_DATABASE_PASSWORD=bitnami1 + - AIRFLOW_PASSWORD=bitnami123 + - AIRFLOW_USERNAME=user + - AIRFLOW_EMAIL=user@example.com + ports: + - '8080:8080' + volumes: + - /path/to/airflow-persistence:/bitnami +``` + +### Mount host directories as data volumes using the Docker command line + +1. Create a network (if it does not exist) + + ```console + $ docker network create airflow-tier + ``` + +2. Create the PostgreSQL container with host volumes + + ```console + $ docker run -d --name postgresql \ + -e POSTGRESQL_USERNAME=bn_airflow \ + -e POSTGRESQL_PASSWORD=bitnami1 \ + -e POSTGRESQL_DATABASE=bitnami_airflow \ + --net airflow-tier \ + --volume /path/to/postgresql-persistence:/bitnami \ + bitnami/postgresql:latest + ``` + +3. Create the Redis(TM) container with host volumes + + ```console + $ docker run -d --name redis \ + -e ALLOW_EMPTY_PASSWORD=yes \ + --net airflow-tier \ + --volume /path/to/redis-persistence:/bitnami \ + bitnami/redis:latest + ``` + +4. Create the Airflow container with host volumes + + ```console + $ docker run -d --name airflow -p 8080:8080 \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + -e AIRFLOW_LOAD_EXAMPLES=yes \ + -e AIRFLOW_PASSWORD=bitnami123 \ + -e AIRFLOW_USERNAME=user \ + -e AIRFLOW_EMAIL=user@example.com \ + --net airflow-tier \ + --volume /path/to/airflow-persistence:/bitnami \ + bitnami/airflow:latest + ``` + +5. Create the Airflow Scheduler container with host volumes + + ```console + $ docker run -d --name airflow-scheduler \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + -e AIRFLOW_LOAD_EXAMPLES=yes \ + --net airflow-tier \ + --volume /path/to/airflow-scheduler-persistence:/bitnami \ + bitnami/airflow-scheduler:latest + ``` + +6. Create the Airflow Worker container with host volumes + + ```console + $ docker run -d --name airflow-worker \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + --net airflow-tier \ + --volume /path/to/airflow-worker-persistence:/bitnami \ + bitnami/airflow-worker:latest + ``` + +# Configuration + +## Load DAG files + +Custom DAG files can be mounted to `/opt/bitnami/airflow/dags`. + +## Installing additional python modules + +This container supports the installation of additional python modules at start-up time. In order to do that, you can mount a `requirements.txt` file with your specific needs under the path `/bitnami/python/requirements.txt`. + +## Environment variables + +The Airflow instance can be customized by specifying environment variables on the first run. The following environment values are provided to customize Airflow: + +##### User configuration + +- `AIRFLOW_USERNAME`: Airflow application username. Default: **user** +- `AIRFLOW_PASSWORD`: Airflow application password. Default: **bitnami** +- `AIRFLOW_EMAIL`: Airflow application email. Default: **user@example.com** + +##### Airflow configuration + +- `AIRFLOW_EXECUTOR`: Airflow executor. Default: **SequentialExecutor** +- `AIRFLOW_FERNET_KEY`: Airflow Fernet key. No defaults. +- `AIRFLOW_WEBSERVER_HOST`: Airflow webserver host. Default: **127.0.0.1** +- `AIRFLOW_WEBSERVER_PORT_NUMBER`: Airflow webserver port. Default: **8080** +- `AIRFLOW_LOAD_EXAMPLES`: To load example tasks into the application. Default: **yes** +- `AIRFLOW_BASE_URL`: Airflow webserver base URL. No defaults. +- `AIRFLOW_HOSTNAME_CALLABLE`: Method to obtain the hostname. No defaults. +- `AIRFLOW_POOL_NAME`: Pool name. No defaults. +- `AIRFLOW_POOL_SIZE`: Pool size, required with `AIRFLOW_POOL_NAME`. No defaults. +- `AIRFLOW_POOL_DESC`: Pool description, required with `AIRFLOW_POOL_NAME`. No defaults. + +##### Use an existing database + +- `AIRFLOW_DATABASE_HOST`: Hostname for PostgreSQL server. Default: **postgresql** +- `AIRFLOW_DATABASE_PORT_NUMBER`: Port used by PostgreSQL server. Default: **5432** +- `AIRFLOW_DATABASE_NAME`: Database name that Airflow will use to connect with the database. Default: **bitnami_airflow** +- `AIRFLOW_DATABASE_USERNAME`: Database user that Airflow will use to connect with the database. Default: **bn_airflow** +- `AIRFLOW_DATABASE_PASSWORD`: Database password that Airflow will use to connect with the database. No defaults. +- `AIRFLOW_DATABASE_USE_SSL`: Set to yes if the database is using SSL. Default: **no** +- `AIRFLOW_REDIS_USE_SSL`: Set to yes if Redis(TM) uses SSL. Default: **no** +- `REDIS_HOST`: Hostname for Redis(TM) server. Default: **redis** +- `REDIS_PORT_NUMBER`: Port used by Redis(TM) server. Default: **6379** +- `REDIS_USER`: User that Airflow will use to connect with Redis(TM). No defaults. +- `REDIS_PASSWORD`: Password that Airflow will use to connect with Redis(TM). No defaults. + +##### Airflow LDAP authentication + +- `AIRFLOW_LDAP_ENABLE`: Enable LDAP authentication. Default: **no** +- `AIRFLOW_LDAP_URI`: LDAP server URI. No defaults. +- `AIRFLOW_LDAP_SEARCH`: LDAP search base. No defaults. +- `AIRFLOW_LDAP_BIND_USER`: LDAP user name. No defaults. +- `AIRFLOW_LDAP_BIND_PASSWORD`: LDAP user password. No defaults. +- `AIRFLOW_LDAP_UID_FIELD`: LDAP field used for uid. Default: **uid**. +- `AIRFLOW_LDAP_USE_TLS`: Use LDAP SSL. Defaults: **False**. +- `AIRFLOW_LDAP_ALLOW_SELF_SIGNED`: Allow self signed certicates in LDAP ssl. Default: **True**. +- `AIRFLOW_LDAP_TLS_CA_CERTIFICATE`: File that store the CA for LDAP ssl. No defaults. +- `AIRFLOW_USER_REGISTRATION_ROLE`: Role for the created user. Default: **Public** + +> In addition to the previous environment variables, all the parameters from the configuration file can be overwritten by using environment variables with this format: `AIRFLOW__{SECTION}__{KEY}`. Note the double underscores. + +### Specifying Environment variables using Docker Compose + +```yaml +version: '2' + +services: + airflow: + image: bitnami/airflow:latest + environment: + - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW_EXECUTOR=CeleryExecutor + - AIRFLOW_DATABASE_NAME=bitnami_airflow + - AIRFLOW_DATABASE_USERNAME=bn_airflow + - AIRFLOW_DATABASE_PASSWORD=bitnami1 + - AIRFLOW_PASSWORD=bitnami123 + - AIRFLOW_USERNAME=user + - AIRFLOW_EMAIL=user@example.com +``` + +### Specifying Environment variables on the Docker command line + +```console +$ docker run -d --name airflow -p 8080:8080 \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + -e AIRFLOW_PASSWORD=bitnami123 \ + -e AIRFLOW_USERNAME=user \ + -e AIRFLOW_EMAIL=user@example.com \ + --volume airflow_data:/bitnami \ + bitnami/airflow:latest +``` + +### SMTP Configuration + +To configure Airflow to send email using SMTP you can set the following environment variables: + +- `AIRFLOW__SMTP__SMTP_HOST`: Host for outgoing SMTP email. Default: **localhost** +- `AIRFLOW__SMTP__SMTP_PORT`: Port for outgoing SMTP email. Default: **25** +- `AIRFLOW__SMTP__SMTP_STARTTLS`: To use TLS communication. Default: **True** +- `AIRFLOW__SMTP__SMTP_SSL`: To use SSL communication. Default: **False** +- `AIRFLOW__SMTP__SMTP_USER`: User of SMTP used for authentication (likely email). No defaults. +- `AIRFLOW__SMTP__SMTP_PASSWORD`: Password for SMTP. No defaults. +- `AIRFLOW__SMTP__SMTP_MAIL_FROM`: To modify the "from email address". Default: **airflow@example.com** + +This would be an example of SMTP configuration using a GMail account: + + * docker-compose (application part): + +```yaml + airflow: + image: bitnami/airflow:latest + environment: + - AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - AIRFLOW_EXECUTOR=CeleryExecutor + - AIRFLOW_DATABASE_NAME=bitnami_airflow + - AIRFLOW_DATABASE_USERNAME=bn_airflow + - AIRFLOW_DATABASE_PASSWORD=bitnami1 + - AIRFLOW_PASSWORD=bitnami + - AIRFLOW_USERNAME=user + - AIRFLOW_EMAIL=user@email.com + - AIRFLOW__SMTP__SMTP_HOST=smtp@gmail.com + - AIRFLOW__SMTP__SMTP_USER=your_email@gmail.com + - AIRFLOW__SMTP__SMTP_PASSWORD=your_password + - AIRFLOW__SMTP__SMTP_PORT=587 + ports: + - '8080:8080' + volumes: + - airflow_data:/bitnami +``` + +* For manual execution: + +```console +$ docker run -d --name airflow -p 8080:8080 \ + -e AIRFLOW_FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= \ + -e AIRFLOW_EXECUTOR=CeleryExecutor \ + -e AIRFLOW_DATABASE_NAME=bitnami_airflow \ + -e AIRFLOW_DATABASE_USERNAME=bn_airflow \ + -e AIRFLOW_DATABASE_PASSWORD=bitnami1 \ + -e AIRFLOW_PASSWORD=bitnami123 \ + -e AIRFLOW_USERNAME=user \ + -e AIRFLOW_EMAIL=user@example.com \ + -e AIRFLOW__SMTP__SMTP_HOST=smtp@gmail.com \ + -e AIRFLOW__SMTP__SMTP_USER=your_email@gmail.com \ + -e AIRFLOW__SMTP__SMTP_PASSWORD=your_password \ + -e AIRFLOW__SMTP__SMTP_PORT=587 \ + --volume airflow_data:/bitnami \ + bitnami/airflow:latest +``` + +# Notable Changes + +## 1.10.15-debian-10-r17 and 2.0.1-debian-10-r50 + +- The size of the container image has been decreased. +- The configuration logic is now based on Bash scripts in the *rootfs/* folder. + +# Contributing + +We'd love for you to contribute to this container. You can request new features by creating an [issue](https://github.com/bitnami/bitnami-docker-airflow/issues), or submit a [pull request](https://github.com/bitnami/bitnami-docker-airflow/pulls) with your contribution. + +# Issues + +If you encountered a problem running this container, you can file an [issue](https://github.com/bitnami/bitnami-docker-airflow/issues/new). For us to provide better support, be sure to include the following information in your issue: + +- Host OS and version +- Docker version (`$ docker version`) +- Output of `$ docker info` +- Version of this container (`$ echo $BITNAMI_IMAGE_VERSION` inside the container) +- The command you used to run the container, and any relevant output you saw (masking any sensitive information) + +# License + +Copyright 2015-2021 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml new file mode 100644 index 0000000..876f9e7 --- /dev/null +++ b/hardening_manifest.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: v1 + +# The repository name in registry1, excluding /ironbank/ +name: "bitnami/airflow-worker" + +# List of tags to push for the repository in registry1 +# The most specific version should be the first tag and will be shown +# on ironbank.dsop.io +tags: +- "2.0.1" + +# Build args passed to Dockerfile ARGs +args: + BASE_IMAGE: "redhat/ubi/ubi8" + BASE_TAG: "8.3" + +# Docker image labels +labels: + org.opencontainers.image.title: "airflow-worker" + ## Human-readable description of the software packaged in the image + org.opencontainers.image.description: "Apache Airflow is a tool to express and execute workflows as directed acyclic graphs (DAGs). It includes utilities to schedule tasks, monitor task progress and handle task dependencies." + ## License(s) under which contained software is distributed + org.opencontainers.image.licenses: "Apache-2.0" + ## URL to find more information on the image + org.opencontainers.image.url: "https://airflow.apache.org" + ## Name of the distributing entity, organization or individual + org.opencontainers.image.vendor: "Bitnami" + org.opencontainers.image.version: "2.0.1" + ## Keywords to help with search (ex. "cicd,gitops,golang") + mil.dso.ironbank.image.keywords: "workflows,dags,tasks" + ## This value can be "opensource" or "commercial" + mil.dso.ironbank.image.type: "opensource" + ## Product the image belongs to for grouping multiple images + mil.dso.ironbank.product.name: "Airflow" + +# List of resources to make available to the offline build context +resources: +- tag: bitnami/airflow-worker:2.0.1-debian-10-r53 + url: docker://docker.io/bitnami/airflow@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae38 + +# List of project maintainers +maintainers: +- name: "Austin Denton" + username: "austindenton" + email: "austindenton@seed-innovations.com" + cht_member: false \ No newline at end of file diff --git a/prebuildfs/opt/bitnami/.bitnami_components.json b/prebuildfs/opt/bitnami/.bitnami_components.json new file mode 100644 index 0000000..67b2f78 --- /dev/null +++ b/prebuildfs/opt/bitnami/.bitnami_components.json @@ -0,0 +1,44 @@ +{ + "airflow": { + "arch": "amd64", + "digest": "d837c8af9305cfcbed7dd0493336ba0e38d7a3aa211192a8f05d117a7b7734ab", + "distro": "debian-10", + "type": "NAMI", + "version": "2.0.1-4" + }, + "gosu": { + "arch": "amd64", + "digest": "4d858ac600c38af8de454c27b7f65c0074ec3069880cb16d259a6e40a46bbc50", + "distro": "debian-10", + "type": "NAMI", + "version": "1.12.0-2" + }, + "ini-file": { + "arch": "amd64", + "digest": "d89528e5d733f34ae030984584659ff10a36370d40332bd8d41c047764d39cda", + "distro": "debian-10", + "type": "NAMI", + "version": "1.3.0-2" + }, + "postgresql-client": { + "arch": "amd64", + "digest": "6ae2df74c4cc145690104c9bfbd4f9977cc00d26b3a010bb1eba74d92048485d", + "distro": "debian-10", + "type": "NAMI", + "version": "10.16.0-0" + }, + "python": { + "arch": "amd64", + "digest": "4f1f6b81a3617dfaaa2c579510118ef6df07119977a5d6ca7df3cf485fca709a", + "distro": "debian-10", + "type": "NAMI", + "version": "3.8.9-0" + }, + "wait-for-port": { + "arch": "amd64", + "digest": "7521d9a4f9e4e182bf32977e234026caa7b03759799868335bccb1edd8f8fd12", + "distro": "debian-10", + "type": "NAMI", + "version": "1.0.0-3" + } +} \ No newline at end of file diff --git a/prebuildfs/opt/bitnami/licenses/licenses.txt b/prebuildfs/opt/bitnami/licenses/licenses.txt new file mode 100644 index 0000000..c76ba31 --- /dev/null +++ b/prebuildfs/opt/bitnami/licenses/licenses.txt @@ -0,0 +1,3 @@ +Bitnami containers ship with software bundles. You can find the licenses under: +/opt/bitnami/nami/COPYING +/opt/bitnami/[name-of-bundle]/licenses/[bundle-version].txt diff --git a/prebuildfs/opt/bitnami/scripts/libbitnami.sh b/prebuildfs/opt/bitnami/scripts/libbitnami.sh new file mode 100644 index 0000000..ef29e36 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libbitnami.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Bitnami custom library + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/liblog.sh + +# Constants +BOLD='\033[1m' + +# Functions + +######################## +# Print the welcome page +# Globals: +# DISABLE_WELCOME_MESSAGE +# BITNAMI_APP_NAME +# Arguments: +# None +# Returns: +# None +######################### +print_welcome_page() { + if [[ -z "${DISABLE_WELCOME_MESSAGE:-}" ]]; then + if [[ -n "$BITNAMI_APP_NAME" ]]; then + print_image_welcome_page + fi + fi +} + +######################## +# Print the welcome page for a Bitnami Docker image +# Globals: +# BITNAMI_APP_NAME +# Arguments: +# None +# Returns: +# None +######################### +print_image_welcome_page() { + local github_url="https://github.com/bitnami/bitnami-docker-${BITNAMI_APP_NAME}" + + log "" + log "${BOLD}Welcome to the Bitnami ${BITNAMI_APP_NAME} container${RESET}" + log "Subscribe to project updates by watching ${BOLD}${github_url}${RESET}" + log "Submit issues and feature requests at ${BOLD}${github_url}/issues${RESET}" + log "" +} + diff --git a/prebuildfs/opt/bitnami/scripts/libcomponent.sh b/prebuildfs/opt/bitnami/scripts/libcomponent.sh new file mode 100644 index 0000000..1d8c6bf --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libcomponent.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Library for managing Bitnami components + +# Constants +CACHE_ROOT="/tmp/bitnami/pkg/cache" +DOWNLOAD_URL="https://downloads.bitnami.com/files/stacksmith" + +# Functions + +######################## +# Download and unpack a Bitnami package +# Globals: +# OS_NAME +# OS_ARCH +# OS_FLAVOUR +# Arguments: +# $1 - component's name +# $2 - component's version +# Returns: +# None +######################### +component_unpack() { + local name="${1:?name is required}" + local version="${2:?version is required}" + local base_name="${name}-${version}-${OS_NAME}-${OS_ARCH}-${OS_FLAVOUR}" + local package_sha256="" + local directory="/opt/bitnami" + + # Validate arguments + shift 2 + while [ "$#" -gt 0 ]; do + case "$1" in + -c|--checksum) + shift + package_sha256="${1:?missing package checksum}" + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + + echo "Downloading $base_name package" + if [ -f "${CACHE_ROOT}/${base_name}.tar.gz" ]; then + echo "${CACHE_ROOT}/${base_name}.tar.gz already exists, skipping download." + cp "${CACHE_ROOT}/${base_name}.tar.gz" . + rm "${CACHE_ROOT}/${base_name}.tar.gz" + if [ -f "${CACHE_ROOT}/${base_name}.tar.gz.sha256" ]; then + echo "Using the local sha256 from ${CACHE_ROOT}/${base_name}.tar.gz.sha256" + package_sha256="$(< "${CACHE_ROOT}/${base_name}.tar.gz.sha256")" + rm "${CACHE_ROOT}/${base_name}.tar.gz.sha256" + fi + else + curl --remote-name --silent "${DOWNLOAD_URL}/${base_name}.tar.gz" + fi + if [ -n "$package_sha256" ]; then + echo "Verifying package integrity" + echo "$package_sha256 ${base_name}.tar.gz" | sha256sum --check - + fi + tar --directory "${directory}" --extract --gunzip --file "${base_name}.tar.gz" --no-same-owner --strip-components=2 "${base_name}/files/" + rm "${base_name}.tar.gz" +} diff --git a/prebuildfs/opt/bitnami/scripts/libfile.sh b/prebuildfs/opt/bitnami/scripts/libfile.sh new file mode 100644 index 0000000..b09575c --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libfile.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Library for managing files + +# Functions + +######################## +# Replace a regex in a file +# Arguments: +# $1 - filename +# $2 - match regex +# $3 - substitute regex +# $4 - use POSIX regex. Default: true +# Returns: +# None +######################### +replace_in_file() { + local filename="${1:?filename is required}" + local match_regex="${2:?match regex is required}" + local substitute_regex="${3:?substitute regex is required}" + local posix_regex=${4:-true} + + local result + + # We should avoid using 'sed in-place' substitutions + # 1) They are not compatible with files mounted from ConfigMap(s) + # 2) We found incompatibility issues with Debian10 and "in-place" substitutions + del=$'\001' # Use a non-printable character as a 'sed' delimiter to avoid issues + if [[ $posix_regex = true ]]; then + result="$(sed -E "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")" + else + result="$(sed "s${del}${match_regex}${del}${substitute_regex}${del}g" "$filename")" + fi + echo "$result" > "$filename" +} + +######################## +# Remove a line in a file based on a regex +# Arguments: +# $1 - filename +# $2 - match regex +# $3 - use POSIX regex. Default: true +# Returns: +# None +######################### +remove_in_file() { + local filename="${1:?filename is required}" + local match_regex="${2:?match regex is required}" + local posix_regex=${3:-true} + local result + + # We should avoid using 'sed in-place' substitutions + # 1) They are not compatible with files mounted from ConfigMap(s) + # 2) We found incompatibility issues with Debian10 and "in-place" substitutions + if [[ $posix_regex = true ]]; then + result="$(sed -E "/$match_regex/d" "$filename")" + else + result="$(sed "/$match_regex/d" "$filename")" + fi + echo "$result" > "$filename" +} + +######################## +# Appends text after the last line matching a pattern +# Arguments: +# $1 - file +# $2 - match regex +# $3 - contents to add +# Returns: +# None +######################### +append_file_after_last_match() { + local file="${1:?missing file}" + local match_regex="${2:?missing pattern}" + local value="${3:?missing value}" + + # We read the file in reverse, replace the first match (0,/pattern/s) and then reverse the results again + result="$(tac "$file" | sed -E "0,/($match_regex)/s||${value}\n\1|" | tac)" + echo "$result" > "$file" +} diff --git a/prebuildfs/opt/bitnami/scripts/libfs.sh b/prebuildfs/opt/bitnami/scripts/libfs.sh new file mode 100644 index 0000000..ae5e245 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libfs.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# +# Library for file system actions + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/liblog.sh + +# Functions + +######################## +# Ensure a file/directory is owned (user and group) but the given user +# Arguments: +# $1 - filepath +# $2 - owner +# Returns: +# None +######################### +owned_by() { + local path="${1:?path is missing}" + local owner="${2:?owner is missing}" + + chown "$owner":"$owner" "$path" +} + +######################## +# Ensure a directory exists and, optionally, is owned by the given user +# Arguments: +# $1 - directory +# $2 - owner +# Returns: +# None +######################### +ensure_dir_exists() { + local dir="${1:?directory is missing}" + local owner="${2:-}" + + mkdir -p "${dir}" + if [[ -n $owner ]]; then + owned_by "$dir" "$owner" + fi +} + +######################## +# Checks whether a directory is empty or not +# arguments: +# $1 - directory +# returns: +# boolean +######################### +is_dir_empty() { + local dir="${1:?missing directory}" + + if [[ ! -e "$dir" ]] || [[ -z "$(ls -A "$dir")" ]]; then + true + else + false + fi +} + +######################## +# Checks whether a mounted directory is empty or not +# arguments: +# $1 - directory +# returns: +# boolean +######################### +is_mounted_dir_empty() { + local dir="${1:?missing directory}" + + if is_dir_empty "$dir" || find "$dir" -mindepth 1 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" -exec false {} +; then + true + else + false + fi +} + +######################## +# Checks whether a file can be written to or not +# arguments: +# $1 - file +# returns: +# boolean +######################### +is_file_writable() { + local file="${1:?missing file}" + local dir + dir="$(dirname "$file")" + + if [[ ( -f "$file" && -w "$file" ) || ( ! -f "$file" && -d "$dir" && -w "$dir" ) ]]; then + true + else + false + fi +} + +######################## +# Relativize a path +# arguments: +# $1 - path +# $2 - base +# returns: +# None +######################### +relativize() { + local -r path="${1:?missing path}" + local -r base="${2:?missing base}" + pushd "$base" >/dev/null || exit + realpath -q --no-symlinks --relative-base="$base" "$path" | sed -e 's|^/$|.|' -e 's|^/||' + popd >/dev/null || exit +} + +######################## +# Configure permisions and ownership recursively +# Globals: +# None +# Arguments: +# $1 - paths (as a string). +# Flags: +# -f|--file-mode - mode for directories. +# -d|--dir-mode - mode for files. +# -u|--user - user +# -g|--group - group +# Returns: +# None +######################### +configure_permissions_ownership() { + local -r paths="${1:?paths is missing}" + local dir_mode="" + local file_mode="" + local user="" + local group="" + + # Validate arguments + shift 1 + while [ "$#" -gt 0 ]; do + case "$1" in + -f|--file-mode) + shift + file_mode="${1:?missing mode for files}" + ;; + -d|--dir-mode) + shift + dir_mode="${1:?missing mode for directories}" + ;; + -u|--user) + shift + user="${1:?missing user}" + ;; + -g|--group) + shift + group="${1:?missing group}" + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + + read -r -a filepaths <<< "$paths" + for p in "${filepaths[@]}"; do + if [[ -e "$p" ]]; then + if [[ -n $dir_mode ]]; then + find -L "$p" -type d -exec chmod "$dir_mode" {} \; + fi + if [[ -n $file_mode ]]; then + find -L "$p" -type f -exec chmod "$file_mode" {} \; + fi + if [[ -n $user ]] && [[ -n $group ]]; then + chown -LR "$user":"$group" "$p" + elif [[ -n $user ]] && [[ -z $group ]]; then + chown -LR "$user" "$p" + elif [[ -z $user ]] && [[ -n $group ]]; then + chgrp -LR "$group" "$p" + fi + else + stderr_print "$p does not exist" + fi + done +} diff --git a/prebuildfs/opt/bitnami/scripts/libhook.sh b/prebuildfs/opt/bitnami/scripts/libhook.sh new file mode 100644 index 0000000..9694852 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libhook.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# +# Library to use for scripts expected to be used as Kubernetes lifecycle hooks + +# shellcheck disable=SC1091 + +# Load generic libraries +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libos.sh + +# Override functions that log to stdout/stderr of the current process, so they print to process 1 +for function_to_override in stderr_print debug_execute; do + # Output is sent to output of process 1 and thus end up in the container log + # The hook output in general isn't saved + eval "$(declare -f "$function_to_override") >/proc/1/fd/1 2>/proc/1/fd/2" +done diff --git a/prebuildfs/opt/bitnami/scripts/liblog.sh b/prebuildfs/opt/bitnami/scripts/liblog.sh new file mode 100644 index 0000000..60ec4cb --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/liblog.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# +# Library for logging functions + +# Constants +RESET='\033[0m' +RED='\033[38;5;1m' +GREEN='\033[38;5;2m' +YELLOW='\033[38;5;3m' +MAGENTA='\033[38;5;5m' +CYAN='\033[38;5;6m' + +# Functions + +######################## +# Print to STDERR +# Arguments: +# Message to print +# Returns: +# None +######################### +stderr_print() { + # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it + local bool="${BITNAMI_QUIET:-false}" + # comparison is performed without regard to the case of alphabetic characters + shopt -s nocasematch + if ! [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + printf "%b\\n" "${*}" >&2 + fi +} + +######################## +# Log message +# Arguments: +# Message to log +# Returns: +# None +######################### +log() { + stderr_print "${CYAN}${MODULE:-} ${MAGENTA}$(date "+%T.%2N ")${RESET}${*}" +} +######################## +# Log an 'info' message +# Arguments: +# Message to log +# Returns: +# None +######################### +info() { + log "${GREEN}INFO ${RESET} ==> ${*}" +} +######################## +# Log message +# Arguments: +# Message to log +# Returns: +# None +######################### +warn() { + log "${YELLOW}WARN ${RESET} ==> ${*}" +} +######################## +# Log an 'error' message +# Arguments: +# Message to log +# Returns: +# None +######################### +error() { + log "${RED}ERROR${RESET} ==> ${*}" +} +######################## +# Log a 'debug' message +# Globals: +# BITNAMI_DEBUG +# Arguments: +# None +# Returns: +# None +######################### +debug() { + # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it + local bool="${BITNAMI_DEBUG:-false}" + # comparison is performed without regard to the case of alphabetic characters + shopt -s nocasematch + if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + log "${MAGENTA}DEBUG${RESET} ==> ${*}" + fi +} + +######################## +# Indent a string +# Arguments: +# $1 - string +# $2 - number of indentation characters (default: 4) +# $3 - indentation character (default: " ") +# Returns: +# None +######################### +indent() { + local string="${1:-}" + local num="${2:?missing num}" + local char="${3:-" "}" + # Build the indentation unit string + local indent_unit="" + for ((i = 0; i < num; i++)); do + indent_unit="${indent_unit}${char}" + done + echo "$string" | sed "s/^/${indent_unit}/" +} diff --git a/prebuildfs/opt/bitnami/scripts/libnet.sh b/prebuildfs/opt/bitnami/scripts/libnet.sh new file mode 100644 index 0000000..4391dac --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libnet.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Library for network functions + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/liblog.sh + +# Functions + +######################## +# Resolve IP address for a host/domain (i.e. DNS lookup) +# Arguments: +# $1 - Hostname to resolve +# $2 - IP address version (v4, v6), leave empty for resolving to any version +# Returns: +# IP +######################### +dns_lookup() { + local host="${1:?host is missing}" + local ip_version="${2:-}" + getent "ahosts${ip_version}" "$host" | awk '/STREAM/ {print $1 }' | head -n 1 +} + +######################### +# Wait for a hostname and return the IP +# Arguments: +# $1 - hostname +# $2 - number of retries +# $3 - seconds to wait between retries +# Returns: +# - IP address that corresponds to the hostname +######################### +wait_for_dns_lookup() { + local hostname="${1:?hostname is missing}" + local retries="${2:-5}" + local seconds="${3:-1}" + check_host() { + if [[ $(dns_lookup "$hostname") == "" ]]; then + false + else + true + fi + } + # Wait for the host to be ready + retry_while "check_host ${hostname}" "$retries" "$seconds" + dns_lookup "$hostname" +} + +######################## +# Get machine's IP +# Arguments: +# None +# Returns: +# Machine IP +######################### +get_machine_ip() { + local -a ip_addresses + local hostname + hostname="$(hostname)" + read -r -a ip_addresses <<< "$(dns_lookup "$hostname" | xargs echo)" + if [[ "${#ip_addresses[@]}" -gt 1 ]]; then + warn "Found more than one IP address associated to hostname ${hostname}: ${ip_addresses[*]}, will use ${ip_addresses[0]}" + elif [[ "${#ip_addresses[@]}" -lt 1 ]]; then + error "Could not find any IP address associated to hostname ${hostname}" + exit 1 + fi + echo "${ip_addresses[0]}" +} + +######################## +# Check if the provided argument is a resolved hostname +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_hostname_resolved() { + local -r host="${1:?missing value}" + if [[ -n "$(dns_lookup "$host")" ]]; then + true + else + false + fi +} + +######################## +# Parse URL +# Globals: +# None +# Arguments: +# $1 - uri - String +# $2 - component to obtain. Valid options (scheme, authority, userinfo, host, port, path, query or fragment) - String +# Returns: +# String +parse_uri() { + local uri="${1:?uri is missing}" + local component="${2:?component is missing}" + + # Solution based on https://tools.ietf.org/html/rfc3986#appendix-B with + # additional sub-expressions to split authority into userinfo, host and port + # Credits to Patryk Obara (see https://stackoverflow.com/a/45977232/6694969) + local -r URI_REGEX='^(([^:/?#]+):)?(//((([^@/?#]+)@)?([^:/?#]+)(:([0-9]+))?))?(/([^?#]*))?(\?([^#]*))?(#(.*))?' + # || | ||| | | | | | | | | | + # |2 scheme | ||6 userinfo 7 host | 9 port | 11 rpath | 13 query | 15 fragment + # 1 scheme: | |5 userinfo@ 8 :... 10 path 12 ?... 14 #... + # | 4 authority + # 3 //... + local index=0 + case "$component" in + scheme) + index=2 + ;; + authority) + index=4 + ;; + userinfo) + index=6 + ;; + host) + index=7 + ;; + port) + index=9 + ;; + path) + index=10 + ;; + query) + index=13 + ;; + fragment) + index=14 + ;; + *) + stderr_print "unrecognized component $component" + return 1 + ;; + esac + [[ "$uri" =~ $URI_REGEX ]] && echo "${BASH_REMATCH[${index}]}" +} diff --git a/prebuildfs/opt/bitnami/scripts/libos.sh b/prebuildfs/opt/bitnami/scripts/libos.sh new file mode 100644 index 0000000..6a75122 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libos.sh @@ -0,0 +1,350 @@ +#!/bin/bash +# +# Library for operating system actions + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libfs.sh + +# Functions + +######################## +# Check if an user exists in the system +# Arguments: +# $1 - user +# Returns: +# Boolean +######################### +user_exists() { + local user="${1:?user is missing}" + id "$user" >/dev/null 2>&1 +} + +######################## +# Check if a group exists in the system +# Arguments: +# $1 - group +# Returns: +# Boolean +######################### +group_exists() { + local group="${1:?group is missing}" + getent group "$group" >/dev/null 2>&1 +} + +######################## +# Create a group in the system if it does not exist already +# Arguments: +# $1 - group +# Flags: +# -s|--system - Whether to create new user as system user (uid <= 999) +# Returns: +# None +######################### +ensure_group_exists() { + local group="${1:?group is missing}" + local is_system_user=false + + # Validate arguments + shift 1 + while [ "$#" -gt 0 ]; do + case "$1" in + -s|--system) + is_system_user=true + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + + if ! group_exists "$group"; then + local -a args=("$group") + $is_system_user && args+=("--system") + groupadd "${args[@]}" >/dev/null 2>&1 + fi +} + +######################## +# Create an user in the system if it does not exist already +# Arguments: +# $1 - user +# Flags: +# -g|--group - the group the new user should belong to +# -h|--home - the home directory for the new user +# -s|--system - whether to create new user as system user (uid <= 999) +# Returns: +# None +######################### +ensure_user_exists() { + local user="${1:?user is missing}" + local group="" + local home="" + local is_system_user=false + + # Validate arguments + shift 1 + while [ "$#" -gt 0 ]; do + case "$1" in + -g|--group) + shift + group="${1:?missing group}" + ;; + -h|--home) + shift + home="${1:?missing home directory}" + ;; + -s|--system) + is_system_user=true + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + + if ! user_exists "$user"; then + local -a user_args=("-N" "$user") + $is_system_user && user_args+=("--system") + useradd "${user_args[@]}" >/dev/null 2>&1 + fi + + if [[ -n "$group" ]]; then + local -a group_args=("$group") + $is_system_user && group_args+=("--system") + ensure_group_exists "${group_args[@]}" + usermod -g "$group" "$user" >/dev/null 2>&1 + fi + + if [[ -n "$home" ]]; then + mkdir -p "$home" + usermod -d "$home" "$user" >/dev/null 2>&1 + configure_permissions_ownership "$home" -d "775" -f "664" -u "$user" -g "$group" + fi +} + +######################## +# Check if the script is currently running as root +# Arguments: +# $1 - user +# $2 - group +# Returns: +# Boolean +######################### +am_i_root() { + if [[ "$(id -u)" = "0" ]]; then + true + else + false + fi +} + +######################## +# Get total memory available +# Arguments: +# None +# Returns: +# Memory in bytes +######################### +get_total_memory() { + echo $(($(grep MemTotal /proc/meminfo | awk '{print $2}') / 1024)) +} + +######################## +# Get machine size depending on specified memory +# Globals: +# None +# Arguments: +# None +# Flags: +# --memory - memory size (optional) +# Returns: +# Detected instance size +######################### +get_machine_size() { + local memory="" + # Validate arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + --memory) + shift + memory="${1:?missing memory}" + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + if [[ -z "$memory" ]]; then + debug "Memory was not specified, detecting available memory automatically" + memory="$(get_total_memory)" + fi + sanitized_memory=$(convert_to_mb "$memory") + if [[ "$sanitized_memory" -gt 26000 ]]; then + echo 2xlarge + elif [[ "$sanitized_memory" -gt 13000 ]]; then + echo xlarge + elif [[ "$sanitized_memory" -gt 6000 ]]; then + echo large + elif [[ "$sanitized_memory" -gt 3000 ]]; then + echo medium + elif [[ "$sanitized_memory" -gt 1500 ]]; then + echo small + else + echo micro + fi +} + +######################## +# Get machine size depending on specified memory +# Globals: +# None +# Arguments: +# $1 - memory size (optional) +# Returns: +# Detected instance size +######################### +get_supported_machine_sizes() { + echo micro small medium large xlarge 2xlarge +} + +######################## +# Convert memory size from string to amount of megabytes (i.e. 2G -> 2048) +# Globals: +# None +# Arguments: +# $1 - memory size +# Returns: +# Result of the conversion +######################### +convert_to_mb() { + local amount="${1:-}" + if [[ $amount =~ ^([0-9]+)(m|M|g|G) ]]; then + size="${BASH_REMATCH[1]}" + unit="${BASH_REMATCH[2]}" + if [[ "$unit" = "g" || "$unit" = "G" ]]; then + amount="$((size * 1024))" + else + amount="$size" + fi + fi + echo "$amount" +} + + +######################### +# Redirects output to /dev/null if debug mode is disabled +# Globals: +# BITNAMI_DEBUG +# Arguments: +# $@ - Command to execute +# Returns: +# None +######################### +debug_execute() { + if ${BITNAMI_DEBUG:-false}; then + "$@" + else + "$@" >/dev/null 2>&1 + fi +} + +######################## +# Retries a command a given number of times +# Arguments: +# $1 - cmd (as a string) +# $2 - max retries. Default: 12 +# $3 - sleep between retries (in seconds). Default: 5 +# Returns: +# Boolean +######################### +retry_while() { + local cmd="${1:?cmd is missing}" + local retries="${2:-12}" + local sleep_time="${3:-5}" + local return_value=1 + + read -r -a command <<< "$cmd" + for ((i = 1 ; i <= retries ; i+=1 )); do + "${command[@]}" && return_value=0 && break + sleep "$sleep_time" + done + return $return_value +} + +######################## +# Generate a random string +# Arguments: +# -t|--type - String type (ascii, alphanumeric, numeric), defaults to ascii +# -c|--count - Number of characters, defaults to 32 +# Arguments: +# None +# Returns: +# None +# Returns: +# String +######################### +generate_random_string() { + local type="ascii" + local count="32" + local filter + local result + # Validate arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + -t|--type) + shift + type="$1" + ;; + -c|--count) + shift + count="$1" + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + # Validate type + case "$type" in + ascii) + filter="[:print:]" + ;; + alphanumeric) + filter="a-zA-Z0-9" + ;; + numeric) + filter="0-9" + ;; + *) + echo "Invalid type ${type}" >&2 + return 1 + esac + # Obtain count + 10 lines from /dev/urandom to ensure that the resulting string has the expected size + # Note there is a very small chance of strings starting with EOL character + # Therefore, the higher amount of lines read, this will happen less frequently + result="$(head -n "$((count + 10))" /dev/urandom | tr -dc "$filter" | head -c "$count")" + echo "$result" +} + +######################## +# Create md5 hash from a string +# Arguments: +# $1 - string +# Returns: +# md5 hash - string +######################### +generate_md5_hash() { + local -r str="${1:?missing input string}" + echo -n "$str" | md5sum | awk '{print $1}' +} diff --git a/prebuildfs/opt/bitnami/scripts/libpersistence.sh b/prebuildfs/opt/bitnami/scripts/libpersistence.sh new file mode 100644 index 0000000..264a929 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libpersistence.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Bitnami persistence library +# Used for bringing persistence capabilities to applications that don't have clear separation of data and logic + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/libfs.sh +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libversion.sh + +# Functions + +######################## +# Persist an application directory +# Globals: +# BITNAMI_ROOT_DIR +# BITNAMI_VOLUME_DIR +# Arguments: +# $1 - App folder name +# $2 - List of app files to persist +# Returns: +# true if all steps succeeded, false otherwise +######################### +persist_app() { + local -r app="${1:?missing app}" + local -a files_to_restore + read -r -a files_to_persist <<< "$2" + local -r install_dir="${BITNAMI_ROOT_DIR}/${app}" + local -r persist_dir="${BITNAMI_VOLUME_DIR}/${app}" + # Persist the individual files + if [[ "${#files_to_persist[@]}" -lt 0 ]]; then + warn "No files are configured to be persisted" + return + fi + pushd "$install_dir" >/dev/null || exit + local file_to_persist_relative file_to_persist_destination file_to_persist_destination_folder + local -r tmp_file="/tmp/perms.acl" + for file_to_persist in "${files_to_persist[@]}"; do + if [[ ! -f "$file_to_persist" && ! -d "$file_to_persist" ]]; then + error "Cannot persist '${file_to_persist}' because it does not exist" + return 1 + fi + file_to_persist_relative="$(relativize "$file_to_persist" "$install_dir")" + file_to_persist_destination="${persist_dir}/${file_to_persist_relative}" + file_to_persist_destination_folder="$(dirname "$file_to_persist_destination")" + # Get original permissions for existing files, which will be applied later + # Exclude the root directory with 'sed', to avoid issues when copying the entirety of it to a volume + getfacl -R "$file_to_persist_relative" | sed -E '/# file: (\..+|[^.])/,$!d' > "$tmp_file" + # Copy directories to the volume + ensure_dir_exists "$file_to_persist_destination_folder" + cp -Lr --preserve=links "$file_to_persist_relative" "$file_to_persist_destination_folder" + # Restore permissions + pushd "$persist_dir" >/dev/null || exit + if am_i_root; then + setfacl --restore="$tmp_file" + else + # When running as non-root, don't change ownership + setfacl --restore=<(grep -E -v '^# (owner|group):' "$tmp_file") + fi + popd >/dev/null || exit + done + popd >/dev/null || exit + rm -f "$tmp_file" + # Install the persisted files into the installation directory, via symlinks + restore_persisted_app "$@" +} + +######################## +# Restore a persisted application directory +# Globals: +# BITNAMI_ROOT_DIR +# BITNAMI_VOLUME_DIR +# FORCE_MAJOR_UPGRADE +# Arguments: +# $1 - App folder name +# $2 - List of app files to restore +# Returns: +# true if all steps succeeded, false otherwise +######################### +restore_persisted_app() { + local -r app="${1:?missing app}" + local -a files_to_restore + read -r -a files_to_restore <<< "$2" + local -r install_dir="${BITNAMI_ROOT_DIR}/${app}" + local -r persist_dir="${BITNAMI_VOLUME_DIR}/${app}" + # Restore the individual persisted files + if [[ "${#files_to_restore[@]}" -lt 0 ]]; then + warn "No persisted files are configured to be restored" + return + fi + local file_to_restore_relative file_to_restore_origin file_to_restore_destination + for file_to_restore in "${files_to_restore[@]}"; do + file_to_restore_relative="$(relativize "$file_to_restore" "$install_dir")" + # We use 'realpath --no-symlinks' to ensure that the case of '.' is covered and the directory is removed + file_to_restore_origin="$(realpath --no-symlinks "${install_dir}/${file_to_restore_relative}")" + file_to_restore_destination="$(realpath --no-symlinks "${persist_dir}/${file_to_restore_relative}")" + rm -rf "$file_to_restore_origin" + ln -sfn "$file_to_restore_destination" "$file_to_restore_origin" + done +} + +######################## +# Check if an application directory was already persisted +# Globals: +# BITNAMI_VOLUME_DIR +# Arguments: +# $1 - App folder name +# Returns: +# true if all steps succeeded, false otherwise +######################### +is_app_initialized() { + local -r app="${1:?missing app}" + local -r persist_dir="${BITNAMI_VOLUME_DIR}/${app}" + if ! is_mounted_dir_empty "$persist_dir"; then + true + else + false + fi +} diff --git a/prebuildfs/opt/bitnami/scripts/libservice.sh b/prebuildfs/opt/bitnami/scripts/libservice.sh new file mode 100644 index 0000000..ae8f310 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libservice.sh @@ -0,0 +1,235 @@ +#!/bin/bash +# +# Library for managing services + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/libvalidations.sh +. /opt/bitnami/scripts/liblog.sh + +# Functions + +######################## +# Read the provided pid file and returns a PID +# Arguments: +# $1 - Pid file +# Returns: +# PID +######################### +get_pid_from_file() { + local pid_file="${1:?pid file is missing}" + + if [[ -f "$pid_file" ]]; then + if [[ -n "$(< "$pid_file")" ]] && [[ "$(< "$pid_file")" -gt 0 ]]; then + echo "$(< "$pid_file")" + fi + fi +} + +######################## +# Check if a provided PID corresponds to a running service +# Arguments: +# $1 - PID +# Returns: +# Boolean +######################### +is_service_running() { + local pid="${1:?pid is missing}" + + kill -0 "$pid" 2>/dev/null +} + +######################## +# Stop a service by sending a termination signal to its pid +# Arguments: +# $1 - Pid file +# $2 - Signal number (optional) +# Returns: +# None +######################### +stop_service_using_pid() { + local pid_file="${1:?pid file is missing}" + local signal="${2:-}" + local pid + + pid="$(get_pid_from_file "$pid_file")" + [[ -z "$pid" ]] || ! is_service_running "$pid" && return + + if [[ -n "$signal" ]]; then + kill "-${signal}" "$pid" + else + kill "$pid" + fi + + local counter=10 + while [[ "$counter" -ne 0 ]] && is_service_running "$pid"; do + sleep 1 + counter=$((counter - 1)) + done +} + +######################## +# Start cron daemon +# Arguments: +# None +# Returns: +# true if started correctly, false otherwise +######################### +cron_start() { + if [[ -x "/usr/sbin/cron" ]]; then + /usr/sbin/cron + elif [[ -x "/usr/sbin/crond" ]]; then + /usr/sbin/crond + else + false + fi +} + +######################## +# Generate a cron configuration file for a given service +# Arguments: +# $1 - Service name +# $2 - Command +# Flags: +# --run-as - User to run as (default: root) +# --schedule - Cron schedule configuration (default: * * * * *) +# Returns: +# None +######################### +generate_cron_conf() { + local service_name="${1:?service name is missing}" + local cmd="${2:?command is missing}" + local run_as="root" + local schedule="* * * * *" + local clean="true" + + local clean="true" + + # Parse optional CLI flags + shift 2 + while [[ "$#" -gt 0 ]]; do + case "$1" in + --run-as) + shift + run_as="$1" + ;; + --schedule) + shift + schedule="$1" + ;; + --no-clean) + clean="false" + ;; + *) + echo "Invalid command line flag ${1}" >&2 + return 1 + ;; + esac + shift + done + + mkdir -p /etc/cron.d + if "$clean"; then + echo "${schedule} ${run_as} ${cmd}" > /etc/cron.d/"$service_name" + else + echo "${schedule} ${run_as} ${cmd}" >> /etc/cron.d/"$service_name" + fi +} + +######################## +# Generate a monit configuration file for a given service +# Arguments: +# $1 - Service name +# $2 - Pid file +# $3 - Start command +# $4 - Stop command +# Flags: +# --disabled - Whether to disable the monit configuration +# Returns: +# None +######################### +generate_monit_conf() { + local service_name="${1:?service name is missing}" + local pid_file="${2:?pid file is missing}" + local start_command="${3:?start command is missing}" + local stop_command="${4:?stop command is missing}" + local monit_conf_dir="/etc/monit/conf.d" + local disabled="no" + + # Parse optional CLI flags + shift 4 + while [[ "$#" -gt 0 ]]; do + case "$1" in + --disabled) + shift + disabled="$1" + ;; + *) + echo "Invalid command line flag ${1}" >&2 + return 1 + ;; + esac + shift + done + + is_boolean_yes "$disabled" && conf_suffix=".disabled" + mkdir -p "$monit_conf_dir" + cat >"${monit_conf_dir}/${service_name}.conf${conf_suffix:-}" <&2 + return 1 + ;; + esac + shift + done + + mkdir -p "$logrotate_conf_dir" + cat <"${logrotate_conf_dir}/${service_name}" +${log_path} { + ${period} + rotate ${rotations} + dateext + compress + copytruncate + missingok +$(indent "$extra" 2) +} +EOF +} diff --git a/prebuildfs/opt/bitnami/scripts/libvalidations.sh b/prebuildfs/opt/bitnami/scripts/libvalidations.sh new file mode 100644 index 0000000..8d82792 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libvalidations.sh @@ -0,0 +1,248 @@ +#!/bin/bash +# +# Validation functions library + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/liblog.sh + +# Functions + +######################## +# Check if the provided argument is an integer +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_int() { + local -r int="${1:?missing value}" + if [[ "$int" =~ ^-?[0-9]+ ]]; then + true + else + false + fi +} + +######################## +# Check if the provided argument is a positive integer +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_positive_int() { + local -r int="${1:?missing value}" + if is_int "$int" && (( "${int}" >= 0 )); then + true + else + false + fi +} + +######################## +# Check if the provided argument is a boolean or is the string 'yes/true' +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_boolean_yes() { + local -r bool="${1:-}" + # comparison is performed without regard to the case of alphabetic characters + shopt -s nocasematch + if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + true + else + false + fi +} + +######################## +# Check if the provided argument is a boolean yes/no value +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_yes_no_value() { + local -r bool="${1:-}" + if [[ "$bool" =~ ^(yes|no)$ ]]; then + true + else + false + fi +} + +######################## +# Check if the provided argument is a boolean true/false value +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_true_false_value() { + local -r bool="${1:-}" + if [[ "$bool" =~ ^(true|false)$ ]]; then + true + else + false + fi +} + +######################## +# Check if the provided argument is an empty string or not defined +# Arguments: +# $1 - Value to check +# Returns: +# Boolean +######################### +is_empty_value() { + local -r val="${1:-}" + if [[ -z "$val" ]]; then + true + else + false + fi +} + +######################## +# Validate if the provided argument is a valid port +# Arguments: +# $1 - Port to validate +# Returns: +# Boolean and error message +######################### +validate_port() { + local value + local unprivileged=0 + + # Parse flags + while [[ "$#" -gt 0 ]]; do + case "$1" in + -unprivileged) + unprivileged=1 + ;; + --) + shift + break + ;; + -*) + stderr_print "unrecognized flag $1" + return 1 + ;; + *) + break + ;; + esac + shift + done + + if [[ "$#" -gt 1 ]]; then + echo "too many arguments provided" + return 2 + elif [[ "$#" -eq 0 ]]; then + stderr_print "missing port argument" + return 1 + else + value=$1 + fi + + if [[ -z "$value" ]]; then + echo "the value is empty" + return 1 + else + if ! is_int "$value"; then + echo "value is not an integer" + return 2 + elif [[ "$value" -lt 0 ]]; then + echo "negative value provided" + return 2 + elif [[ "$value" -gt 65535 ]]; then + echo "requested port is greater than 65535" + return 2 + elif [[ "$unprivileged" = 1 && "$value" -lt 1024 ]]; then + echo "privileged port requested" + return 3 + fi + fi +} + +######################## +# Validate if the provided argument is a valid IPv4 address +# Arguments: +# $1 - IP to validate +# Returns: +# Boolean +######################### +validate_ipv4() { + local ip="${1:?ip is missing}" + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + read -r -a ip_array <<< "$(tr '.' ' ' <<< "$ip")" + [[ ${ip_array[0]} -le 255 && ${ip_array[1]} -le 255 \ + && ${ip_array[2]} -le 255 && ${ip_array[3]} -le 255 ]] + stat=$? + fi + return $stat +} + +######################## +# Validate a string format +# Arguments: +# $1 - String to validate +# Returns: +# Boolean +######################### +validate_string() { + local string + local min_length=-1 + local max_length=-1 + + # Parse flags + while [ "$#" -gt 0 ]; do + case "$1" in + -min-length) + shift + min_length=${1:-} + ;; + -max-length) + shift + max_length=${1:-} + ;; + --) + shift + break + ;; + -*) + stderr_print "unrecognized flag $1" + return 1 + ;; + *) + break + ;; + esac + shift + done + + if [ "$#" -gt 1 ]; then + stderr_print "too many arguments provided" + return 2 + elif [ "$#" -eq 0 ]; then + stderr_print "missing string" + return 1 + else + string=$1 + fi + + if [[ "$min_length" -ge 0 ]] && [[ "${#string}" -lt "$min_length" ]]; then + echo "string length is less than $min_length" + return 1 + fi + if [[ "$max_length" -ge 0 ]] && [[ "${#string}" -gt "$max_length" ]]; then + echo "string length is great than $max_length" + return 1 + fi +} diff --git a/prebuildfs/opt/bitnami/scripts/libversion.sh b/prebuildfs/opt/bitnami/scripts/libversion.sh new file mode 100644 index 0000000..66d4f45 --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libversion.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Library for managing versions strings + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /opt/bitnami/scripts/liblog.sh + +# Functions +######################## +# Gets semantic version +# Arguments: +# $1 - version: string to extract major.minor.patch +# $2 - section: 1 to extract major, 2 to extract minor, 3 to extract patch +# Returns: +# array with the major, minor and release +######################### +get_sematic_version () { + local version="${1:?version is required}" + local section="${2:?section is required}" + local -a version_sections + + #Regex to parse versions: x.y.z + local -r regex='([0-9]+)(\.([0-9]+)(\.([0-9]+))?)?' + + if [[ "$version" =~ $regex ]]; then + local i=1 + local j=1 + local n=${#BASH_REMATCH[*]} + + while [[ $i -lt $n ]]; do + if [[ -n "${BASH_REMATCH[$i]}" ]] && [[ "${BASH_REMATCH[$i]:0:1}" != '.' ]]; then + version_sections[$j]=${BASH_REMATCH[$i]} + ((j++)) + fi + ((i++)) + done + + local number_regex='^[0-9]+$' + if [[ "$section" =~ $number_regex ]] && (( $section > 0 )) && (( $section <= 3 )); then + echo "${version_sections[$section]}" + return + else + stderr_print "Section allowed values are: 1, 2, and 3" + return 1 + fi + fi +} diff --git a/prebuildfs/opt/bitnami/scripts/libwebserver.sh b/prebuildfs/opt/bitnami/scripts/libwebserver.sh new file mode 100644 index 0000000..25bff4a --- /dev/null +++ b/prebuildfs/opt/bitnami/scripts/libwebserver.sh @@ -0,0 +1,421 @@ +#!/bin/bash +# +# Bitnami web server handler library + +# shellcheck disable=SC1090,SC1091 + +# Load generic libraries +. /opt/bitnami/scripts/liblog.sh + +######################## +# Execute a command (or list of commands) with the web server environment and library loaded +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_execute() { + local -r web_server="${1:?missing web server}" + shift + # Run program in sub-shell to avoid web server environment getting loaded when not necessary + ( + . "/opt/bitnami/scripts/lib${web_server}.sh" + . "/opt/bitnami/scripts/${web_server}-env.sh" + "$@" + ) +} + +######################## +# Prints the list of enabled web servers +# Globals: +# None +# Arguments: +# None +# Returns: +# None +######################### +web_server_list() { + local -r -a supported_web_servers=(apache nginx) + local -a existing_web_servers=() + for web_server in "${supported_web_servers[@]}"; do + [[ -f "/opt/bitnami/scripts/${web_server}-env.sh" ]] && existing_web_servers+=("$web_server") + done + echo "${existing_web_servers[@]:-}" +} + +######################## +# Prints the currently-enabled web server type (only one, in order of preference) +# Globals: +# None +# Arguments: +# None +# Returns: +# None +######################### +web_server_type() { + local -a web_servers + read -r -a web_servers <<< "$(web_server_list)" + echo "${web_servers[0]:-}" +} + +######################## +# Validate that a supported web server is configured +# Globals: +# None +# Arguments: +# None +# Returns: +# None +######################### +web_server_validate() { + local error_code=0 + local supported_web_servers=("apache" "nginx") + + # Auxiliary functions + print_validation_error() { + error "$1" + error_code=1 + } + + if [[ -z "$(web_server_type)" || ! " ${supported_web_servers[*]} " == *" $(web_server_type) "* ]]; then + print_validation_error "Could not detect any supported web servers. It must be one of: ${supported_web_servers[*]}" + elif ! web_server_execute "$(web_server_type)" type -t "is_$(web_server_type)_running" >/dev/null; then + print_validation_error "Could not load the $(web_server_type) web server library from /opt/bitnami/scripts. Check that it exists and is readable." + fi + + return "$error_code" +} + +######################## +# Check whether the web server is running +# Globals: +# * +# Arguments: +# None +# Returns: +# true if the web server is running, false otherwise +######################### +is_web_server_running() { + "is_$(web_server_type)_running" +} + +######################## +# Start web server +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_start() { + info "Starting $(web_server_type) in background" + "${BITNAMI_ROOT_DIR}/scripts/$(web_server_type)/start.sh" +} + +######################## +# Stop web server +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_stop() { + info "Stopping $(web_server_type)" + "${BITNAMI_ROOT_DIR}/scripts/$(web_server_type)/stop.sh" +} + +######################## +# Restart web server +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_restart() { + info "Restarting $(web_server_type)" + "${BITNAMI_ROOT_DIR}/scripts/$(web_server_type)/restart.sh" +} + +######################## +# Reload web server +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_reload() { + "${BITNAMI_ROOT_DIR}/scripts/$(web_server_type)/reload.sh" +} + +######################## +# Ensure a web server application configuration exists (i.e. Apache virtual host format or NGINX server block) +# It serves as a wrapper for the specific web server function +# Globals: +# * +# Arguments: +# $1 - App name +# Flags: +# --hosts - Hosts to enable +# --type - Application type, which has an effect on which configuration template to use +# --allow-remote-connections - Whether to allow remote connections or to require local connections +# --disabled - Whether to render the file with a .disabled prefix +# --enable-https - Enable app configuration on HTTPS port +# --http-port - HTTP port number +# --https-port - HTTPS port number +# --document-root - Path to document root directory +# Apache-specific flags: +# --apache-additional-configuration - Additional vhost configuration (no default) +# --apache-before-vhost-configuration - Configuration to add before the directive (no default) +# --apache-allow-override - Whether to allow .htaccess files (only allowed when --move-htaccess is set to 'no') +# --apache-extra-directory-configuration - Extra configuration for the document root directory +# --apache-move-htaccess - Move .htaccess files to a common place so they can be loaded during Apache startup +# NGINX-specific flags: +# --nginx-additional-configuration - Additional server block configuration (no default) +# Returns: +# true if the configuration was enabled, false otherwise +######################## +ensure_web_server_app_configuration_exists() { + local app="${1:?missing app}" + shift + local -a apache_args nginx_args web_servers args_var + apache_args=("$app") + nginx_args=("$app") + # Validate arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + # Common flags + --hosts \ + | --type \ + | --allow-remote-connections \ + | --disabled \ + | --enable-https \ + | --http-port \ + | --https-port \ + | --document-root \ + ) + apache_args+=("$1" "${2:?missing value}") + nginx_args+=("$1" "${2:?missing value}") + shift + ;; + + # Specific Apache flags + --apache-additional-configuration \ + | --apache-before-vhost-configuration \ + | --apache-allow-override \ + | --apache-extra-directory-configuration \ + | --apache-move-htaccess \ + ) + apache_args+=("${1//apache-/}" "${2:?missing value}") + shift + ;; + + # Specific NGINX flags + --nginx-additional-configuration) + nginx_args+=("${1//nginx-/}" "${2:?missing value}") + shift + ;; + + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + read -r -a web_servers <<< "$(web_server_list)" + for web_server in "${web_servers[@]}"; do + args_var="${web_server}_args[@]" + web_server_execute "$web_server" "ensure_${web_server}_app_configuration_exists" "${!args_var}" + done +} + +######################## +# Ensure a web server application configuration does not exist anymore (i.e. Apache virtual host format or NGINX server block) +# It serves as a wrapper for the specific web server function +# Globals: +# * +# Arguments: +# $1 - App name +# Returns: +# true if the configuration was disabled, false otherwise +######################## +ensure_web_server_app_configuration_not_exists() { + local app="${1:?missing app}" + local -a web_servers + read -r -a web_servers <<< "$(web_server_list)" + for web_server in "${web_servers[@]}"; do + web_server_execute "$web_server" "ensure_${web_server}_app_configuration_not_exists" "$app" + done +} + +######################## +# Ensure the web server loads the configuration for an application in a URL prefix +# It serves as a wrapper for the specific web server function +# Globals: +# * +# Arguments: +# $1 - App name +# Flags: +# --allow-remote-connections - Whether to allow remote connections or to require local connections +# --document-root - Path to document root directory +# --prefix - URL prefix from where it will be accessible (i.e. /myapp) +# --type - Application type, which has an effect on what configuration template will be used +# Apache-specific flags: +# --apache-additional-configuration - Additional vhost configuration (no default) +# --apache-allow-override - Whether to allow .htaccess files (only allowed when --move-htaccess is set to 'no') +# --apache-extra-directory-configuration - Extra configuration for the document root directory +# --apache-move-htaccess - Move .htaccess files to a common place so they can be loaded during Apache startup +# NGINX-specific flags: +# --nginx-additional-configuration - Additional server block configuration (no default) +# Returns: +# true if the configuration was enabled, false otherwise +######################## +ensure_web_server_prefix_configuration_exists() { + local app="${1:?missing app}" + shift + local -a apache_args nginx_args web_servers args_var + apache_args=("$app") + nginx_args=("$app") + # Validate arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + # Common flags + --allow-remote-connections \ + | --document-root \ + | --prefix \ + | --type \ + ) + apache_args+=("$1" "${2:?missing value}") + nginx_args+=("$1" "${2:?missing value}") + shift + ;; + + # Specific Apache flags + --apache-additional-configuration \ + | --apache-allow-override \ + | --apache-extra-directory-configuration \ + | --apache-move-htaccess \ + ) + apache_args+=("${1//apache-/}" "$2") + shift + ;; + + # Specific NGINX flags + --nginx-additional-configuration) + nginx_args+=("${1//nginx-/}" "$2") + shift + ;; + + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + read -r -a web_servers <<< "$(web_server_list)" + for web_server in "${web_servers[@]}"; do + args_var="${web_server}_args[@]" + web_server_execute "$web_server" "ensure_${web_server}_prefix_configuration_exists" "${!args_var}" + done +} + +######################## +# Ensure a web server application configuration is updated with the runtime configuration (i.e. ports) +# It serves as a wrapper for the specific web server function +# Globals: +# * +# Arguments: +# $1 - App name +# Flags: +# --hosts - Hosts to enable +# --enable-https - Update HTTPS app configuration +# --http-port - HTTP port number +# --https-port - HTTPS port number +# Returns: +# true if the configuration was updated, false otherwise +######################## +web_server_update_app_configuration() { + local app="${1:?missing app}" + shift + local -a args web_servers + args=("$app") + # Validate arguments + while [[ "$#" -gt 0 ]]; do + case "$1" in + # Common flags + --hosts \ + | --enable-https \ + | --http-port \ + | --https-port \ + ) + args+=("$1" "${2:?missing value}") + shift + ;; + + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + read -r -a web_servers <<< "$(web_server_list)" + for web_server in "${web_servers[@]}"; do + web_server_execute "$web_server" "${web_server}_update_app_configuration" "${args[@]}" + done +} + +######################## +# Enable loading page, which shows users that the initialization process is not yet completed +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_enable_loading_page() { + ensure_web_server_app_configuration_exists "__loading" --hosts "_default_" \ + --apache-additional-configuration " +# Show a HTTP 503 Service Unavailable page by default +RedirectMatch 503 ^/$ +# Show index.html if server is answering with 404 Not Found or 503 Service Unavailable status codes +ErrorDocument 404 /index.html +ErrorDocument 503 /index.html" \ + --nginx-additional-configuration " +# Show a HTTP 503 Service Unavailable page by default +location / { + return 503; +} +# Show index.html if server is answering with 404 Not Found or 503 Service Unavailable status codes +error_page 404 @installing; +error_page 503 @installing; +location @installing { + rewrite ^(.*)$ /index.html break; +}" + web_server_reload +} + +######################## +# Enable loading page, which shows users that the initialization process is not yet completed +# Globals: +# * +# Arguments: +# None +# Returns: +# None +######################### +web_server_disable_install_page() { + ensure_web_server_app_configuration_not_exists "__loading" + web_server_reload +} diff --git a/prebuildfs/usr/sbin/install_packages b/prebuildfs/usr/sbin/install_packages new file mode 100755 index 0000000..c957764 --- /dev/null +++ b/prebuildfs/usr/sbin/install_packages @@ -0,0 +1,24 @@ +#!/bin/sh +set -e +set -u +export DEBIAN_FRONTEND=noninteractive +n=0 +max=2 +until [ $n -gt $max ]; do + set +e + ( + apt-get update -qq && + apt-get install -y --no-install-recommends "$@" + ) + CODE=$? + set -e + if [ $CODE -eq 0 ]; then + break + fi + if [ $n -eq $max ]; then + exit $CODE + fi + echo "apt failed, retrying" + n=$(($n + 1)) +done +rm -r /var/lib/apt/lists /var/cache/apt/archives diff --git a/rootfs/opt/bitnami/scripts/airflow-env.sh b/rootfs/opt/bitnami/scripts/airflow-env.sh new file mode 100644 index 0000000..c7e4caa --- /dev/null +++ b/rootfs/opt/bitnami/scripts/airflow-env.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Environment configuration for airflow + +# The values for all environment variables will be set in the below order of precedence +# 1. Custom environment variables defined below after Bitnami defaults +# 2. Constants defined in this file (environment variables with no default), i.e. BITNAMI_ROOT_DIR +# 3. Environment variables overridden via external files using *_FILE variables (see below) +# 4. Environment variables set externally (i.e. current Bash context/Dockerfile/userdata) + +# Load logging library +. /opt/bitnami/scripts/liblog.sh + +export BITNAMI_ROOT_DIR="/opt/bitnami" +export BITNAMI_VOLUME_DIR="/bitnami" + +# Logging configuration +export MODULE="${MODULE:-airflow}" +export BITNAMI_DEBUG="${BITNAMI_DEBUG:-false}" + +# By setting an environment variable matching *_FILE to a file path, the prefixed environment +# variable will be overridden with the value specified in that file +airflow_env_vars=( + AIRFLOW_USERNAME + AIRFLOW_PASSWORD + AIRFLOW_FIRSTNAME + AIRFLOW_LASTNAME + AIRFLOW_EMAIL + AIRFLOW_EXECUTOR + AIRFLOW_FERNET_KEY + AIRFLOW_WEBSERVER_HOST + AIRFLOW_WEBSERVER_PORT_NUMBER + AIRFLOW_LOAD_EXAMPLES + AIRFLOW_BASE_URL + AIRFLOW_HOSTNAME_CALLABLE + AIRFLOW_POOL_NAME + AIRFLOW_POOL_SIZE + AIRFLOW_POOL_DESC + AIRFLOW_DATABASE_HOST + AIRFLOW_DATABASE_PORT_NUMBER + AIRFLOW_DATABASE_NAME + AIRFLOW_DATABASE_USERNAME + AIRFLOW_DATABASE_PASSWORD + AIRFLOW_DATABASE_USE_SSL + AIRFLOW_REDIS_USE_SSL + REDIS_HOST + REDIS_PORT_NUMBER + REDIS_USER + REDIS_PASSWORD + AIRFLOW_LDAP_ENABLE + AIRFLOW_LDAP_URI + AIRFLOW_LDAP_SEARCH + AIRFLOW_LDAP_BIND_USER + AIRFLOW_LDAP_BIND_PASSWORD + AIRFLOW_LDAP_UID_FIELD + AIRFLOW_LDAP_USE_TLS + AIRFLOW_LDAP_ALLOW_SELF_SIGNED + AIRFLOW_LDAP_TLS_CA_CERTIFICATE + AIRFLOW_USER_REGISTRATION_ROLE +) +for env_var in "${airflow_env_vars[@]}"; do + file_env_var="${env_var}_FILE" + if [[ -n "${!file_env_var:-}" ]]; then + if [[ -r "${!file_env_var:-}" ]]; then + export "${env_var}=$(< "${!file_env_var}")" + unset "${file_env_var}" + else + warn "Skipping export of '${env_var}'. '${!file_env_var:-}' is not readable." + fi + fi +done +unset airflow_env_vars + +# Airflow paths +export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow" +export AIRFLOW_HOME="${AIRFLOW_BASE_DIR}" +export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin" +export AIRFLOW_VOLUME_DIR="/bitnami/airflow" +export AIRFLOW_DATA_DIR="${AIRFLOW_BASE_DIR}/data" +export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs" +export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-webserver.log" +export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg" +export AIRFLOW_WEBSERVER_CONF_FILE="${AIRFLOW_BASE_DIR}/webserver_config.py" +export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp" +export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-webserver.pid" +export AIRFLOW_DATA_TO_PERSIST="$AIRFLOW_DATA_DIR" +export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags" + +# System users (when running with a privileged user) +export AIRFLOW_DAEMON_USER="airflow" +export AIRFLOW_DAEMON_GROUP="airflow" + +# User configuration +export AIRFLOW_USERNAME="${AIRFLOW_USERNAME:-user}" +export AIRFLOW_PASSWORD="${AIRFLOW_PASSWORD:-bitnami}" +export AIRFLOW_FIRSTNAME="${AIRFLOW_FIRSTNAME:-Firstname}" +export AIRFLOW_LASTNAME="${AIRFLOW_LASTNAME:-Lastname}" +export AIRFLOW_EMAIL="${AIRFLOW_EMAIL:-user@example.com}" + +# Airflow configuration +export AIRFLOW_EXECUTOR="${AIRFLOW_EXECUTOR:-SequentialExecutor}" +export AIRFLOW_FERNET_KEY="${AIRFLOW_FERNET_KEY:-}" +export AIRFLOW_WEBSERVER_HOST="${AIRFLOW_WEBSERVER_HOST:-127.0.0.1}" +export AIRFLOW_WEBSERVER_PORT_NUMBER="${AIRFLOW_WEBSERVER_PORT_NUMBER:-8080}" +export AIRFLOW_LOAD_EXAMPLES="${AIRFLOW_LOAD_EXAMPLES:-yes}" +export AIRFLOW_BASE_URL="${AIRFLOW_BASE_URL:-}" +export AIRFLOW_HOSTNAME_CALLABLE="${AIRFLOW_HOSTNAME_CALLABLE:-}" +export AIRFLOW_POOL_NAME="${AIRFLOW_POOL_NAME:-}" +export AIRFLOW_POOL_SIZE="${AIRFLOW_POOL_SIZE:-}" +export AIRFLOW_POOL_DESC="${AIRFLOW_POOL_DESC:-}" + +# Airflow database configuration +export AIRFLOW_DATABASE_HOST="${AIRFLOW_DATABASE_HOST:-postgresql}" +export AIRFLOW_DATABASE_PORT_NUMBER="${AIRFLOW_DATABASE_PORT_NUMBER:-5432}" +export AIRFLOW_DATABASE_NAME="${AIRFLOW_DATABASE_NAME:-bitnami_airflow}" +export AIRFLOW_DATABASE_USERNAME="${AIRFLOW_DATABASE_USERNAME:-bn_airflow}" +export AIRFLOW_DATABASE_PASSWORD="${AIRFLOW_DATABASE_PASSWORD:-}" +export AIRFLOW_DATABASE_USE_SSL="${AIRFLOW_DATABASE_USE_SSL:-no}" +export AIRFLOW_REDIS_USE_SSL="${AIRFLOW_REDIS_USE_SSL:-no}" +export REDIS_HOST="${REDIS_HOST:-redis}" +export REDIS_PORT_NUMBER="${REDIS_PORT_NUMBER:-6379}" +export REDIS_USER="${REDIS_USER:-}" +export REDIS_PASSWORD="${REDIS_PASSWORD:-}" + +# Airflow LDAP configuration +export AIRFLOW_LDAP_ENABLE="${AIRFLOW_LDAP_ENABLE:-no}" +export AIRFLOW_LDAP_URI="${AIRFLOW_LDAP_URI:-}" +export AIRFLOW_LDAP_SEARCH="${AIRFLOW_LDAP_SEARCH:-}" +export AIRFLOW_LDAP_BIND_USER="${AIRFLOW_LDAP_BIND_USER:-}" +export AIRFLOW_LDAP_BIND_PASSWORD="${AIRFLOW_LDAP_BIND_PASSWORD:-}" +export AIRFLOW_LDAP_UID_FIELD="${AIRFLOW_LDAP_UID_FIELD:-}" +export AIRFLOW_LDAP_USE_TLS="${AIRFLOW_LDAP_USE_TLS:-False}" +export AIRFLOW_LDAP_ALLOW_SELF_SIGNED="${AIRFLOW_LDAP_ALLOW_SELF_SIGNED:-True}" +export AIRFLOW_LDAP_TLS_CA_CERTIFICATE="${AIRFLOW_LDAP_TLS_CA_CERTIFICATE:-}" +export AIRFLOW_USER_REGISTRATION_ROLE="${AIRFLOW_USER_REGISTRATION_ROLE:-Public}" + +# Custom environment variables may be defined below diff --git a/rootfs/opt/bitnami/scripts/airflow/entrypoint.sh b/rootfs/opt/bitnami/scripts/airflow/entrypoint.sh new file mode 100755 index 0000000..c808f10 --- /dev/null +++ b/rootfs/opt/bitnami/scripts/airflow/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# shellcheck disable=SC1091 + +set -o errexit +set -o nounset +set -o pipefail +# set -o xtrace # Uncomment this line for debugging purpose + +# Load Airflow environment variables +. /opt/bitnami/scripts/airflow-env.sh + +# Load libraries +. /opt/bitnami/scripts/libbitnami.sh +. /opt/bitnami/scripts/libairflow.sh + +print_welcome_page + +if [[ "$*" = *"/opt/bitnami/scripts/airflow/run.sh"* || "$*" = *"/run.sh"* ]]; then + info "** Starting Airflow setup **" + /opt/bitnami/scripts/airflow/setup.sh + info "** Airflow setup finished! **" +fi + +echo "" +exec "$@" diff --git a/rootfs/opt/bitnami/scripts/airflow/postunpack.sh b/rootfs/opt/bitnami/scripts/airflow/postunpack.sh new file mode 100755 index 0000000..61cd4c3 --- /dev/null +++ b/rootfs/opt/bitnami/scripts/airflow/postunpack.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# shellcheck disable=SC1091 + +set -o errexit +set -o nounset +set -o pipefail +# set -o xtrace # Uncomment this line for debugging purpose + +# Load Airflow environment variables +. /opt/bitnami/scripts/airflow-env.sh + +# Load libraries +. /opt/bitnami/scripts/libairflow.sh +. /opt/bitnami/scripts/libfs.sh +. /opt/bitnami/scripts/libos.sh + +for dir in "$AIRFLOW_VOLUME_DIR" "$AIRFLOW_BASE_DIR" "$AIRFLOW_DATA_DIR"; do + ensure_dir_exists "$dir" +done + +# Ensure the needed directories exist with write permissions +for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do + ensure_dir_exists "$dir" + configure_permissions_ownership "$dir" -d "775" -f "664" -g "root" +done + +chmod -R g+rwX /bitnami "$AIRFLOW_VOLUME_DIR" "$AIRFLOW_BASE_DIR" diff --git a/rootfs/opt/bitnami/scripts/airflow/run.sh b/rootfs/opt/bitnami/scripts/airflow/run.sh new file mode 100755 index 0000000..00a2bd0 --- /dev/null +++ b/rootfs/opt/bitnami/scripts/airflow/run.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# shellcheck disable=SC1091 + +set -o errexit +set -o nounset +set -o pipefail +# set -o xtrace # Uncomment this line for debugging purposes + +# Load Airflow environment variables +. /opt/bitnami/scripts/airflow-env.sh + +# Load libraries +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/libairflow.sh + +args=("--pid" "$AIRFLOW_PID_FILE" "$@") + +info "** Starting Airflow **" +if am_i_root; then + exec gosu "$AIRFLOW_DAEMON_USER" "${AIRFLOW_BIN_DIR}/airflow" "webserver" "${args[@]}" +else + exec "${AIRFLOW_BIN_DIR}/airflow" "webserver" "${args[@]}" +fi diff --git a/rootfs/opt/bitnami/scripts/airflow/setup.sh b/rootfs/opt/bitnami/scripts/airflow/setup.sh new file mode 100755 index 0000000..758d2e8 --- /dev/null +++ b/rootfs/opt/bitnami/scripts/airflow/setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# shellcheck disable=SC1091 + +set -o errexit +set -o nounset +set -o pipefail +# set -o xtrace # Uncomment this line for debugging purposes + +# Load Airflow environment variables +. /opt/bitnami/scripts/airflow-env.sh + +# Load libraries +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/libfs.sh +. /opt/bitnami/scripts/libairflow.sh + +# Ensure Airflow environment variables settings are valid +airflow_validate +# Ensure Airflow daemon user exists when running as root +am_i_root && ensure_user_exists "$AIRFLOW_DAEMON_USER" --group "$AIRFLOW_DAEMON_GROUP" +# Ensure Airflow is initialized +airflow_initialize diff --git a/rootfs/opt/bitnami/scripts/libairflow.sh b/rootfs/opt/bitnami/scripts/libairflow.sh new file mode 100644 index 0000000..7a6cde9 --- /dev/null +++ b/rootfs/opt/bitnami/scripts/libairflow.sh @@ -0,0 +1,432 @@ +#!/bin/bash + +# Bitnami Airflow library + +# shellcheck disable=SC1091 +# shellcheck disable=SC2153 + +# Load Generic Libraries +. /opt/bitnami/scripts/libfile.sh +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libnet.sh +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/libservice.sh +. /opt/bitnami/scripts/libvalidations.sh +. /opt/bitnami/scripts/libpersistence.sh + +# Functions + +######################## +# Validate Airflow inputs +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_validate() { + # Check postgresql host + [[ -z "$AIRFLOW_DATABASE_HOST" ]] && print_validation_error "Missing AIRFLOW_DATABASE_HOST" + + # Check LDAP parameters + if is_boolean_yes "$AIRFLOW_LDAP_ENABLE"; then + [[ -z "$AIRFLOW_LDAP_URI" ]] && print_validation_error "Missing AIRFLOW_LDAP_URI" + [[ -z "$AIRFLOW_LDAP_SEARCH" ]] && print_validation_error "Missing AIRFLOW_LDAP_SEARCH" + [[ -z "$AIRFLOW_LDAP_BIND_USER" ]] && print_validation_error "Missing AIRFLOW_LDAP_BIND_USER" + [[ -z "$AIRFLOW_LDAP_UID_FIELD" ]] && print_validation_error "Missing AIRFLOW_LDAP_UID_FIELD" + [[ -z "$AIRFLOW_LDAP_BIND_PASSWORD" ]] && print_validation_error "Missing AIRFLOW_LDAP_BIND_PASSWORD" + if [[ "$AIRFLOW_LDAP_USE_TLS" == "True" ]]; then + [[ -z "$AIRFLOW_LDAP_TLS_CA_CERTIFICATE" ]] && print_validation_error "Missing AIRFLOW_LDAP_TLS_CA_CERTIFICATE" + fi + fi + + # Check pool parameters + if [[ -n "$AIRFLOW_POOL_NAME" ]]; then + [[ -z "$AIRFLOW_POOL_DESC" ]] && print_validation_error "Provided AIRFLOW_POOL_NAME but missing AIRFLOW_POOL_DESC" + [[ -z "$AIRFLOW_POOL_SIZE" ]] && print_validation_error "Provided AIRFLOW_POOL_NAME but missing AIRFLOW_POOL_SIZE" + fi +} + +######################## +# Ensure Airflow is initialized +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_initialize() { + info "Initializing Airflow ..." + + # Change permissions if running as root + for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DAGS_DIR"; do + ensure_dir_exists "$dir" + am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir" + done + + # The configuration file is not persisted. If it is not provided, generate it based on env vars + if [[ ! -f "$AIRFLOW_CONF_FILE" ]]; then + info "No injected configuration file found. Creating default config file" + airflow_generate_config + else + info "Configuration file found, loading configuration" + fi + + # Check if Airflow has already been initialized and persisted in a previous run + local -r app_name="airflow" + if ! is_app_initialized "$app_name"; then + # Delete pid file + rm -f "$AIRFLOW_PID_FILE" + + airflow_wait_for_postgresql "$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" + + # Initialize database + airflow_execute_command "initdb" "db init" + + airflow_create_admin_user + + airflow_create_pool + + info "Persisting Airflow installation" + persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST" + else + # Check database connection + airflow_wait_for_postgresql "$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" + + # Restore persisted data + info "Restoring persisted Airflow installation" + restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST" + + # Upgrade database + airflow_execute_command "upgradedb" "db upgrade" + + # Change the permissions after restoring the persisted data in case we are root + for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do + ensure_dir_exists "$dir" + am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir" + done + true # Avoid return false when I am not root + fi +} + +######################## +# Executes airflow command +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_execute_command() { + local oldCommand="${1?Missing old command}" + local newCommand="${2?Missing new command}" + local flags="${3:-}" + + # The commands can contain more than one argument. Convert them to an array + IFS=' ' read -ra oldCommand <<< "$oldCommand" + IFS=' ' read -ra newCommand <<< "$newCommand" + + # Execute commands depending on the version + command=("${oldCommand[@]}") + [[ "${BITNAMI_IMAGE_VERSION:0:1}" == "2" ]] && command=("${newCommand[@]}") + + # Add flags if provided + [[ -n "$flags" ]] && IFS=' ' read -ra flags <<< "$flags" && command+=("${flags[@]}") + + debug "Executing ${AIRFLOW_BIN_DIR}/airflow ${command[*]}" + debug_execute "${AIRFLOW_BIN_DIR}/airflow" "${command[@]}" +} + +######################## +# Generate Airflow conf file +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_generate_config() { + # Generate Airflow default files + airflow_execute_command "version" "version" + + # Setup Airflow base URL + airflow_configure_base_url + # Configure Airflow Hostname + [[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE" + # Configure Airflow webserver authentication + airflow_configure_webserver_authentication + # Configure Airflow to load examples + if is_boolean_yes "$AIRFLOW_LOAD_EXAMPLES"; then + airflow_conf_set "core" "load_examples" "True" + else + airflow_conf_set "core" "load_examples" "False" + fi + # Configure Airflow database + airflow_configure_database + # Configure the Webserver port + airflow_conf_set "webserver" "web_server_port" "$AIRFLOW_WEBSERVER_PORT_NUMBER" + # Setup fernet key + [[ -n "$AIRFLOW_FERNET_KEY" ]] && airflow_conf_set "core" "fernet_key" "$AIRFLOW_FERNET_KEY" + # Configure Airflow executor + airflow_conf_set "core" "executor" "$AIRFLOW_EXECUTOR" + [[ "$AIRFLOW_EXECUTOR" == "CeleryExecutor" ]] && airflow_configure_celery_executor + true # Avoid the function to fail due to the check above +} + +######################## +# Set property on the Airflow configuration file +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_conf_set() { + local -r section="${1:?section is required}" + local -r key="${2:?key is required}" + local -r value="${3:?value is required}" + local -r file="${4:-${AIRFLOW_CONF_FILE}}" + + ini-file set --section "$section" --key "$key" --value "$value" "$file" +} + +######################## +# Configure Airflow base url +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_configure_base_url() { + if [[ -z "$AIRFLOW_BASE_URL" ]]; then + airflow_conf_set "webserver" "base_url" "http://${AIRFLOW_WEBSERVER_HOST}:${AIRFLOW_WEBSERVER_PORT_NUMBER}" + else + airflow_conf_set "webserver" "base_url" "$AIRFLOW_BASE_URL" + fi +} + +######################## +# Configure Airflow webserver authentication +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_configure_webserver_authentication() { + info "Configuring Airflow webserver authentication" + airflow_conf_set "webserver" "rbac" "true" + + if is_boolean_yes "$AIRFLOW_LDAP_ENABLE"; then + info "Enabling LDAP authentication" + airflow_webserver_conf_set "AUTH_USER_REGISTRATION" "True", + airflow_webserver_conf_set "AUTH_TYPE" "AUTH_LDAP", + airflow_webserver_conf_set "AUTH_LDAP_SERVER" "$AIRFLOW_LDAP_URI", + airflow_webserver_conf_set "AUTH_LDAP_SEARCH" "$AIRFLOW_LDAP_SEARCH", + airflow_webserver_conf_set "AUTH_LDAP_BIND_USER" "$AIRFLOW_LDAP_BIND_USER", + airflow_webserver_conf_set "AUTH_LDAP_BIND_PASSWORD" "$AIRFLOW_LDAP_BIND_PASSWORD", + airflow_webserver_conf_set "AUTH_LDAP_UID_FIELD" "$AIRFLOW_LDAP_UID_FIELD", + airflow_webserver_conf_set "AUTH_LDAP_USE_TLS" "$AIRFLOW_LDAP_USE_TLS", + airflow_webserver_conf_set "AUTH_LDAP_ALLOW_SELF_SIGNED" "$AIRFLOW_LDAP_ALLOW_SELF_SIGNED", + airflow_webserver_conf_set "AUTH_LDAP_TLS_CACERTFILE" "$AIRFLOW_LDAP_TLS_CA_CERTIFICATE", + airflow_webserver_conf_set "AUTH_USER_REGISTRATION_ROLE" "$AIRFLOW_USER_REGISTRATION_ROLE", + fi +} + +######################## +# Set properties in Airflow's webserver_config.py +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_webserver_conf_set() { + local -r key="${1:?missing key}" + local -r value="${2:?missing key}" + shift 2 + + local -r file="$AIRFLOW_WEBSERVER_CONF_FILE" + # Check if the value was set before + if grep -q "^#*\\s*${key}=.*$" "$file"; then + # Update the existing key + replace_in_file "$file" "^#*\\s*${key}=.*$" "${key} = \"${value}\"" false + else + # Add a new key + printf '\n%s="%s"' "$key" "$value" >>"$file" + fi +} + +######################## +# Configure Airflow database +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_configure_database() { + local -r user=$(airflow_encode_url "$AIRFLOW_DATABASE_USERNAME") + local -r password=$(airflow_encode_url "$AIRFLOW_DATABASE_PASSWORD") + local extra_options + is_boolean_yes "$AIRFLOW_REDIS_USE_SSL" && extra_options="?sslmode=require" + + info "Configuring Airflow database" + airflow_conf_set "core" "sql_alchemy_conn" "postgresql+psycopg2://${user}:${password}@${AIRFLOW_DATABASE_HOST}:${AIRFLOW_DATABASE_PORT_NUMBER}/${AIRFLOW_DATABASE_NAME}${extra_options:-}" +} + +######################## +# Return URL encoded string in the airflow conf format +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_encode_url() { + local -r url="${1?Missing url}" + + urlencode() { + old_lc_collate="${LC_COLLATE:-}" + LC_COLLATE=C + + local length="${#1}" + for (( i = 0; i < length; i++ )); do + local c="${1:$i:1}" + case $c in + [a-zA-Z0-9.~_-]) printf '%s' "$c" ;; + *) printf '%%%02X' "'$c" ;; + esac + done + + LC_COLLATE="$old_lc_collate" + } + + local -r url_encoded=$(urlencode "$url") + # Replace % by %% + echo "${url_encoded//\%/\%\%}" +} + +######################## +# Configure Airflow celery executor +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_configure_celery_executor() { + info "Configuring Celery Executor" + + # Configure celery Redis url + local -r redis_user=$(airflow_encode_url "$REDIS_USER") + local -r redis_password=$(airflow_encode_url "$REDIS_PASSWORD") + airflow_conf_set "celery" "broker_url" "redis://${redis_user}:${redis_password}@${REDIS_HOST}:${REDIS_PORT_NUMBER}/1" + is_boolean_yes "$AIRFLOW_REDIS_USE_SSL" && airflow_conf_set "celery" "redis_backend_use_ssl" "true" + + # Configure celery backend + local -r database_user=$(airflow_encode_url "$AIRFLOW_DATABASE_USERNAME") + local -r database_password=$(airflow_encode_url "$AIRFLOW_DATABASE_PASSWORD") + local database_extra_options + is_boolean_yes "$AIRFLOW_REDIS_USE_SSL" && database_extra_options="?sslmode=require" + airflow_conf_set "celery" "result_backend" "db+postgresql://${database_user}:${database_password}@${AIRFLOW_DATABASE_HOST}:${AIRFLOW_DATABASE_PORT_NUMBER}/${AIRFLOW_DATABASE_NAME}${database_extra_options:-}" +} + +######################## +# Wait for PostgreSQL +# Arguments: +# None +# Returns: +# None +######################### +airflow_wait_for_postgresql() { + local -r postgresql_host="${1?Missing host}" + local -r postgresql_port="${2?Missing port}" + + info "Waiting for PostgreSQL to be available at ${postgresql_host}:${postgresql_port}..." + wait-for-port --host "$postgresql_host" "$postgresql_port" +} + +######################## +# Airflow create admin user +# Arguments: +# None +# Returns: +# None +######################### +airflow_create_admin_user() { + info "Creating Airflow admin user" + airflow_execute_command "create_user" "users create" "-r Admin -u ${AIRFLOW_USERNAME} -e ${AIRFLOW_EMAIL} -p ${AIRFLOW_PASSWORD} -f ${AIRFLOW_FIRSTNAME} -l ${AIRFLOW_LASTNAME}" +} + +######################## +# Airflow create pool +# Arguments: +# None +# Returns: +# None +######################### +airflow_create_pool() { + if [[ -n "$AIRFLOW_POOL_NAME" ]] && [[ -n "$AIRFLOW_POOL_SIZE" ]] && [[ -n "$AIRFLOW_POOL_DESC" ]]; then + info "Creating Airflow pool" + airflow_execute_command "pool" "pool" "-s ${AIRFLOW_POOL_NAME} ${AIRFLOW_POOL_SIZE} ${AIRFLOW_POOL_DESC}" + fi +} + +######################## +# Check if Airflow is running +# Globals: +# AIRFLOW_PID_FILE +# Arguments: +# None +# Returns: +# Whether Airflow is running +######################## +is_airflow_running() { + local pid + pid="$(get_pid_from_file "$AIRFLOW_PID_FILE")" + if [[ -n "$pid" ]]; then + is_service_running "$pid" + else + false + fi +} + +######################## +# Check if Airflow is running +# Globals: +# AIRFLOW_PID_FILE +# Arguments: +# None +# Returns: +# Whether Airflow is not running +######################## +is_airflow_not_running() { + ! is_airflow_running +} + +######################## +# Stop Airflow +# Globals: +# AIRFLOW* +# Arguments: +# None +# Returns: +# None +######################### +airflow_stop() { + info "Stopping Airflow..." + stop_service_using_pid "$AIRFLOW_PID_FILE" +} \ No newline at end of file diff --git a/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh b/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh new file mode 100755 index 0000000..601a83c --- /dev/null +++ b/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# shellcheck disable=SC1091 + +set -o errexit +set -o nounset +set -o pipefail +# set -o xtrace # Uncomment this line for debugging purpose + +# Defaults +WITH_ALL_LOCALES="${WITH_ALL_LOCALES:-no}" +EXTRA_LOCALES="${EXTRA_LOCALES:-}" + +# Constants +LOCALES_FILE="/etc/locale.gen" +SUPPORTED_LOCALES_FILE="/usr/share/i18n/SUPPORTED" + +# Helper function for enabling locale only when it was not added before +enable_locale() { + local -r locale="${1:?missing locale}" + if ! grep -q -E "^${locale}$" "$SUPPORTED_LOCALES_FILE"; then + echo "Locale ${locale} is not supported in this system" + return 1 + fi + if ! grep -q -E "^${locale}" "$LOCALES_FILE"; then + echo "$locale" >> "$LOCALES_FILE" + else + echo "Locale ${locale} is already enabled" + fi +} + +if [[ "$WITH_ALL_LOCALES" =~ ^(yes|true|1)$ ]]; then + echo "Enabling all locales" + cp "$SUPPORTED_LOCALES_FILE" "$LOCALES_FILE" +else + LOCALES_TO_ADD="$(sed 's/[,;]\s*/\n/g' <<< "$EXTRA_LOCALES")" + while [[ -n "$LOCALES_TO_ADD" ]] && read -r locale; do + echo "Enabling locale ${locale}" + enable_locale "$locale" + done <<< "$LOCALES_TO_ADD" +fi + +locale-gen -- GitLab From 8f3b02cedcca98661a8d810721ad8b11c27bdb6e Mon Sep 17 00:00:00 2001 From: Austin Denton Date: Wed, 7 Apr 2021 10:23:34 -0600 Subject: [PATCH 2/5] Update to R53 to not use nami tool --- hardening_manifest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 876f9e7..806490f 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -37,7 +37,7 @@ labels: # List of resources to make available to the offline build context resources: - tag: bitnami/airflow-worker:2.0.1-debian-10-r53 - url: docker://docker.io/bitnami/airflow@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae38 + url: docker://docker.io/bitnami/airflow-worker@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae389 # List of project maintainers maintainers: -- GitLab From 3d752484323138ea5aef7618dadf474eb7e9210c Mon Sep 17 00:00:00 2001 From: Austin Denton Date: Thu, 8 Apr 2021 10:20:07 -0600 Subject: [PATCH 3/5] Update worker --- Dockerfile | 36 +++-- hardening_manifest.yaml | 30 ++++ .../opt/bitnami/.bitnami_components.json | 6 +- prebuildfs/opt/bitnami/scripts/liblog.sh | 2 + prebuildfs/opt/bitnami/scripts/libversion.sh | 4 +- .../{airflow-env.sh => airflow-worker-env.sh} | 59 +------ .../{airflow => airflow-worker}/entrypoint.sh | 8 +- .../{airflow => airflow-worker}/postunpack.sh | 4 +- .../{airflow => airflow-worker}/run.sh | 8 +- .../{airflow => airflow-worker}/setup.sh | 8 +- .../opt/bitnami/scripts/libairflowworker.sh | 146 ++++++++++++++++++ .../scripts/locales/add-extra-locales.sh | 1 + 12 files changed, 229 insertions(+), 83 deletions(-) rename rootfs/opt/bitnami/scripts/{airflow-env.sh => airflow-worker-env.sh} (61%) rename rootfs/opt/bitnami/scripts/{airflow => airflow-worker}/entrypoint.sh (61%) rename rootfs/opt/bitnami/scripts/{airflow => airflow-worker}/postunpack.sh (88%) rename rootfs/opt/bitnami/scripts/{airflow => airflow-worker}/run.sh (67%) rename rootfs/opt/bitnami/scripts/{airflow => airflow-worker}/setup.sh (78%) create mode 100644 rootfs/opt/bitnami/scripts/libairflowworker.sh diff --git a/Dockerfile b/Dockerfile index d4d9b76..edda403 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,26 +16,38 @@ ENV BITNAMI_PKG_EXTRA_DIRS="/opt/bitnami/airflow/dags" \ COPY --from=base ${BITNAMI_HOME} ${BITNAMI_HOME} COPY --from=base ${BITNAMI_DIR} ${BITNAMI_DIR} +COPY --from=base \ + /lib/x86_64-linux-gnu/libbz2.so.1.0 \ + /usr/lib64/ + COPY prebuildfs / RUN dnf update -y --nodocs && \ - dnf install -y curl tar gzip ca-certificates libxml2 \ - procps glibc-locale-source glibc-langpack-en && \ + dnf install -y curl tar gzip ca-certificates libxml2 procps glibc-langpack-en && \ dnf clean all && \ rm -rf /var/cache/dnf && \ - localedef -c -f UTF-8 -i en_US en_US.UTF-8 && \ + mkdir -p /local/wheels && \ chmod g+rwX /opt/bitnami COPY rootfs / -RUN /opt/bitnami/scripts/airflow/postunpack.sh +COPY *.whl *.tar.gz /local/wheels/ -COPY --from=base \ - /lib/x86_64-linux-gnu/libbz2.so.1.0 \ - /usr/lib64/ +RUN /opt/bitnami/scripts/airflow-worker/postunpack.sh && \ + source /opt/bitnami/airflow/venv/bin/activate && \ + # Remove for security advisory GHSA-x9p2-fxq6-2m5f GHSA-4f9m-pxwh-68hg GHSA-388g-jwpg-x6j4 + rm -rf ${BITNAMI_HOME}/airflow/venv/lib/python3.8/site-packages/swagger_ui_bundle/vendor/swagger-ui-2.2.10 && \ + # Update to thrift-0.14 per CVE-2020-13949 + tar xfz /local/wheels/thrift-0.14.1.tar.gz -C /local/wheels && \ + pip install --no-index --no-deps /local/wheels/thrift-0.14.1/lib/py && \ + rm -rf /local/wheels/thrift-0.14.1* && \ + for f in $(ls -l /local/wheels | awk '{print $9}' |sed '/^$/d'); do pip install --no-index --no-deps /local/wheels/$f; done && \ + find /opt/bitnami/airflow/venv/lib/python3.8/site-packages -name "*.pem" -o -name "*.key" | egrep ".*test.*/.*\.pem|.*test.*/.*\.key" | xargs rm -f && \ + rm -rf /local/* ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ - BITNAMI_APP_NAME="airflow" \ - BITNAMI_IMAGE_VERSION="2.0.1-debian-10-r53" \ + BITNAMI_APP_NAME="airflow-worker" \ + BITNAMI_IMAGE_VERSION="2.0.1-debian-10-r54" \ + C_FORCE_ROOT="True" \ LANG="en_US.UTF-8" \ LANGUAGE="en_US:en" \ LD_LIBRARY_PATH="/opt/bitnami/python/lib/:/opt/bitnami/airflow/venv/lib/python3.8/site-packages/numpy.libs/:$LD_LIBRARY_PATH" \ @@ -45,8 +57,8 @@ ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ NSS_WRAPPER_PASSWD="/opt/bitnami/airflow/nss_passwd" \ PATH="/opt/bitnami/common/bin:/opt/bitnami/python/bin:/opt/bitnami/postgresql/bin:/opt/bitnami/airflow/venv/bin:$PATH" -EXPOSE 8080 +EXPOSE 8793 USER 1001 -ENTRYPOINT [ "/opt/bitnami/scripts/airflow/entrypoint.sh" ] -CMD [ "/opt/bitnami/scripts/airflow/run.sh" ] +ENTRYPOINT [ "/opt/bitnami/scripts/airflow-worker/entrypoint.sh" ] +CMD [ "/opt/bitnami/scripts/airflow-worker/run.sh" ] diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 806490f..63aa51d 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -38,6 +38,36 @@ labels: resources: - tag: bitnami/airflow-worker:2.0.1-debian-10-r53 url: docker://docker.io/bitnami/airflow-worker@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae389 +- filename: thrift-0.14.1.tar.gz + url: https://mirror.jframeworks.com/apache/thrift/0.14.1/thrift-0.14.1.tar.gz + validation: + type: sha256 + value: 13da5e1cd9c8a3bb89778c0337cc57eb0c29b08f3090b41cf6ab78594b410ca5 +- filename: pylint-2.7.2-py3-none-any.whl + url: https://files.pythonhosted.org/packages/b3/66/af8f80d4fa77dcd4cba9e56e136522838920a2eaf6794b784e1f377f84d9/pylint-2.7.2-py3-none-any.whl + validation: + type: sha256 + value: d09b0b07ba06bcdff463958f53f23df25e740ecd81895f7d2699ec04bbd8dc3b +- filename: networkx-2.5.1-py3-none-any.whl + url: https://files.pythonhosted.org/packages/f3/b7/c7f488101c0bb5e4178f3cde416004280fd40262433496830de8a8c21613/networkx-2.5.1-py3-none-any.whl + validation: + type: sha256 + value: 0635858ed7e989f4c574c2328380b452df892ae85084144c73d8cd819f0c4e06 +- filename: decorator-4.4.2-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/ed/1b/72a1821152d07cf1d8b6fce298aeb06a7eb90f4d6d41acec9861e7cc6df0/decorator-4.4.2-py2.py3-none-any.whl + validation: + type: sha256 + value: 41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760 +- filename: aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_x86_64.whl + url: https://files.pythonhosted.org/packages/a6/76/f18138b0ff84fcd939667a2efc2e1b49c871299f9091f84c06bb4c350c01/aiohttp-3.7.4.post0-cp38-cp38-manylinux2014_x86_64.whl + validation: + type: sha256 + value: 79ebfc238612123a713a457d92afb4096e2148be17df6c50fb9bf7a81c2f8013 +- filename: azure_storage_blob-12.8.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/09/14/4ca417a9c92b0fb93516575dd7be9b058bf13d531dcc21239b5f8f216a69/azure_storage_blob-12.8.0-py2.py3-none-any.whl + validation: + type: sha256 + value: 46999df6e2cde8773739f7c3bd1eb5846d4b7dc1ef6e2161f3b6d1d0f21726ba # List of project maintainers maintainers: diff --git a/prebuildfs/opt/bitnami/.bitnami_components.json b/prebuildfs/opt/bitnami/.bitnami_components.json index 67b2f78..1c30dae 100644 --- a/prebuildfs/opt/bitnami/.bitnami_components.json +++ b/prebuildfs/opt/bitnami/.bitnami_components.json @@ -1,10 +1,10 @@ { - "airflow": { + "airflow-worker": { "arch": "amd64", - "digest": "d837c8af9305cfcbed7dd0493336ba0e38d7a3aa211192a8f05d117a7b7734ab", + "digest": "cf754bdc8e231d408275c1971d9a732576e346e0cd4af1ea426739bf3e62538d", "distro": "debian-10", "type": "NAMI", - "version": "2.0.1-4" + "version": "2.0.1-2" }, "gosu": { "arch": "amd64", diff --git a/prebuildfs/opt/bitnami/scripts/liblog.sh b/prebuildfs/opt/bitnami/scripts/liblog.sh index 60ec4cb..c7c0f6d 100644 --- a/prebuildfs/opt/bitnami/scripts/liblog.sh +++ b/prebuildfs/opt/bitnami/scripts/liblog.sh @@ -106,5 +106,7 @@ indent() { for ((i = 0; i < num; i++)); do indent_unit="${indent_unit}${char}" done + # shellcheck disable=SC2001 + # Complex regex, see https://github.com/koalaman/shellcheck/wiki/SC2001#exceptions echo "$string" | sed "s/^/${indent_unit}/" } diff --git a/prebuildfs/opt/bitnami/scripts/libversion.sh b/prebuildfs/opt/bitnami/scripts/libversion.sh index 66d4f45..f3bc756 100644 --- a/prebuildfs/opt/bitnami/scripts/libversion.sh +++ b/prebuildfs/opt/bitnami/scripts/libversion.sh @@ -9,7 +9,7 @@ # Functions ######################## -# Gets semantic version +# Gets semantic version # Arguments: # $1 - version: string to extract major.minor.patch # $2 - section: 1 to extract major, 2 to extract minor, 3 to extract patch @@ -38,7 +38,7 @@ get_sematic_version () { done local number_regex='^[0-9]+$' - if [[ "$section" =~ $number_regex ]] && (( $section > 0 )) && (( $section <= 3 )); then + if [[ "$section" =~ $number_regex ]] && (( section > 0 )) && (( section <= 3 )); then echo "${version_sections[$section]}" return else diff --git a/rootfs/opt/bitnami/scripts/airflow-env.sh b/rootfs/opt/bitnami/scripts/airflow-worker-env.sh similarity index 61% rename from rootfs/opt/bitnami/scripts/airflow-env.sh rename to rootfs/opt/bitnami/scripts/airflow-worker-env.sh index c7e4caa..ed444ca 100644 --- a/rootfs/opt/bitnami/scripts/airflow-env.sh +++ b/rootfs/opt/bitnami/scripts/airflow-worker-env.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Environment configuration for airflow +# Environment configuration for airflow-worker # The values for all environment variables will be set in the below order of precedence # 1. Custom environment variables defined below after Bitnami defaults @@ -15,27 +15,17 @@ export BITNAMI_ROOT_DIR="/opt/bitnami" export BITNAMI_VOLUME_DIR="/bitnami" # Logging configuration -export MODULE="${MODULE:-airflow}" +export MODULE="${MODULE:-airflow-worker}" export BITNAMI_DEBUG="${BITNAMI_DEBUG:-false}" # By setting an environment variable matching *_FILE to a file path, the prefixed environment # variable will be overridden with the value specified in that file -airflow_env_vars=( - AIRFLOW_USERNAME - AIRFLOW_PASSWORD - AIRFLOW_FIRSTNAME - AIRFLOW_LASTNAME - AIRFLOW_EMAIL +airflow_worker_env_vars=( AIRFLOW_EXECUTOR AIRFLOW_FERNET_KEY AIRFLOW_WEBSERVER_HOST AIRFLOW_WEBSERVER_PORT_NUMBER - AIRFLOW_LOAD_EXAMPLES - AIRFLOW_BASE_URL AIRFLOW_HOSTNAME_CALLABLE - AIRFLOW_POOL_NAME - AIRFLOW_POOL_SIZE - AIRFLOW_POOL_DESC AIRFLOW_DATABASE_HOST AIRFLOW_DATABASE_PORT_NUMBER AIRFLOW_DATABASE_NAME @@ -47,18 +37,8 @@ airflow_env_vars=( REDIS_PORT_NUMBER REDIS_USER REDIS_PASSWORD - AIRFLOW_LDAP_ENABLE - AIRFLOW_LDAP_URI - AIRFLOW_LDAP_SEARCH - AIRFLOW_LDAP_BIND_USER - AIRFLOW_LDAP_BIND_PASSWORD - AIRFLOW_LDAP_UID_FIELD - AIRFLOW_LDAP_USE_TLS - AIRFLOW_LDAP_ALLOW_SELF_SIGNED - AIRFLOW_LDAP_TLS_CA_CERTIFICATE - AIRFLOW_USER_REGISTRATION_ROLE ) -for env_var in "${airflow_env_vars[@]}"; do +for env_var in "${airflow_worker_env_vars[@]}"; do file_env_var="${env_var}_FILE" if [[ -n "${!file_env_var:-}" ]]; then if [[ -r "${!file_env_var:-}" ]]; then @@ -69,7 +49,7 @@ for env_var in "${airflow_env_vars[@]}"; do fi fi done -unset airflow_env_vars +unset airflow_worker_env_vars # Airflow paths export AIRFLOW_BASE_DIR="${BITNAMI_ROOT_DIR}/airflow" @@ -78,11 +58,10 @@ export AIRFLOW_BIN_DIR="${AIRFLOW_BASE_DIR}/venv/bin" export AIRFLOW_VOLUME_DIR="/bitnami/airflow" export AIRFLOW_DATA_DIR="${AIRFLOW_BASE_DIR}/data" export AIRFLOW_LOGS_DIR="${AIRFLOW_BASE_DIR}/logs" -export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-webserver.log" +export AIRFLOW_LOG_FILE="${AIRFLOW_LOGS_DIR}/airflow-worker.log" export AIRFLOW_CONF_FILE="${AIRFLOW_BASE_DIR}/airflow.cfg" -export AIRFLOW_WEBSERVER_CONF_FILE="${AIRFLOW_BASE_DIR}/webserver_config.py" export AIRFLOW_TMP_DIR="${AIRFLOW_BASE_DIR}/tmp" -export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-webserver.pid" +export AIRFLOW_PID_FILE="${AIRFLOW_TMP_DIR}/airflow-worker.pid" export AIRFLOW_DATA_TO_PERSIST="$AIRFLOW_DATA_DIR" export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags" @@ -90,24 +69,12 @@ export AIRFLOW_DAGS_DIR="${AIRFLOW_BASE_DIR}/dags" export AIRFLOW_DAEMON_USER="airflow" export AIRFLOW_DAEMON_GROUP="airflow" -# User configuration -export AIRFLOW_USERNAME="${AIRFLOW_USERNAME:-user}" -export AIRFLOW_PASSWORD="${AIRFLOW_PASSWORD:-bitnami}" -export AIRFLOW_FIRSTNAME="${AIRFLOW_FIRSTNAME:-Firstname}" -export AIRFLOW_LASTNAME="${AIRFLOW_LASTNAME:-Lastname}" -export AIRFLOW_EMAIL="${AIRFLOW_EMAIL:-user@example.com}" - # Airflow configuration export AIRFLOW_EXECUTOR="${AIRFLOW_EXECUTOR:-SequentialExecutor}" export AIRFLOW_FERNET_KEY="${AIRFLOW_FERNET_KEY:-}" export AIRFLOW_WEBSERVER_HOST="${AIRFLOW_WEBSERVER_HOST:-127.0.0.1}" export AIRFLOW_WEBSERVER_PORT_NUMBER="${AIRFLOW_WEBSERVER_PORT_NUMBER:-8080}" -export AIRFLOW_LOAD_EXAMPLES="${AIRFLOW_LOAD_EXAMPLES:-yes}" -export AIRFLOW_BASE_URL="${AIRFLOW_BASE_URL:-}" export AIRFLOW_HOSTNAME_CALLABLE="${AIRFLOW_HOSTNAME_CALLABLE:-}" -export AIRFLOW_POOL_NAME="${AIRFLOW_POOL_NAME:-}" -export AIRFLOW_POOL_SIZE="${AIRFLOW_POOL_SIZE:-}" -export AIRFLOW_POOL_DESC="${AIRFLOW_POOL_DESC:-}" # Airflow database configuration export AIRFLOW_DATABASE_HOST="${AIRFLOW_DATABASE_HOST:-postgresql}" @@ -122,16 +89,4 @@ export REDIS_PORT_NUMBER="${REDIS_PORT_NUMBER:-6379}" export REDIS_USER="${REDIS_USER:-}" export REDIS_PASSWORD="${REDIS_PASSWORD:-}" -# Airflow LDAP configuration -export AIRFLOW_LDAP_ENABLE="${AIRFLOW_LDAP_ENABLE:-no}" -export AIRFLOW_LDAP_URI="${AIRFLOW_LDAP_URI:-}" -export AIRFLOW_LDAP_SEARCH="${AIRFLOW_LDAP_SEARCH:-}" -export AIRFLOW_LDAP_BIND_USER="${AIRFLOW_LDAP_BIND_USER:-}" -export AIRFLOW_LDAP_BIND_PASSWORD="${AIRFLOW_LDAP_BIND_PASSWORD:-}" -export AIRFLOW_LDAP_UID_FIELD="${AIRFLOW_LDAP_UID_FIELD:-}" -export AIRFLOW_LDAP_USE_TLS="${AIRFLOW_LDAP_USE_TLS:-False}" -export AIRFLOW_LDAP_ALLOW_SELF_SIGNED="${AIRFLOW_LDAP_ALLOW_SELF_SIGNED:-True}" -export AIRFLOW_LDAP_TLS_CA_CERTIFICATE="${AIRFLOW_LDAP_TLS_CA_CERTIFICATE:-}" -export AIRFLOW_USER_REGISTRATION_ROLE="${AIRFLOW_USER_REGISTRATION_ROLE:-Public}" - # Custom environment variables may be defined below diff --git a/rootfs/opt/bitnami/scripts/airflow/entrypoint.sh b/rootfs/opt/bitnami/scripts/airflow-worker/entrypoint.sh similarity index 61% rename from rootfs/opt/bitnami/scripts/airflow/entrypoint.sh rename to rootfs/opt/bitnami/scripts/airflow-worker/entrypoint.sh index c808f10..3f04f77 100755 --- a/rootfs/opt/bitnami/scripts/airflow/entrypoint.sh +++ b/rootfs/opt/bitnami/scripts/airflow-worker/entrypoint.sh @@ -8,17 +8,17 @@ set -o pipefail # set -o xtrace # Uncomment this line for debugging purpose # Load Airflow environment variables -. /opt/bitnami/scripts/airflow-env.sh +. /opt/bitnami/scripts/airflow-worker-env.sh # Load libraries . /opt/bitnami/scripts/libbitnami.sh -. /opt/bitnami/scripts/libairflow.sh +. /opt/bitnami/scripts/libairflowworker.sh print_welcome_page -if [[ "$*" = *"/opt/bitnami/scripts/airflow/run.sh"* || "$*" = *"/run.sh"* ]]; then +if [[ "$*" = *"/opt/bitnami/scripts/airflow-worker/run.sh"* || "$*" = *"/run.sh"* ]]; then info "** Starting Airflow setup **" - /opt/bitnami/scripts/airflow/setup.sh + /opt/bitnami/scripts/airflow-worker/setup.sh info "** Airflow setup finished! **" fi diff --git a/rootfs/opt/bitnami/scripts/airflow/postunpack.sh b/rootfs/opt/bitnami/scripts/airflow-worker/postunpack.sh similarity index 88% rename from rootfs/opt/bitnami/scripts/airflow/postunpack.sh rename to rootfs/opt/bitnami/scripts/airflow-worker/postunpack.sh index 61cd4c3..8a5fccc 100755 --- a/rootfs/opt/bitnami/scripts/airflow/postunpack.sh +++ b/rootfs/opt/bitnami/scripts/airflow-worker/postunpack.sh @@ -8,10 +8,10 @@ set -o pipefail # set -o xtrace # Uncomment this line for debugging purpose # Load Airflow environment variables -. /opt/bitnami/scripts/airflow-env.sh +. /opt/bitnami/scripts/airflow-worker-env.sh # Load libraries -. /opt/bitnami/scripts/libairflow.sh +. /opt/bitnami/scripts/libairflowworker.sh . /opt/bitnami/scripts/libfs.sh . /opt/bitnami/scripts/libos.sh diff --git a/rootfs/opt/bitnami/scripts/airflow/run.sh b/rootfs/opt/bitnami/scripts/airflow-worker/run.sh similarity index 67% rename from rootfs/opt/bitnami/scripts/airflow/run.sh rename to rootfs/opt/bitnami/scripts/airflow-worker/run.sh index 00a2bd0..ef2db82 100755 --- a/rootfs/opt/bitnami/scripts/airflow/run.sh +++ b/rootfs/opt/bitnami/scripts/airflow-worker/run.sh @@ -8,17 +8,17 @@ set -o pipefail # set -o xtrace # Uncomment this line for debugging purposes # Load Airflow environment variables -. /opt/bitnami/scripts/airflow-env.sh +. /opt/bitnami/scripts/airflow-worker-env.sh # Load libraries . /opt/bitnami/scripts/libos.sh -. /opt/bitnami/scripts/libairflow.sh +. /opt/bitnami/scripts/libairflowworker.sh args=("--pid" "$AIRFLOW_PID_FILE" "$@") info "** Starting Airflow **" if am_i_root; then - exec gosu "$AIRFLOW_DAEMON_USER" "${AIRFLOW_BIN_DIR}/airflow" "webserver" "${args[@]}" + exec gosu "$AIRFLOW_DAEMON_USER" "${AIRFLOW_BIN_DIR}/airflow" "celery" "worker" "${args[@]}" else - exec "${AIRFLOW_BIN_DIR}/airflow" "webserver" "${args[@]}" + exec "${AIRFLOW_BIN_DIR}/airflow" "celery" "worker" "${args[@]}" fi diff --git a/rootfs/opt/bitnami/scripts/airflow/setup.sh b/rootfs/opt/bitnami/scripts/airflow-worker/setup.sh similarity index 78% rename from rootfs/opt/bitnami/scripts/airflow/setup.sh rename to rootfs/opt/bitnami/scripts/airflow-worker/setup.sh index 758d2e8..597b0d1 100755 --- a/rootfs/opt/bitnami/scripts/airflow/setup.sh +++ b/rootfs/opt/bitnami/scripts/airflow-worker/setup.sh @@ -8,16 +8,16 @@ set -o pipefail # set -o xtrace # Uncomment this line for debugging purposes # Load Airflow environment variables -. /opt/bitnami/scripts/airflow-env.sh +. /opt/bitnami/scripts/airflow-worker-env.sh # Load libraries . /opt/bitnami/scripts/libos.sh . /opt/bitnami/scripts/libfs.sh -. /opt/bitnami/scripts/libairflow.sh +. /opt/bitnami/scripts/libairflowworker.sh # Ensure Airflow environment variables settings are valid -airflow_validate +airflow_worker_validate # Ensure Airflow daemon user exists when running as root am_i_root && ensure_user_exists "$AIRFLOW_DAEMON_USER" --group "$AIRFLOW_DAEMON_GROUP" # Ensure Airflow is initialized -airflow_initialize +airflow_worker_initialize diff --git a/rootfs/opt/bitnami/scripts/libairflowworker.sh b/rootfs/opt/bitnami/scripts/libairflowworker.sh new file mode 100644 index 0000000..e440bd8 --- /dev/null +++ b/rootfs/opt/bitnami/scripts/libairflowworker.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# Bitnami Airflow library + +# shellcheck disable=SC1091 +# shellcheck disable=SC2153 + +# Load Generic Libraries +. /opt/bitnami/scripts/libfile.sh +. /opt/bitnami/scripts/liblog.sh +. /opt/bitnami/scripts/libnet.sh +. /opt/bitnami/scripts/libos.sh +. /opt/bitnami/scripts/libservice.sh +. /opt/bitnami/scripts/libvalidations.sh +. /opt/bitnami/scripts/libpersistence.sh + +# Load airflow library +. /opt/bitnami/scripts/libairflow.sh + +######################## +# Validate Airflow Scheduler inputs +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_worker_validate() { + # Check postgresql host + [[ -z "$AIRFLOW_WEBSERVER_HOST" ]] && print_validation_error "Missing AIRFLOW_WEBSERVER_HOST" + [[ -z "$AIRFLOW_WEBSERVER_PORT_NUMBER" ]] && print_validation_error "Missing AIRFLOW_WEBSERVER_PORT_NUMBER" + # Check postgresql host + [[ -z "$AIRFLOW_DATABASE_HOST" ]] && print_validation_error "Missing AIRFLOW_DATABASE_HOST" + + # Avoid fail because of the above check + true +} + +######################## +# Ensure Airflow Scheduler is initialized +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_worker_initialize() { + # Change permissions if running as root + for dir in "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR" "$AIRFLOW_DATA_DIR"; do + ensure_dir_exists "$dir" + am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir" + done + + # The configuration file is not persisted. If it is not provided, generate it based on env vars + if [[ ! -f "$AIRFLOW_CONF_FILE" ]]; then + info "No injected configuration file found. Creating default config file" + airflow_worker_generate_config + else + info "Configuration file found, loading configuration" + fi + + # Check if Airflow has already been initialized and persisted in a previous run + local -r app_name="airflow" + if ! is_app_initialized "$app_name"; then + airflow_wait_for_postgresql "$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" + + info "Persisting Airflow installation" + persist_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST" + else + # Check database connection + airflow_wait_for_postgresql "$AIRFLOW_DATABASE_HOST" "$AIRFLOW_DATABASE_PORT_NUMBER" + + # Restore persisted data + info "Restoring persisted Airflow installation" + restore_persisted_app "$app_name" "$AIRFLOW_DATA_TO_PERSIST" + + # Change the permissions after restoring the persisted data in case we are root + for dir in "$AIRFLOW_DATA_DIR" "$AIRFLOW_TMP_DIR" "$AIRFLOW_LOGS_DIR"; do + ensure_dir_exists "$dir" + am_i_root && chown "$AIRFLOW_DAEMON_USER:$AIRFLOW_DAEMON_GROUP" "$dir" + done + true # Avoid return false when I am not root + fi + + # Wait for airflow webserver to be available + info "Waiting for Airflow Webserser to be up" + airflow_worker_wait_for_webserver "$AIRFLOW_WEBSERVER_HOST" "$AIRFLOW_WEBSERVER_PORT_NUMBER" + [[ "$AIRFLOW_EXECUTOR" == "CeleryExecutor" ]] && wait-for-port --host "$REDIS_HOST" "$REDIS_PORT_NUMBER" + + # Avoid to fail when the executor is not celery + true +} + +######################## +# Generate Airflow Scheduler conf file +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_worker_generate_config() { + # Generate Airflow default files + airflow_execute_command "version" "version" + + # Configure Airflow Hostname + [[ -n "$AIRFLOW_HOSTNAME_CALLABLE" ]] && airflow_conf_set "core" "hostname_callable" "$AIRFLOW_HOSTNAME_CALLABLE" + + # Configure Airflow database + airflow_configure_database + + # Configure the Webserver port + airflow_conf_set "webserver" "web_server_port" "$AIRFLOW_WEBSERVER_PORT_NUMBER" + + # Setup fernet key + [[ -n "$AIRFLOW_FERNET_KEY" ]] && airflow_conf_set "core" "fernet_key" "$AIRFLOW_FERNET_KEY" + + # Configure Airflow executor + airflow_conf_set "core" "executor" "$AIRFLOW_EXECUTOR" + [[ "$AIRFLOW_EXECUTOR" == "CeleryExecutor" ]] && airflow_configure_celery_executor + true # Avoid the function to fail due to the check above +} + +######################## +# Wait Ariflow webserver +# Globals: +# AIRFLOW_* +# Arguments: +# None +# Returns: +# None +######################### +airflow_worker_wait_for_webserver() { + local -r webserver_host="${1:?missing database host}" + local -r webserver_port="${2:?missing database port}" + check_webserver_connection() { + wait-for-port --host "$webserver_host" "$webserver_port" + } + if ! retry_while "check_webserver_connection"; then + error "Could not connect to the Airflow webserver" + return 1 + fi +} diff --git a/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh b/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh index 601a83c..de3120b 100755 --- a/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh +++ b/rootfs/opt/bitnami/scripts/locales/add-extra-locales.sh @@ -33,6 +33,7 @@ if [[ "$WITH_ALL_LOCALES" =~ ^(yes|true|1)$ ]]; then echo "Enabling all locales" cp "$SUPPORTED_LOCALES_FILE" "$LOCALES_FILE" else + # shellcheck disable=SC2001 LOCALES_TO_ADD="$(sed 's/[,;]\s*/\n/g' <<< "$EXTRA_LOCALES")" while [[ -n "$LOCALES_TO_ADD" ]] && read -r locale; do echo "Enabling locale ${locale}" -- GitLab From ad7863369277eedda9d2c8f357a6b33c95f11fdf Mon Sep 17 00:00:00 2001 From: Austin Denton Date: Fri, 9 Apr 2021 06:48:32 -0600 Subject: [PATCH 4/5] Update to r54 --- Dockerfile | 3 ++- hardening_manifest.yaml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index edda403..d455211 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_REGISTRY=registry1.dsop.io ARG BASE_IMAGE=ironbank/redhat/ubi/ubi8 ARG BASE_TAG=8.3 -FROM bitnami/airflow-worker:2.0.1-debian-10-r53 as base +FROM bitnami/airflow-worker:2.0.1-debian-10-r54 as base FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} ARG BITNAMI_HOME=/opt/bitnami ARG BITNAMI_DIR=/bitnami @@ -42,6 +42,7 @@ RUN /opt/bitnami/scripts/airflow-worker/postunpack.sh && \ rm -rf /local/wheels/thrift-0.14.1* && \ for f in $(ls -l /local/wheels | awk '{print $9}' |sed '/^$/d'); do pip install --no-index --no-deps /local/wheels/$f; done && \ find /opt/bitnami/airflow/venv/lib/python3.8/site-packages -name "*.pem" -o -name "*.key" | egrep ".*test.*/.*\.pem|.*test.*/.*\.key" | xargs rm -f && \ + chmod +x /opt/bitnami/scripts/* && \ rm -rf /local/* ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 63aa51d..89c9894 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -36,8 +36,8 @@ labels: # List of resources to make available to the offline build context resources: -- tag: bitnami/airflow-worker:2.0.1-debian-10-r53 - url: docker://docker.io/bitnami/airflow-worker@sha256:fdb55ad52b70ea398597634a5d2495c2c1d24935da25e242a08e853191bae389 +- tag: bitnami/airflow-worker:2.0.1-debian-10-r54 + url: docker://docker.io/bitnami/airflow-worker@sha256:224ad64fbbf7cae7930fb3241b8e515499cde298cae639a4b037c0c3f8d8cd2c - filename: thrift-0.14.1.tar.gz url: https://mirror.jframeworks.com/apache/thrift/0.14.1/thrift-0.14.1.tar.gz validation: -- GitLab From a34acdccaa34adbf3e04382ce31315cc8f171e5e Mon Sep 17 00:00:00 2001 From: Austin Denton Date: Tue, 13 Apr 2021 10:53:16 -0600 Subject: [PATCH 5/5] Add health check --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d455211..963c126 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,8 +58,11 @@ ENV AIRFLOW_HOME="/opt/bitnami/airflow" \ NSS_WRAPPER_PASSWD="/opt/bitnami/airflow/nss_passwd" \ PATH="/opt/bitnami/common/bin:/opt/bitnami/python/bin:/opt/bitnami/postgresql/bin:/opt/bitnami/airflow/venv/bin:$PATH" +USER 1001 + EXPOSE 8793 -USER 1001 +HEALTHCHECK --start-period=10s CMD bash -c '/dev/null' || exit 1 + ENTRYPOINT [ "/opt/bitnami/scripts/airflow-worker/entrypoint.sh" ] CMD [ "/opt/bitnami/scripts/airflow-worker/run.sh" ] -- GitLab