UNCLASSIFIED

Skip to content

GitLab

Commit 4ac04ef1 authored by alexander.klepal's avatar alexander.klepal
Browse files

Merge branch 'ubi8-update' into 'development' 

Ubi8 update

See merge request !7
parents fe6770d7 d4d7f59c
Pipeline #174020 passed with stages
in 27 minutes and 54 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 376 deletions
Dockerfile
View file @ 4ac04ef1
#
# Base Template Dockerfile for Database Image.
#
ARG BASE_REGISTRY=registry1.dsop.io
ARG BASE_IMAGE=redhat/ubi/ubi7
ARG BASE_TAG=7.8
ARG BASE_IMAGE=redhat/ubi/ubi8
ARG BASE_TAG=8.3
FROM quay.io/mongodb/mongodb-enterprise-database-ubi:1.5.3 as official
FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
ARG RELEASE=1.5.3
ARG TARBALL=mongodb-enterprise-operator-binaries-release-${RELEASE}.tar.gz
ENV MMS_HOME /mongodb-automation
ENV MMS_LOG_DIR /var/log/mongodb-mms-automation
ENV MDB_DIR /var/lib/mongodb-mms-automation/downloads
ARG MDB_URL
ARG BINARY_NAME
ARG AA_DOWNLOAD_URL
ARG AA_VERSION
LABEL name="MongoDB Enterprise Database" \
version="1.5.3" \
summary="MongoDB Enterprise Database Image" \
......@@ -23,48 +15,48 @@ LABEL name="MongoDB Enterprise Database" \
vendor="MongoDB" \
release="1" \
maintainer="support@mongodb.com"
COPY LICENSE /licenses/mongodb-enterprise-database
RUN yum update -y && yum install \
libpcap \
RUN yum update -y && rm -rf /var/cache/yum \
# these are the packages needed for the agent
&& yum install -y --disableplugin=subscription-manager \
hostname \
nss_wrapper --exclude perl-IO-Socket-SSL \
procps \
# these are the packages needed for MongoDB
# (https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ "RHEL/CentOS 8" tab)
&& yum install -y --disableplugin=subscription-manager \
cyrus-sasl \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
krb5-libs \
libcurl \
lm_sensors-libs \
net-snmp \
net-snmp-agent-libs \
rpm-libs \
tcp_wrappers-libs \
nss_wrapper; \
yum clean all; \
ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2
# TODO: remove once database is using init container
COPY scripts/ "${MMS_HOME}"/files/
# Copy readinessprobe and set the required permissions
WORKDIR /opt
COPY ${TARBALL} .
RUN tar -zxf ./${TARBALL} \
&& mv ./readinessprobe "${MMS_HOME}"/files/ \
&& rm -rfv /opt/* /var/cache/yum \
&& mkdir -p "${MMS_LOG_DIR}" \
&& chmod 0775 "${MMS_LOG_DIR}" \
&& mkdir -p /var/lib/mongodb-mms-automation \
&& chmod 0775 /var/lib/mongodb-mms-automation \
&& mkdir -p /data \
&& chmod 0775 /data \
&& mkdir -p /journal \
&& chmod 0775 /journal \
&& chmod -R 0775 "${MMS_HOME}"
openldap \
openssl \
jq \
# mongodb enterprise expects this library /usr/lib64/libsasl2.so.2 but
# cyrus-sasl creates it in /usr/lib64/libsasl2.so.3 instead
&& ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2
# Copy all the required scripts from the official database image
COPY --from=official "${MMS_HOME}" ${MMS_HOME}/
# Set the required perms
RUN mkdir -p "${MMS_LOG_DIR}" \
&& chmod 0775 "${MMS_LOG_DIR}" \
&& mkdir -p /var/lib/mongodb-mms-automation \
&& chmod 0775 /var/lib/mongodb-mms-automation \
&& mkdir -p /data \
&& chmod 0775 /data \
&& mkdir -p /journal \
&& chmod 0775 /journal \
&& mkdir -p "${MMS_HOME}" \
&& chmod -R 0775 "${MMS_HOME}"
# USER needs to be set for this image to pass RedHat verification. Some customers have these requirements as well
# It does not matter what number it is, as long as it is set to something.
# However, OpenShift will run the container as a random user,
# and the number in this configuration is not relevant.
USER 2000
# TODO: make entrypoint be a sleep infinity once MDB is using init container
ENTRYPOINT ["/mongodb-automation/files/agent-launcher.sh"]
# TODO: switch this to actually run the probe
HEALTHCHECK --timeout=30s CMD ls "${MMS_HOME}"/files/probe.sh || exit 1
Jenkinsfile deleted 100644 → 0
View file @ fe6770d7
@Library('DCCSCR@master') _
dccscrPipeline(version: "1.5.3")
README.md
View file @ 4ac04ef1
......@@ -7,9 +7,9 @@
For licensing and technical information, please use the following contact information:
Anton Hoffman, EAE USAF
781-996-8860
anton.hoffman@mongodb.com
Keegan Wetzel, EAE USAF
708-256-7904
keegan.wetzel@mongodb.com
Please also CC: publicsector@mongodb.com
......
download.json deleted 100755 → 0
View file @ fe6770d7
{
"resources": [
{
"url": "https://s3.amazonaws.com/ops-manager-kubernetes-build/releases/mongodb-enterprise-operator-binaries-release-1.5.3.tar.gz",
"filename": "mongodb-enterprise-operator-binaries-release-1.5.3.tar.gz",
"validation": {
"type": "sha256",
"value": "251bf6aa9e6deeba3bd5366228ce81f6322c6a15aa8b07558cb23092d73333b8"
}
}
]
}
\ No newline at end of file
hardening_manifest.yaml 0 → 100644
View file @ 4ac04ef1
---
apiVersion: v1
# The repository name in registry1, excluding /ironbank/
name: "mongodb/mongodb-enterprise/mongodb-enterprise-database"
# List of tags to push for the repository in registry1
# The most specific version should be the first tag and will be shown
# on ironbank.dsop.io
tags:
- "1.5.3"
- "latest"
# Build args passed to Dockerfile ARGs
args:
BASE_IMAGE: "redhat/ubi/ubi8"
BASE_TAG: "8.2"
# Docker image labels
labels:
org.opencontainers.image.title: "mongodb-enterprise-database"
## Human-readable description of the software packaged in the image
org.opencontainers.image.description: "Base Database Image used in conjunction with MongoDB Enterprise Kubernetes Operator"
## License(s) under which contained software is distributed
org.opencontainers.image.licenses: "license"
## URL to find more information on the image
org.opencontainers.image.url: "https://docs.mongodb.com/kubernetes-operator/stable/"
## Name of the distributing entity, organization or individual
org.opencontainers.image.vendor: "MongoDB"
org.opencontainers.image.version: "1.5.3"
## Keywords to help with search (ex. "cicd,gitops,golang")
mil.dso.ironbank.image.keywords: "mongodb,nosql,json,database"
## This value can be "opensource" or "commercial"
mil.dso.ironbank.image.type: "commercial"
## Product the image belongs to for grouping multiple images
mil.dso.ironbank.product.name: "mongodb-enterprise"
# List of resources to make available to the offline build context
resources:
- tag: quay.io/mongodb/mongodb-enterprise-database-ubi:1.5.3
url: docker://quay.io/mongodb/mongodb-enterprise-database-ubi@sha256:8f2734cea7fc31f829a0f8752be5bfd1c69b3d2e59e677bb23f685aac7f58eb9
# List of project maintainers
# FIXME: Fill in the following details for the current container owner in the whitelist
# FIXME: Include any other vendor information if applicable
maintainers:
- email: "jonathan.janos@mongodb.com"
# The name of the current container owner
name: "Jonathan Janos"
# The gitlab username of the current container owner
username: "jonathan.janos"
- email: "klepal_alexander@bah.com"
name: "Alexander Klepal"
username: "alexander.klepal"
cht_member: true
scripts/agent-launcher-lib.sh deleted 100755 → 0
View file @ fe6770d7
#!/usr/bin/env bash
# This is a file containing all the functions which may be needed for other shell scripts
# see if jq is available for json logging
use_jq="$(command -v jq)"
# log stdout as structured json with given log type
json_log () {
if [ "$use_jq" ]; then
jq --unbuffered --null-input -c --raw-input "inputs | {\"logType\": \"$1\", \"contents\": .}";
else
echo "$1"
fi
}
# log a given message in json format
script_log () {
echo "$1" | json_log 'agent-launcher-script'
}
# the function reacting on SIGTERM command sent by the container on its shutdown. Makes sure all processes started (including
# mongodb) receive the signal. For MongoDB this results in graceful shutdown of replication (starting from 4.0.9) which may
# take some time. The script waits for all the processes to finish, otherwise the container would terminate as Kubernetes
# waits only for the process with pid #1 to end
cleanup () {
# Important! Keep this in sync with DefaultPodTerminationPeriodSeconds constant from constants.go
termination_timeout_seconds=600
script_log "Caught SIGTERM signal. Passing the signal to the automation agent and the mongod processes."
kill -15 "$agentPid"
wait "$agentPid"
mongoPid="$(cat /data/mongod.lock)"
kill -15 "$mongoPid"
script_log "Waiting until mongod process is shutdown. Note, that if mongod process fails to shutdown in the time specified by the 'terminationGracePeriodSeconds' property (default $termination_timeout_seconds seconds) then the container will be killed by Kubernetes."
# dev note: we cannot use 'wait' for the external processes, seems the spinning loop is the best option
while [ -e "/proc/$mongoPid" ]; do sleep 0.1; done
script_log "Mongod and automation agent processes are shutdown"
}
# ensure_certs_symlinks function checks if certificates and CAs are mounted and creates symlinks to them
ensure_certs_symlinks () {
# the paths inside the pod. Move to parameters if multiple usage is needed
secrets_dir="/var/lib/mongodb-automation/secrets"
custom_ca_dir="${secrets_dir}/ca"
pod_secrets_dir="/mongodb-automation"
if [ -d "${secrets_dir}/certs" ]; then
script_log "Found certificates in the host, will symlink to where the automation agent expects them to be"
podname=$(hostname)
if [ ! -f "${secrets_dir}/certs/${podname}-pem" ]; then
script_log "PEM Certificate file does not exist in ${secrets_dir}/certs/${podname}-pem. Check the Secret object with certificates is well formed."
exit 1
fi
ln -s "${secrets_dir}/certs/${podname}-pem" "${pod_secrets_dir}/server.pem"
fi
if [ -d "${custom_ca_dir}" ]; then
if [ -f "${custom_ca_dir}/ca-pem" ]; then
script_log "Using CA file provided by user"
ln -s "${custom_ca_dir}/ca-pem" "${pod_secrets_dir}/ca.pem"
else
script_log "Could not find CA file. The name of the entry on the Secret object should be 'ca-pem'"
exit 1
fi
else
script_log "Using Kubernetes CA file"
ln -s "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" "${pod_secrets_dir}/ca.pem"
fi
}
# download_agent function downloads and unpacks the Mongodb Agent
download_agent () {
script_log "Downloading a Mongodb Agent from ${base_url}"
pushd /tmp >/dev/null
curl_opts=(
"${base_url}/download/agent/automation/mongodb-mms-automation-agent-latest.linux_x86_64.tar.gz"
"--location" "--silent" "--retry" "3" "--fail" "-v"
"--output" "automation-agent.tar.gz"
)
if [ "${SSL_REQUIRE_VALID_MMS_CERTIFICATES-}" = "false" ]; then
# If we are not expecting valid certs, `curl` should be run with `--insecure` option.
# The default is NOT to accept insecure connections.
curl_opts+=("--insecure")
fi
if [ -n "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE-}" ]; then
curl_opts+=("--cacert" "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE}")
fi
if ! curl "${curl_opts[@]}" &> "${MMS_LOG_DIR}/agent-launcher-script.log"; then
script_log "Error while downloading the Mongodb agent"
cat "${MMS_LOG_DIR}/agent-launcher-script.log" | json_log 'agent-launcher-script'
exit 1
fi
script_log "The Mongodb Agent binary downloaded, unpacking"
tar -xzf automation-agent.tar.gz
AGENT_VERSION=$(find . -name mongodb-mms-automation-agent-* | awk -F"-" '{ print $5 }')
echo "${AGENT_VERSION}" > "${MMS_HOME}/files/agent-version"
mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent "${MMS_HOME}/files/"
chmod +x "${MMS_HOME}/files/mongodb-mms-automation-agent"
rm -rf automation-agent.tar.gz mongodb-mms-automation-agent-*.linux_x86_64
script_log "The Automation Agent was deployed at ${MMS_HOME}/files/mongodb-mms-automation-agent"
popd >/dev/null
}
#https://stackoverflow.com/a/4025065/614239
compare_versions () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
}
scripts/agent-launcher.sh deleted 100755 → 0
View file @ fe6770d7
#!/usr/bin/env bash
set -o nounset
set -o errexit
set -o pipefail
source "${MMS_HOME}/files/agent-launcher-lib.sh"
# The path to the automation config file in case the agent is run in headless mode
cluster_config_file="/var/lib/mongodb-automation/cluster-config.json"
# file required by Automation Agents of authentication is enabled.
keyfile_dir="/var/lib/mongodb-mms-automation"
mkdir -p ${keyfile_dir}
touch "${keyfile_dir}/keyfile"
chmod 600 "${keyfile_dir}/keyfile"
ensure_certs_symlinks
# Ensure that the user has an entry in /etc/passwd
current_uid=$(id -u)
declare -r current_uid
if ! grep -q "${current_uid}" /etc/passwd ; then
# Adding it here to avoid panics in the automation agent
sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd
echo "mongodb:x:$(id -u):$(id -g):,,,:/mongodb-automation:/bin/bash" >> /tmp/passwd
export LD_PRELOAD=libnss_wrapper.so
export NSS_WRAPPER_PASSWD=/tmp/passwd
export NSS_WRAPPER_GROUP=/etc/group
fi
# Create a symlink, after the volumes have been mounted
# If the journal directory already exists (this could be the migration of the existing MongoDB database) - we need
# to copy it to the correct location first and remove a directory
if [[ -d /data/journal ]] && [[ ! -L /data/journal ]]; then
script_log "The journal directory /data/journal already exists - moving its content to /journal"
if [[ $(ls -1 /data/journal | wc -l) -gt 0 ]]; then
mv /data/journal/* /journal
fi
rm -rf /data/journal
fi
ln -sf /journal /data/
script_log "Created symlink: /data/journal -> $(readlink -f /data/journal)"
# If it is a migration of the existing MongoDB - then there could be a mongodb.log in a default location -
# let's try to copy it to a new directory
if [[ -f /data/mongodb.log ]] && [[ ! -f "${MMS_LOG_DIR}/mongodb.log" ]]; then
script_log "The mongodb log file /data/mongodb.log already exists - moving it to ${MMS_LOG_DIR}"
mv /data/mongodb.log ${MMS_LOG_DIR}
fi
base_url="${BASE_URL-}" # If unassigned, set to empty string to avoid set-u errors
base_url="${base_url%/}" # Remove any accidentally defined trailing slashes
declare -r base_url
# Download the Automation Agent from Ops Manager
# Note, that it will be skipped if the agent is supposed to be run in headless mode
if [[ -n "${base_url}" ]]; then
download_agent
fi
AGENT_VERSION="$(cat ${MMS_HOME}/files/agent-version)"
# Start the Automation Agent
agentOpts=(
"-mmsGroupId" "${GROUP_ID-}"
"-pidfilepath" "${MMS_HOME}/mongodb-mms-automation-agent.pid"
"-maxLogFileDurationHrs" "24"
"-logLevel" "${LOG_LEVEL:-INFO}"
"-logFile" "${MMS_LOG_DIR}/automation-agent.log"
)
script_log "Automation Agent version: ${AGENT_VERSION}"
# this is the version of Automation Agent which has fixes for health file bugs
set +e
compare_versions "${AGENT_VERSION}" 10.2.3.5866-1
if [[ $? -le 1 ]]; then
agentOpts+=("-healthCheckFilePath" "${MMS_LOG_DIR}/agent-health-status.json")
fi
set -e
if [[ -n "${base_url}" ]]; then
agentOpts+=("-mmsBaseUrl" "${base_url}")
else
agentOpts+=("-cluster" "${cluster_config_file}")
# we need to open the web server on localhost even though we don't use it - otherwise Agent doesn't
# produce status information at all (we need it in health file)
agentOpts+=("-serveStatusPort" "5000")
script_log "Mongodb Agent is configured to run in \"headless\" mode using local config file"
fi
if [[ -n "${HTTP_PROXY-}" ]]; then
agentOpts+=("-httpProxy" "${HTTP_PROXY}")
fi
if [[ -n "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE-}" ]]; then
agentOpts+=("-sslTrustedMMSServerCertificate" "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE}")
fi
if [[ "${SSL_REQUIRE_VALID_MMS_CERTIFICATES-}" != "false" ]]; then
# Only set this option when valid certs are required. The default is false
agentOpts+=("-sslRequireValidMMSServerCertificates")
fi
script_log "Launching automation agent with following arguments: ${agentOpts[*]} -mmsApiKey ${AGENT_API_KEY+<hidden>}"
agentOpts+=("-mmsApiKey" "${AGENT_API_KEY-}")
# Note, that we do logging in subshell - this allows us to save the сorrect PID to variable (not the logging one)
"${MMS_HOME}/files/mongodb-mms-automation-agent" "${agentOpts[@]}" 2>> "${MMS_LOG_DIR}/automation-agent-stderr.log" > >(json_log "automation-agent-stdout") &
agentPid=$!
trap cleanup SIGTERM
# Note that we don't care about orphan processes as they will die together with container in case of any troubles
# tail's -F flag is equivalent to --follow=name --retry. Should we track log rotation events?
AGENT_VERBOSE_LOG="${MMS_LOG_DIR}/automation-agent-verbose.log" && touch "${AGENT_VERBOSE_LOG}"
AGENT_STDERR_LOG="${MMS_LOG_DIR}/automation-agent-stderr.log" && touch "${AGENT_STDERR_LOG}"
MONGODB_LOG="${MMS_LOG_DIR}/mongodb.log" && touch "${MONGODB_LOG}"
tail -F "${AGENT_VERBOSE_LOG}" 2> /dev/null | json_log 'automation-agent-verbose' &
tail -F "${AGENT_STDERR_LOG}" 2> /dev/null | json_log 'automation-agent-stderr' &
tail -F "${MONGODB_LOG}" 2> /dev/null | json_log 'mongodb' &
wait
scripts/probe.sh deleted 100755 → 0
View file @ fe6770d7
#!/bin/bash
agent_pid=/mongodb-automation/mongodb-mms-automation-agent.pid
check_agent_pid () {
# the agent PID must exists always
# it it does not exists, we assume it is being updated
# so we have a failure threshold of a few minutes.
[ -f $agent_pid ]
}
baby_container () {
# returns 0 if host's uptime is less than 1 hour
# To check if container uptime is less than 1 hour,
# we check for how long the pid1 process has
# been running.
pid1_alive_secs=$(ps -o etimes= -p 1)
pid1_alive_mins=$((pid1_alive_secs / 60))
[ $pid1_alive_mins -lt 60 ]
}
check_mongod_alive () {
pgrep --exact 'mongod'
}
check_mongos_alive () {
pgrep --exact 'mongos'
}
check_mongo_process_alive () {
# the mongod process pid might not always exist
# 1. when the container is being created the mongod package needs to be
# downloaded. the agent will wait for 1 hour before giving up.
# 2. the mongod process might be getting updated, we'll set a
# failureThreshold on the livenessProbe to a few minutes before we
# give up.
baby_container || check_mongod_alive || check_mongos_alive
}
check_agent_pid && check_mongo_process_alive
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

UNCLASSIFIED