UNCLASSIFIED

Skip to content

GitLab

Commit 5bf9fe25 authored by thomas.shepherd's avatar thomas.shepherd
Browse files

Merge branch 'development' into 'master' 

Merge Development into Master

See merge request !11
parents 3f41bcbf 9ab81dbb
Pipeline #327706 failed with stages
in 2 minutes and 40 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 99 additions and 374 deletions
Dockerfile
View file @ 5bf9fe25
#
# Base Template Dockerfile for Database Image.
#
ARG BASE_REGISTRY=registry1.dsop.io
ARG BASE_IMAGE=redhat/ubi/ubi7
ARG BASE_TAG=7.8
ARG BASE_IMAGE=redhat/ubi/ubi8
ARG BASE_TAG=8.3
FROM quay.io/mongodb/mongodb-enterprise-database-ubi:1.5.3 as official
FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
ARG RELEASE=1.5.3
ARG TARBALL=mongodb-enterprise-operator-binaries-release-${RELEASE}.tar.gz
ENV MMS_HOME /mongodb-automation
ENV MMS_LOG_DIR /var/log/mongodb-mms-automation
ENV MDB_DIR /var/lib/mongodb-mms-automation/downloads
ARG MDB_URL
ARG BINARY_NAME
ARG AA_DOWNLOAD_URL
ARG AA_VERSION
LABEL name="MongoDB Enterprise Database" \
version="1.5.3" \
summary="MongoDB Enterprise Database Image" \
description="MongoDB Enterprise Database Image" \
vendor="MongoDB" \
release="1" \
maintainer="support@mongodb.com"
COPY LICENSE /licenses/mongodb-enterprise-database
RUN yum update -y && yum install \
libpcap \
RUN yum update -y && rm -rf /var/cache/yum \
# these are the packages needed for the agent
&& yum install -y --disableplugin=subscription-manager \
hostname \
nss_wrapper --exclude perl-IO-Socket-SSL \
procps \
# these are the packages needed for MongoDB
# (https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ "RHEL/CentOS 8" tab)
&& yum install -y --disableplugin=subscription-manager \
cyrus-sasl \
cyrus-sasl-gssapi \
cyrus-sasl-plain \
krb5-libs \
libcurl \
lm_sensors-libs \
net-snmp \
net-snmp-agent-libs \
rpm-libs \
tcp_wrappers-libs \
nss_wrapper; \
yum clean all; \
ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2
openldap \
openssl \
jq \
# mongodb enterprise expects this library /usr/lib64/libsasl2.so.2 but
# cyrus-sasl creates it in /usr/lib64/libsasl2.so.3 instead
&& ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2
# TODO: remove once database is using init container
COPY scripts/ "${MMS_HOME}"/files/
# Copy all the required scripts from the official database image
COPY --from=official "${MMS_HOME}" ${MMS_HOME}/
# Copy readinessprobe and set the required permissions
WORKDIR /opt
COPY ${TARBALL} .
RUN tar -zxf ./${TARBALL} \
&& mv ./readinessprobe "${MMS_HOME}"/files/ \
&& rm -rfv /opt/* /var/cache/yum \
&& mkdir -p "${MMS_LOG_DIR}" \
&& chmod 0775 "${MMS_LOG_DIR}" \
&& mkdir -p /var/lib/mongodb-mms-automation \
&& chmod 0775 /var/lib/mongodb-mms-automation \
&& mkdir -p /data \
&& chmod 0775 /data \
&& mkdir -p /journal \
&& chmod 0775 /journal \
&& chmod -R 0775 "${MMS_HOME}"
# Set the required perms
RUN mkdir -p "${MMS_LOG_DIR}" \
&& chmod 0775 "${MMS_LOG_DIR}" \
&& mkdir -p /var/lib/mongodb-mms-automation \
&& chmod 0775 /var/lib/mongodb-mms-automation \
&& mkdir -p /data \
&& chmod 0775 /data \
&& mkdir -p /journal \
&& chmod 0775 /journal \
&& mkdir -p "${MMS_HOME}" \
&& chmod -R 0775 "${MMS_HOME}"
# USER needs to be set for this image to pass RedHat verification. Some customers have these requirements as well
# It does not matter what number it is, as long as it is set to something.
......@@ -62,9 +58,7 @@ RUN tar -zxf ./${TARBALL} \
# and the number in this configuration is not relevant.
USER 2000
# TODO: make entrypoint be a sleep infinity once MDB is using init container
ENTRYPOINT ["/mongodb-automation/files/agent-launcher.sh"]
# TODO: switch this to actually run the probe
HEALTHCHECK --timeout=30s CMD ls "${MMS_HOME}"/files/probe.sh || exit 1
Jenkinsfile deleted 100644 → 0
View file @ 3f41bcbf
@Library('DCCSCR@master') _
dccscrPipeline(version: "1.5.3")
README.md
View file @ 5bf9fe25
......@@ -7,9 +7,9 @@
For licensing and technical information, please use the following contact information:
Anton Hoffman, EAE USAF
781-996-8860
anton.hoffman@mongodb.com
Keegan Wetzel, EAE USAF
708-256-7904
keegan.wetzel@mongodb.com
Please also CC: publicsector@mongodb.com
......
download.json deleted 100755 → 0
View file @ 3f41bcbf
{
"resources": [
{
"url": "https://s3.amazonaws.com/ops-manager-kubernetes-build/releases/mongodb-enterprise-operator-binaries-release-1.5.3.tar.gz",
"filename": "mongodb-enterprise-operator-binaries-release-1.5.3.tar.gz",
"validation": {
"type": "sha256",
"value": "251bf6aa9e6deeba3bd5366228ce81f6322c6a15aa8b07558cb23092d73333b8"
}
}
]
}
\ No newline at end of file
hardening_manifest.yaml 0 → 100644
View file @ 5bf9fe25
---
apiVersion: v1
# The repository name in registry1, excluding /ironbank/
name: "mongodb/mongodb-enterprise/mongodb-enterprise-database"
# List of tags to push for the repository in registry1
# The most specific version should be the first tag and will be shown
# on ironbank.dsop.io
tags:
- "1.5.3"
- "latest"
# Build args passed to Dockerfile ARGs
args:
BASE_IMAGE: "redhat/ubi/ubi8"
BASE_TAG: "8.3"
# Docker image labels
labels:
org.opencontainers.image.title: "mongodb-enterprise-database"
## Human-readable description of the software packaged in the image
org.opencontainers.image.description: "Base Database Image used in conjunction with MongoDB Enterprise Kubernetes Operator"
## License(s) under which contained software is distributed
org.opencontainers.image.licenses: "license"
## URL to find more information on the image
org.opencontainers.image.url: "https://docs.mongodb.com/kubernetes-operator/stable/"
## Name of the distributing entity, organization or individual
org.opencontainers.image.vendor: "MongoDB"
org.opencontainers.image.version: "1.5.3"
## Keywords to help with search (ex. "cicd,gitops,golang")
mil.dso.ironbank.image.keywords: "mongodb,nosql,json,database"
## This value can be "opensource" or "commercial"
mil.dso.ironbank.image.type: "commercial"
## Product the image belongs to for grouping multiple images
mil.dso.ironbank.product.name: "mongodb-enterprise"
# List of resources to make available to the offline build context
resources:
- tag: quay.io/mongodb/mongodb-enterprise-database-ubi:1.5.3
url: docker://quay.io/mongodb/mongodb-enterprise-database-ubi@sha256:8f2734cea7fc31f829a0f8752be5bfd1c69b3d2e59e677bb23f685aac7f58eb9
# List of project maintainers
# FIXME: Fill in the following details for the current container owner in the whitelist
# FIXME: Include any other vendor information if applicable
maintainers:
- email: "jonathan.janos@mongodb.com"
# The name of the current container owner
name: "Jonathan Janos"
# The gitlab username of the current container owner
username: "jonathan.janos"
- email: "klepal_alexander@bah.com"
name: "Alexander Klepal"
username: "alexander.klepal"
cht_member: true
scripts/agent-launcher-lib.sh deleted 100755 → 0
View file @ 3f41bcbf
#!/usr/bin/env bash
# This is a file containing all the functions which may be needed for other shell scripts
# see if jq is available for json logging
use_jq="$(command -v jq)"
# log stdout as structured json with given log type
json_log () {
if [ "$use_jq" ]; then
jq --unbuffered --null-input -c --raw-input "inputs | {\"logType\": \"$1\", \"contents\": .}";
else
echo "$1"
fi
}
# log a given message in json format
script_log () {
echo "$1" | json_log 'agent-launcher-script'
}
# the function reacting on SIGTERM command sent by the container on its shutdown. Makes sure all processes started (including
# mongodb) receive the signal. For MongoDB this results in graceful shutdown of replication (starting from 4.0.9) which may
# take some time. The script waits for all the processes to finish, otherwise the container would terminate as Kubernetes
# waits only for the process with pid #1 to end
cleanup () {
# Important! Keep this in sync with DefaultPodTerminationPeriodSeconds constant from constants.go
termination_timeout_seconds=600
script_log "Caught SIGTERM signal. Passing the signal to the automation agent and the mongod processes."
kill -15 "$agentPid"
wait "$agentPid"
mongoPid="$(cat /data/mongod.lock)"
kill -15 "$mongoPid"
script_log "Waiting until mongod process is shutdown. Note, that if mongod process fails to shutdown in the time specified by the 'terminationGracePeriodSeconds' property (default $termination_timeout_seconds seconds) then the container will be killed by Kubernetes."
# dev note: we cannot use 'wait' for the external processes, seems the spinning loop is the best option
while [ -e "/proc/$mongoPid" ]; do sleep 0.1; done
script_log "Mongod and automation agent processes are shutdown"
}
# ensure_certs_symlinks function checks if certificates and CAs are mounted and creates symlinks to them
ensure_certs_symlinks () {
# the paths inside the pod. Move to parameters if multiple usage is needed
secrets_dir="/var/lib/mongodb-automation/secrets"
custom_ca_dir="${secrets_dir}/ca"
pod_secrets_dir="/mongodb-automation"
if [ -d "${secrets_dir}/certs" ]; then
script_log "Found certificates in the host, will symlink to where the automation agent expects them to be"
podname=$(hostname)
if [ ! -f "${secrets_dir}/certs/${podname}-pem" ]; then
script_log "PEM Certificate file does not exist in ${secrets_dir}/certs/${podname}-pem. Check the Secret object with certificates is well formed."
exit 1
fi
ln -s "${secrets_dir}/certs/${podname}-pem" "${pod_secrets_dir}/server.pem"
fi
if [ -d "${custom_ca_dir}" ]; then
if [ -f "${custom_ca_dir}/ca-pem" ]; then
script_log "Using CA file provided by user"
ln -s "${custom_ca_dir}/ca-pem" "${pod_secrets_dir}/ca.pem"
else
script_log "Could not find CA file. The name of the entry on the Secret object should be 'ca-pem'"
exit 1
fi
else
script_log "Using Kubernetes CA file"
ln -s "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" "${pod_secrets_dir}/ca.pem"
fi
}
# download_agent function downloads and unpacks the Mongodb Agent
download_agent () {
script_log "Downloading a Mongodb Agent from ${base_url}"
pushd /tmp >/dev/null
curl_opts=(
"${base_url}/download/agent/automation/mongodb-mms-automation-agent-latest.linux_x86_64.tar.gz"
"--location" "--silent" "--retry" "3" "--fail" "-v"
"--output" "automation-agent.tar.gz"
)
if [ "${SSL_REQUIRE_VALID_MMS_CERTIFICATES-}" = "false" ]; then
# If we are not expecting valid certs, `curl` should be run with `--insecure` option.
# The default is NOT to accept insecure connections.
curl_opts+=("--insecure")
fi
if [ -n "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE-}" ]; then
curl_opts+=("--cacert" "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE}")
fi
if ! curl "${curl_opts[@]}" &> "${MMS_LOG_DIR}/agent-launcher-script.log"; then
script_log "Error while downloading the Mongodb agent"
cat "${MMS_LOG_DIR}/agent-launcher-script.log" | json_log 'agent-launcher-script'
exit 1
fi
script_log "The Mongodb Agent binary downloaded, unpacking"
tar -xzf automation-agent.tar.gz
AGENT_VERSION=$(find . -name mongodb-mms-automation-agent-* | awk -F"-" '{ print $5 }')
echo "${AGENT_VERSION}" > "${MMS_HOME}/files/agent-version"
mv mongodb-mms-automation-agent-*/mongodb-mms-automation-agent "${MMS_HOME}/files/"
chmod +x "${MMS_HOME}/files/mongodb-mms-automation-agent"
rm -rf automation-agent.tar.gz mongodb-mms-automation-agent-*.linux_x86_64
script_log "The Automation Agent was deployed at ${MMS_HOME}/files/mongodb-mms-automation-agent"
popd >/dev/null
}
#https://stackoverflow.com/a/4025065/614239
compare_versions () {
if [[ $1 == $2 ]]
then
return 0
fi
local IFS=.
local i ver1=($1) ver2=($2)
# fill empty fields in ver1 with zeros
for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
do
ver1[i]=0
done
for ((i=0; i<${#ver1[@]}; i++))
do
if [[ -z ${ver2[i]} ]]
then
# fill empty fields in ver2 with zeros
ver2[i]=0
fi
if ((10#${ver1[i]} > 10#${ver2[i]}))
then
return 1
fi
if ((10#${ver1[i]} < 10#${ver2[i]}))
then
return 2
fi
done
return 0
}
scripts/agent-launcher.sh deleted 100755 → 0
View file @ 3f41bcbf
#!/usr/bin/env bash
set -o nounset
set -o errexit
set -o pipefail
source "${MMS_HOME}/files/agent-launcher-lib.sh"
# The path to the automation config file in case the agent is run in headless mode
cluster_config_file="/var/lib/mongodb-automation/cluster-config.json"
# file required by Automation Agents of authentication is enabled.
keyfile_dir="/var/lib/mongodb-mms-automation"
mkdir -p ${keyfile_dir}
touch "${keyfile_dir}/keyfile"
chmod 600 "${keyfile_dir}/keyfile"
ensure_certs_symlinks
# Ensure that the user has an entry in /etc/passwd
current_uid=$(id -u)
declare -r current_uid
if ! grep -q "${current_uid}" /etc/passwd ; then
# Adding it here to avoid panics in the automation agent
sed -e "s/^mongodb:/builder:/" /etc/passwd > /tmp/passwd
echo "mongodb:x:$(id -u):$(id -g):,,,:/mongodb-automation:/bin/bash" >> /tmp/passwd
export LD_PRELOAD=libnss_wrapper.so
export NSS_WRAPPER_PASSWD=/tmp/passwd
export NSS_WRAPPER_GROUP=/etc/group
fi
# Create a symlink, after the volumes have been mounted
# If the journal directory already exists (this could be the migration of the existing MongoDB database) - we need
# to copy it to the correct location first and remove a directory
if [[ -d /data/journal ]] && [[ ! -L /data/journal ]]; then
script_log "The journal directory /data/journal already exists - moving its content to /journal"
if [[ $(ls -1 /data/journal | wc -l) -gt 0 ]]; then
mv /data/journal/* /journal
fi
rm -rf /data/journal
fi
ln -sf /journal /data/
script_log "Created symlink: /data/journal -> $(readlink -f /data/journal)"
# If it is a migration of the existing MongoDB - then there could be a mongodb.log in a default location -
# let's try to copy it to a new directory
if [[ -f /data/mongodb.log ]] && [[ ! -f "${MMS_LOG_DIR}/mongodb.log" ]]; then
script_log "The mongodb log file /data/mongodb.log already exists - moving it to ${MMS_LOG_DIR}"
mv /data/mongodb.log ${MMS_LOG_DIR}
fi
base_url="${BASE_URL-}" # If unassigned, set to empty string to avoid set-u errors
base_url="${base_url%/}" # Remove any accidentally defined trailing slashes
declare -r base_url
# Download the Automation Agent from Ops Manager
# Note, that it will be skipped if the agent is supposed to be run in headless mode
if [[ -n "${base_url}" ]]; then
download_agent
fi
AGENT_VERSION="$(cat ${MMS_HOME}/files/agent-version)"
# Start the Automation Agent
agentOpts=(
"-mmsGroupId" "${GROUP_ID-}"
"-pidfilepath" "${MMS_HOME}/mongodb-mms-automation-agent.pid"
"-maxLogFileDurationHrs" "24"
"-logLevel" "${LOG_LEVEL:-INFO}"
"-logFile" "${MMS_LOG_DIR}/automation-agent.log"
)
script_log "Automation Agent version: ${AGENT_VERSION}"
# this is the version of Automation Agent which has fixes for health file bugs
set +e
compare_versions "${AGENT_VERSION}" 10.2.3.5866-1
if [[ $? -le 1 ]]; then
agentOpts+=("-healthCheckFilePath" "${MMS_LOG_DIR}/agent-health-status.json")
fi
set -e
if [[ -n "${base_url}" ]]; then
agentOpts+=("-mmsBaseUrl" "${base_url}")
else
agentOpts+=("-cluster" "${cluster_config_file}")
# we need to open the web server on localhost even though we don't use it - otherwise Agent doesn't
# produce status information at all (we need it in health file)
agentOpts+=("-serveStatusPort" "5000")
script_log "Mongodb Agent is configured to run in \"headless\" mode using local config file"
fi
if [[ -n "${HTTP_PROXY-}" ]]; then
agentOpts+=("-httpProxy" "${HTTP_PROXY}")
fi
if [[ -n "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE-}" ]]; then
agentOpts+=("-sslTrustedMMSServerCertificate" "${SSL_TRUSTED_MMS_SERVER_CERTIFICATE}")
fi
if [[ "${SSL_REQUIRE_VALID_MMS_CERTIFICATES-}" != "false" ]]; then
# Only set this option when valid certs are required. The default is false
agentOpts+=("-sslRequireValidMMSServerCertificates")
fi
script_log "Launching automation agent with following arguments: ${agentOpts[*]} -mmsApiKey ${AGENT_API_KEY+<hidden>}"
agentOpts+=("-mmsApiKey" "${AGENT_API_KEY-}")
# Note, that we do logging in subshell - this allows us to save the сorrect PID to variable (not the logging one)
"${MMS_HOME}/files/mongodb-mms-automation-agent" "${agentOpts[@]}" 2>> "${MMS_LOG_DIR}/automation-agent-stderr.log" > >(json_log "automation-agent-stdout") &
agentPid=$!
trap cleanup SIGTERM
# Note that we don't care about orphan processes as they will die together with container in case of any troubles
# tail's -F flag is equivalent to --follow=name --retry. Should we track log rotation events?
AGENT_VERBOSE_LOG="${MMS_LOG_DIR}/automation-agent-verbose.log" && touch "${AGENT_VERBOSE_LOG}"
AGENT_STDERR_LOG="${MMS_LOG_DIR}/automation-agent-stderr.log" && touch "${AGENT_STDERR_LOG}"
MONGODB_LOG="${MMS_LOG_DIR}/mongodb.log" && touch "${MONGODB_LOG}"
tail -F "${AGENT_VERBOSE_LOG}" 2> /dev/null | json_log 'automation-agent-verbose' &
tail -F "${AGENT_STDERR_LOG}" 2> /dev/null | json_log 'automation-agent-stderr' &
tail -F "${MONGODB_LOG}" 2> /dev/null | json_log 'mongodb' &
wait
scripts/probe.sh deleted 100755 → 0
View file @ 3f41bcbf
#!/bin/bash
agent_pid=/mongodb-automation/mongodb-mms-automation-agent.pid
check_agent_pid () {
# the agent PID must exists always
# it it does not exists, we assume it is being updated
# so we have a failure threshold of a few minutes.
[ -f $agent_pid ]
}
baby_container () {
# returns 0 if host's uptime is less than 1 hour
# To check if container uptime is less than 1 hour,
# we check for how long the pid1 process has
# been running.
pid1_alive_secs=$(ps -o etimes= -p 1)
pid1_alive_mins=$((pid1_alive_secs / 60))
[ $pid1_alive_mins -lt 60 ]
}
check_mongod_alive () {
pgrep --exact 'mongod'
}
check_mongos_alive () {
pgrep --exact 'mongos'
}
check_mongo_process_alive () {
# the mongod process pid might not always exist
# 1. when the container is being created the mongod package needs to be
# downloaded. the agent will wait for 1 hour before giving up.
# 2. the mongod process might be getting updated, we'll set a
# failureThreshold on the livenessProbe to a few minutes before we
# give up.
baby_container || check_mongod_alive || check_mongos_alive
}
check_agent_pid && check_mongo_process_alive
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

UNCLASSIFIED