Merge branch 'release-gitlab-13-12-0' into 'development'

Update GitLab to the 13.12.0 Feature Release

See merge request !76

Dockerfile

-ARG GITLAB_VERSION=v13.11.2-ubi8
+ARG GITLAB_VERSION=v13.12.0-ubi8
 
 ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082
 ARG BASE_IMAGE=gitlab/gitlab/gitlab-rails
-ARG BASE_TAG=13.11.2
+ARG BASE_TAG=13.12.0
 
 ARG RAILS_IMAGE=${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
 
@@ -11,6 +11,7 @@ FROM ${RAILS_IMAGE}
 ARG GITLAB_VERSION
 ARG GITLAB_USER=git
 ARG DNF_OPTS
+ENV LIBDIR ${LIBDIR:-"/usr/lib64"}
 
 ADD gitlab-task-runner-ee.tar.gz /
 ADD gitlab-python.tar.gz /

build-scripts/build.sh

@@ -4,7 +4,7 @@
 
 set -euxo pipefail
 
-TAG=${1:-13.11.2}
+TAG=${1:-13.12.0}
 REPOSITORY=${2:-}
 DOCKER_OPTS=${DOCKER_OPTS:-""}
 

hardening_manifest.yaml

@@ -5,12 +5,12 @@ name: "gitlab/gitlab/gitlab-task-runner"
 # The most specific version should be the first tag and will be shown
 # on ironbank.dsop.io
 tags:
-  - "13.11.2"
+  - "13.12.0"
   - "latest"
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "gitlab/gitlab/gitlab-rails"
-  BASE_TAG: "13.11.2"
+  BASE_TAG: "13.12.0"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "Gitlab Task Runner"
@@ -22,7 +22,7 @@ labels:
   org.opencontainers.image.url: "https://about.gitlab.com/"
   ## Name of the distributing entity, organization or individual
   org.opencontainers.image.vendor: "Gitlab"
-  org.opencontainers.image.version: "13.11.2"
+  org.opencontainers.image.version: "13.12.0"
   ## Keywords to help with search (ex. "cicd,gitops,golang")
   mil.dso.ironbank.image.keywords: "gitlab, git, gitops"
   ## This value can be "opensource" or "commercial"
@@ -43,13 +43,13 @@ maintainers:
     username: "alfontaine"
     email: "alan.fontaine@centauricorp.com"
 resources: 
-  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v13.11.2-ubi8/gitlab-task-runner-ee.tar.gz"
+  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v13.12.0-ubi8/gitlab-task-runner-ee.tar.gz"
     filename: "gitlab-task-runner-ee.tar.gz"
     validation:
       type: "sha256"
-      value: "c3414b48294616b28febb71473f1c15f00a76b8c0f4108a1276bd866aecca971"
-  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v13.11.2-ubi8/gitlab-python.tar.gz"
+      value: "13bd77fe4a0077119138c7e7ddf0cbbf7d247b87c503f289f6140e1f413b524c"
+  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v13.12.0-ubi8/gitlab-python.tar.gz"
     filename: "gitlab-python.tar.gz"
     validation:
       type: "sha256"
-      value: "1199b25cd6f32816c584e2dd275556e0e98f0612bdd51a55b18091328ea3f66b"
+      value: "1326de3e530204bf98181de17d197ddb47aa4c429a4da9c318dfff9180df34ed"

scripts/bin/backup-utility

 #!/bin/bash
 set -e
 
-ACTION="backup"
+ACTION=""
 export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups}
 export BACKUP_BACKEND=${BACKUP_BACKEND-s3}
 S3_CMD_BACKUP_OPTION=""
@@ -9,7 +9,7 @@ S3_CMD_BACKUP_OPTION=""
 rails_dir=/srv/gitlab
 backups_path=$rails_dir/tmp/backups
 backup_tars_path=$rails_dir/tmp/backup_tars
-object_storage_backends=( registry uploads artifacts lfs packages external_diffs terraform_state )
+object_storage_backends=( registry uploads artifacts lfs packages external_diffs terraform_state pages )
 
 skipping_backup_for=()
 
@@ -17,7 +17,7 @@ function usage()
 {
   cat << HEREDOC
 
-   Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] [--s3config CONFIG]
+   Usage: backup-utility [--restore|--cleanup] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] [--s3config CONFIG]
 
    Options:
      -h, --help                             Show this help message and exit.
@@ -36,6 +36,10 @@ function usage()
                                             Special config file for s3cmd (see: https://s3tools.org/usage)
      --storage-class CLASSNAME              Pass this storage class to the gcs or s3cmd for more cost-efficient
                                             storage of backups.
+     --maximum-backups N                    Only keep the most recent N number of backups, deleting others after success.
+                                            Requires s3config credentials to be able to list and delete objects.
+     --cleanup                              Run the backup cleanup without creating a new backup. Can be used with the 
+                                            'maximum-backups' option to clean old remote backups.
 HEREDOC
 }
 
@@ -100,9 +104,55 @@ function get_backup_name(){
   fi
 }
 
+function get_existing_backups(){
+  # This will only match backups with the same naming convention as backups generated by this script
+  # Example: TIMESTAMP_YYYY_MM_DD_VERSION_gitlab_backup.tar
+  case $BACKUP_BACKEND in
+    s3)
+      existing_backups=($(s3cmd ls s3://$BACKUP_BUCKET_NAME --rinclude '^\d{10}_\d{4}_\d{2}_\d{2}_.+_gitlab_backup.tar$' | awk '{print $4}' | LC_ALL=C sort))
+      ;;
+    gcs)
+      # Note: gsutil doesn't support regex, so we need to try to match the prefix as best we can with wildcards
+      # https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames#other-wildcard-characters
+      existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
+      ;;
+    *)
+      echo "Unknown backend for backup: ${BACKUP_BACKEND}"
+      exit 1
+      ;;
+  esac
+}
+
+function remove_backup(){
+  local backup_to_remove=$1
+  if [ "${BACKUP_BACKEND}" = "s3" ]; then
+    s3cmd ${S3_CMD_BACKUP_OPTION} del ${backup_to_remove} > /dev/null
+  elif [ "${BACKUP_BACKEND}" = "gcs" ]; then
+    gsutil rm ${backup_to_remove} > /dev/null
+  else
+    echo "Unknown backend for backup: ${BACKUP_BACKEND}"
+    exit 1
+  fi
+}
+
 function cleanup(){
   rm -rf $backups_path/*
   rm -rf $backup_tars_path/*
+
+  if [ -n "$MAXIMUM_BACKUPS" ]; then
+    get_existing_backups
+
+    echo "Found ${#existing_backups[@]} existing backups. Maximum allowed is $MAXIMUM_BACKUPS"
+    if [ ${#existing_backups[@]} -gt $MAXIMUM_BACKUPS ]; then
+      i=0
+      while [ $i -lt $(expr ${#existing_backups[@]} - $MAXIMUM_BACKUPS) ]; do
+        echo "Deleting old backup ${existing_backups[$i]}"
+        remove_backup ${existing_backups[$i]}
+        ((++i))
+      done
+    fi
+    echo "[DONE] Finished pruning old backups"
+  fi
 }
 
 function write_backup_info(){
@@ -134,7 +184,7 @@ function get_skipped(){
 
 function backup(){
   backup_name=$(get_backup_name)
-  mkdir -p $backup_tars_path
+  mkdir -p $backup_tars_path $backups_path
 
   if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then
     gitlab-rake gitlab:backup:db:create
@@ -260,7 +310,12 @@ do
       shift
       ;;
     --restore)
+      if [ -z "$ACTION" ]; then
         ACTION="restore"
+      else 
+        echo "Only one action at a time is supported"
+        exit 1
+      fi
       shift
       ;;
     --rsyncable)
@@ -277,6 +332,24 @@ do
       shift
       shift
       ;;
+    --maximum-backups)
+      export MAXIMUM_BACKUPS="$2"
+      if ! [[ $MAXIMUM_BACKUPS =~ ^-?[0-9]+$ ]]; then 
+        echo "Value specified for --maximum-backups must be an integer. Got: ${MAXIMUM_BACKUPS}"
+        exit 1
+      fi
+      shift
+      shift
+      ;;
+    --cleanup)
+      if [ -z "$ACTION" ]; then
+        ACTION="cleanup"
+      else 
+        echo "Only one action at a time is supported"
+        exit 1
+      fi
+      shift
+      ;;
     *)
       usage
       echo "Unexpected parameter: $key"
@@ -287,6 +360,9 @@ done
 
 if [ "$ACTION" = "restore" ]; then
   restore
-elif [ "$ACTION" = "backup" ]; then
+elif [ "$ACTION" = "cleanup" ]; then
+  cleanup
+elif [ -z "$ACTION" ]; then
+  ACTION="backup"
   backup
 fi