Merge branch 'development' into 'master'

Development

See merge request !79

Dockerfile

-ARG GITLAB_VERSION=v13.11.2-ubi8
+ARG GITLAB_VERSION=v14.0.0-ubi8
 
 ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082
 ARG BASE_IMAGE=gitlab/gitlab/gitlab-rails
-ARG BASE_TAG=13.11.2
+ARG BASE_TAG=14.0.0
 
 ARG RAILS_IMAGE=${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
 
@@ -11,6 +11,7 @@ FROM ${RAILS_IMAGE}
 ARG GITLAB_VERSION
 ARG GITLAB_USER=git
 ARG DNF_OPTS
+ENV LIBDIR ${LIBDIR:-"/usr/lib64"}
 
 ADD gitlab-task-runner-ee.tar.gz /
 ADD gitlab-python.tar.gz /

build-scripts/build.sh

@@ -4,7 +4,7 @@
 
 set -euxo pipefail
 
-TAG=${1:-13.11.2}
+TAG=${1:-14.0.0}
 REPOSITORY=${2:-}
 DOCKER_OPTS=${DOCKER_OPTS:-""}
 

hardening_manifest.yaml

@@ -5,12 +5,12 @@ name: "gitlab/gitlab/gitlab-task-runner"
 # The most specific version should be the first tag and will be shown
 # on ironbank.dsop.io
 tags:
-  - "13.11.2"
+  - "14.0.0"
   - "latest"
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "gitlab/gitlab/gitlab-rails"
-  BASE_TAG: "13.11.2"
+  BASE_TAG: "14.0.0"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "Gitlab Task Runner"
@@ -22,7 +22,7 @@ labels:
   org.opencontainers.image.url: "https://about.gitlab.com/"
   ## Name of the distributing entity, organization or individual
   org.opencontainers.image.vendor: "Gitlab"
-  org.opencontainers.image.version: "13.11.2"
+  org.opencontainers.image.version: "14.0.0"
   ## Keywords to help with search (ex. "cicd,gitops,golang")
   mil.dso.ironbank.image.keywords: "gitlab, git, gitops"
   ## This value can be "opensource" or "commercial"
@@ -43,13 +43,13 @@ maintainers:
     username: "alfontaine"
     email: "alan.fontaine@centauricorp.com"
 resources: 
-  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v13.11.2-ubi8/gitlab-task-runner-ee.tar.gz"
+  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v14.0.0-ubi8/gitlab-task-runner-ee.tar.gz"
     filename: "gitlab-task-runner-ee.tar.gz"
     validation:
       type: "sha256"
-      value: "c3414b48294616b28febb71473f1c15f00a76b8c0f4108a1276bd866aecca971"
-  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v13.11.2-ubi8/gitlab-python.tar.gz"
+      value: "bf8db92af98abe995f4411a73acede96a01da96b5fee5915173a05fe6605d15d"
+  - url: "https://gitlab-ubi.s3.amazonaws.com/ubi8-build-dependencies-v14.0.0-ubi8/gitlab-python.tar.gz"
     filename: "gitlab-python.tar.gz"
     validation:
       type: "sha256"
-      value: "1199b25cd6f32816c584e2dd275556e0e98f0612bdd51a55b18091328ea3f66b"
+      value: "48068e0e17bb7f3083861f050cad709115360c072e89cce31566def73760c723"

scripts/bin/backup-utility

 #!/bin/bash
 set -e
 
-ACTION="backup"
+ACTION=""
 export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups}
 export BACKUP_BACKEND=${BACKUP_BACKEND-s3}
 S3_CMD_BACKUP_OPTION=""
@@ -9,7 +9,7 @@ S3_CMD_BACKUP_OPTION=""
 rails_dir=/srv/gitlab
 backups_path=$rails_dir/tmp/backups
 backup_tars_path=$rails_dir/tmp/backup_tars
-object_storage_backends=( registry uploads artifacts lfs packages external_diffs terraform_state )
+object_storage_backends=( registry uploads artifacts lfs packages external_diffs terraform_state pages )
 
 skipping_backup_for=()
 
@@ -17,7 +17,7 @@ function usage()
 {
   cat << HEREDOC
 
-   Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] [--s3config CONFIG]
+   Usage: backup-utility [--restore|--cleanup] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] [--s3config CONFIG]
 
    Options:
      -h, --help                             Show this help message and exit.
@@ -36,6 +36,10 @@ function usage()
                                             Special config file for s3cmd (see: https://s3tools.org/usage)
      --storage-class CLASSNAME              Pass this storage class to the gcs or s3cmd for more cost-efficient
                                             storage of backups.
+     --maximum-backups N                    Only keep the most recent N number of backups, deleting others after success.
+                                            Requires s3config credentials to be able to list and delete objects.
+     --cleanup                              Run the backup cleanup without creating a new backup. Can be used with the 
+                                            'maximum-backups' option to clean old remote backups.
 HEREDOC
 }
 
@@ -100,9 +104,55 @@ function get_backup_name(){
   fi
 }
 
+function get_existing_backups(){
+  # This will only match backups with the same naming convention as backups generated by this script
+  # Example: TIMESTAMP_YYYY_MM_DD_VERSION_gitlab_backup.tar
+  case $BACKUP_BACKEND in
+    s3)
+      existing_backups=($(s3cmd ${S3_CMD_BACKUP_OPTION} ls s3://$BACKUP_BUCKET_NAME --rinclude '^\d{10}_\d{4}_\d{2}_\d{2}_.+_gitlab_backup.tar$' | awk '{print $4}' | LC_ALL=C sort))
+      ;;
+    gcs)
+      # Note: gsutil doesn't support regex, so we need to try to match the prefix as best we can with wildcards
+      # https://cloud.google.com/storage/docs/gsutil/addlhelp/WildcardNames#other-wildcard-characters
+      existing_backups=($(gsutil ls gs://$BACKUP_BUCKET_NAME/[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]_[0-9][0-9][0-9][0-9]_[0-9][0-9]_[0-9][0-9]_\*_gitlab_backup.tar | LC_ALL=C sort))
+      ;;
+    *)
+      echo "Unknown backend for backup: ${BACKUP_BACKEND}"
+      exit 1
+      ;;
+  esac
+}
+
+function remove_backup(){
+  local backup_to_remove=$1
+  if [ "${BACKUP_BACKEND}" = "s3" ]; then
+    s3cmd ${S3_CMD_BACKUP_OPTION} del ${backup_to_remove} > /dev/null
+  elif [ "${BACKUP_BACKEND}" = "gcs" ]; then
+    gsutil rm ${backup_to_remove} > /dev/null
+  else
+    echo "Unknown backend for backup: ${BACKUP_BACKEND}"
+    exit 1
+  fi
+}
+
 function cleanup(){
   rm -rf $backups_path/*
   rm -rf $backup_tars_path/*
+
+  if [ -n "$MAXIMUM_BACKUPS" ]; then
+    get_existing_backups
+
+    echo "Found ${#existing_backups[@]} existing backups. Maximum allowed is $MAXIMUM_BACKUPS"
+    if [ ${#existing_backups[@]} -gt $MAXIMUM_BACKUPS ]; then
+      i=0
+      while [ $i -lt $(expr ${#existing_backups[@]} - $MAXIMUM_BACKUPS) ]; do
+        echo "Deleting old backup ${existing_backups[$i]}"
+        remove_backup ${existing_backups[$i]}
+        ((++i))
+      done
+    fi
+    echo "[DONE] Finished pruning old backups"
+  fi
 }
 
 function write_backup_info(){
@@ -134,7 +184,7 @@ function get_skipped(){
 
 function backup(){
   backup_name=$(get_backup_name)
-  mkdir -p $backup_tars_path
+  mkdir -p $backup_tars_path $backups_path
 
   if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then
     gitlab-rake gitlab:backup:db:create
@@ -260,7 +310,12 @@ do
       shift
       ;;
     --restore)
-      ACTION="restore"
+      if [ -z "$ACTION" ]; then
+        ACTION="restore"
+      else 
+        echo "Only one action at a time is supported"
+        exit 1
+      fi
       shift
       ;;
     --rsyncable)
@@ -277,6 +332,24 @@ do
       shift
       shift
       ;;
+    --maximum-backups)
+      export MAXIMUM_BACKUPS="$2"
+      if ! [[ $MAXIMUM_BACKUPS =~ ^-?[0-9]+$ ]]; then 
+        echo "Value specified for --maximum-backups must be an integer. Got: ${MAXIMUM_BACKUPS}"
+        exit 1
+      fi
+      shift
+      shift
+      ;;
+    --cleanup)
+      if [ -z "$ACTION" ]; then
+        ACTION="cleanup"
+      else 
+        echo "Only one action at a time is supported"
+        exit 1
+      fi
+      shift
+      ;;
     *)
       usage
       echo "Unexpected parameter: $key"
@@ -287,6 +360,9 @@ done
 
 if [ "$ACTION" = "restore" ]; then
   restore
-elif [ "$ACTION" = "backup" ]; then
+elif [ "$ACTION" = "cleanup" ]; then
+  cleanup
+elif [ -z "$ACTION" ]; then
+  ACTION="backup"
   backup
 fi

scripts/lib/object_storage_backup.rb

@@ -41,7 +41,7 @@ class ObjectStorageBackup
     FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700)
 
     output, status = run_cmd(cmd)
-    failure_abort(output) unless status.zero?
+    failure_abort('creation of working directory', output) unless status.zero?
 
     # check the destiation for contents. Bucket may have been empty.
     if Dir.empty? "/srv/gitlab/tmp/#{@name}"
@@ -54,7 +54,7 @@ class ObjectStorageBackup
 
     cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . )
     output, status = run_cmd(cmd)
-    failure_abort(output) unless status.zero?
+    failure_abort('archive', output) unless status.zero?
 
     puts "done".green
   end
@@ -68,9 +68,9 @@ class ObjectStorageBackup
     puts "done".green
   end
 
-  def failure_abort(error_message)
+  def failure_abort(action, error_message)
     puts "[Error] #{error_message}".red
-    abort "Restore #{@name} failed"
+    abort "#{action} of #{@name} failed"
   end
 
   def upload_to_object_storage(source_path)
@@ -83,7 +83,7 @@ class ObjectStorageBackup
 
     output, status = run_cmd(cmd)
 
-    failure_abort(output) unless status.zero?
+    failure_abort('upload', output) unless status.zero?
   end
 
   def backup_existing
@@ -97,7 +97,7 @@ class ObjectStorageBackup
 
     output, status = run_cmd(cmd)
 
-    failure_abort(output) unless status.zero?
+    failure_abort('sync existing', output) unless status.zero?
   end
 
   def cleanup
@@ -107,7 +107,7 @@ class ObjectStorageBackup
       # Check if the bucket has any objects
       list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/)
       output, status = run_cmd(list_objects_cmd)
-      failure_abort(output) unless status.zero?
+      failure_abort('GCS ls', output) unless status.zero?
 
       # There are no objects in the bucket so skip the cleanup
       if output.length == 0
@@ -117,20 +117,20 @@ class ObjectStorageBackup
       cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*)
     end
     output, status = run_cmd(cmd)
-    failure_abort(output) unless status.zero?
+    failure_abort('bucket cleanup', output) unless status.zero?
   end
 
   def restore_from_backup
     extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}")
     FileUtils.mkdir_p(extracted_tar_path, mode: 0700)
 
-    failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path)
+    failure_abort('restore', "#{@local_tar_path} not found") unless File.exist?(@local_tar_path)
 
     untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path})
 
     output, status = run_cmd(untar_cmd)
 
-    failure_abort(output) unless status.zero?
+    failure_abort('un-archive', output) unless status.zero?
 
     Dir.glob("#{extracted_tar_path}/*").each do |file|
      upload_to_object_storage(file)