From 5598658cf63aecd290e1b0b98488610f7c7b95ee Mon Sep 17 00:00:00 2001 From: DJ Mountney Date: Mon, 9 Mar 2020 11:46:20 -0700 Subject: [PATCH] Refactor to new CHT pipeline pattern - Removes gitlab versioned folders prior to 12.8 - Unwraps the 12.8 folder into the root - Removes the prebuild.sh script in favor of download.json - Drops the build script for now - Update prepare script to use local downloaded artifact --- 12.3/Dockerfile | 28 -- 12.3/scripts/bin/backup-utility | 257 ----------------- 12.3/scripts/lib/object_storage_backup.rb | 149 ---------- 12.4/Dockerfile | 38 --- 12.4/scripts/bin/backup-utility | 257 ----------------- 12.4/scripts/bin/entrypoint.sh | 9 - 12.4/scripts/bin/gitlab-rails | 5 - 12.4/scripts/bin/gitlab-rake | 5 - 12.4/scripts/bin/object-storage-backup | 10 - 12.4/scripts/bin/object-storage-restore | 10 - 12.5/Dockerfile | 45 --- 12.5/README.md | 41 --- 12.5/build-scripts/build.sh | 41 --- 12.5/build-scripts/prepare.sh | 24 -- 12.5/scripts/bin/backup-utility | 257 ----------------- 12.5/scripts/bin/entrypoint.sh | 9 - 12.5/scripts/bin/gitlab-rails | 5 - 12.5/scripts/bin/gitlab-rake | 5 - 12.5/scripts/bin/object-storage-backup | 10 - 12.5/scripts/bin/object-storage-restore | 10 - 12.5/scripts/lib/object_storage_backup.rb | 144 ---------- 12.5/scripts/prebuild.sh | 57 ---- 12.6/Dockerfile | 47 ---- 12.6/LICENSE | 21 -- 12.6/README.md | 42 --- 12.6/build-scripts/build.sh | 41 --- 12.6/build-scripts/cleanup.sh | 5 - 12.6/build-scripts/prepare.sh | 24 -- 12.6/scripts/bin/backup-utility | 257 ----------------- 12.6/scripts/bin/entrypoint.sh | 9 - 12.6/scripts/bin/gitlab-rails | 5 - 12.6/scripts/bin/gitlab-rake | 5 - 12.6/scripts/bin/object-storage-backup | 10 - 12.6/scripts/bin/object-storage-restore | 10 - 12.6/scripts/lib/object_storage_backup.rb | 144 ---------- 12.6/scripts/prebuild.sh | 57 ---- 12.7/Dockerfile | 46 --- 12.7/LICENSE | 21 -- 12.7/README.md | 42 --- 12.7/build-scripts/build.sh | 41 --- 12.7/build-scripts/cleanup.sh | 5 - 12.7/build-scripts/prepare.sh | 24 -- 12.7/scripts/bin/entrypoint.sh | 9 - 12.7/scripts/bin/gitlab-rails | 5 - 12.7/scripts/bin/gitlab-rake | 5 - 12.7/scripts/bin/object-storage-backup | 10 - 12.7/scripts/bin/object-storage-restore | 10 - 12.7/scripts/lib/object_storage_backup.rb | 144 ---------- 12.7/scripts/prebuild.sh | 57 ---- 12.8/LICENSE | 21 -- 12.8/README.md | 42 --- 12.8/build-scripts/cleanup.sh | 5 - 12.8/build-scripts/prepare.sh | 24 -- 12.8/scripts/bin/backup-utility | 262 ------------------ 12.8/scripts/bin/entrypoint.sh | 9 - 12.8/scripts/bin/gitlab-rails | 5 - 12.8/scripts/bin/gitlab-rake | 5 - 12.8/scripts/bin/object-storage-backup | 10 - 12.8/scripts/bin/object-storage-restore | 10 - 12.8/scripts/lib/object_storage_backup.rb | 144 ---------- 12.8/scripts/prebuild.sh | 57 ---- 12.8/Dockerfile => Dockerfile | 8 +- 12.5/LICENSE => LICENSE | 0 README.md | 39 +++ .../build-scripts => build-scripts}/build.sh | 10 - .../cleanup.sh | 0 build-scripts/prepare.sh | 15 + download.json | 7 + {12.7/scripts => scripts}/bin/backup-utility | 0 {12.3/scripts => scripts}/bin/entrypoint.sh | 0 {12.3/scripts => scripts}/bin/gitlab-rails | 0 {12.3/scripts => scripts}/bin/gitlab-rake | 0 .../bin/object-storage-backup | 0 .../bin/object-storage-restore | 0 .../lib/object_storage_backup.rb | 0 75 files changed, 64 insertions(+), 3121 deletions(-) delete mode 100644 12.3/Dockerfile delete mode 100755 12.3/scripts/bin/backup-utility delete mode 100755 12.3/scripts/lib/object_storage_backup.rb delete mode 100644 12.4/Dockerfile delete mode 100755 12.4/scripts/bin/backup-utility delete mode 100755 12.4/scripts/bin/entrypoint.sh delete mode 100755 12.4/scripts/bin/gitlab-rails delete mode 100755 12.4/scripts/bin/gitlab-rake delete mode 100755 12.4/scripts/bin/object-storage-backup delete mode 100755 12.4/scripts/bin/object-storage-restore delete mode 100644 12.5/Dockerfile delete mode 100644 12.5/README.md delete mode 100755 12.5/build-scripts/build.sh delete mode 100755 12.5/build-scripts/prepare.sh delete mode 100755 12.5/scripts/bin/backup-utility delete mode 100755 12.5/scripts/bin/entrypoint.sh delete mode 100755 12.5/scripts/bin/gitlab-rails delete mode 100755 12.5/scripts/bin/gitlab-rake delete mode 100755 12.5/scripts/bin/object-storage-backup delete mode 100755 12.5/scripts/bin/object-storage-restore delete mode 100644 12.5/scripts/lib/object_storage_backup.rb delete mode 100755 12.5/scripts/prebuild.sh delete mode 100644 12.6/Dockerfile delete mode 100644 12.6/LICENSE delete mode 100644 12.6/README.md delete mode 100755 12.6/build-scripts/build.sh delete mode 100755 12.6/build-scripts/cleanup.sh delete mode 100755 12.6/build-scripts/prepare.sh delete mode 100755 12.6/scripts/bin/backup-utility delete mode 100755 12.6/scripts/bin/entrypoint.sh delete mode 100755 12.6/scripts/bin/gitlab-rails delete mode 100755 12.6/scripts/bin/gitlab-rake delete mode 100755 12.6/scripts/bin/object-storage-backup delete mode 100755 12.6/scripts/bin/object-storage-restore delete mode 100644 12.6/scripts/lib/object_storage_backup.rb delete mode 100755 12.6/scripts/prebuild.sh delete mode 100644 12.7/Dockerfile delete mode 100644 12.7/LICENSE delete mode 100644 12.7/README.md delete mode 100755 12.7/build-scripts/build.sh delete mode 100755 12.7/build-scripts/cleanup.sh delete mode 100755 12.7/build-scripts/prepare.sh delete mode 100755 12.7/scripts/bin/entrypoint.sh delete mode 100755 12.7/scripts/bin/gitlab-rails delete mode 100755 12.7/scripts/bin/gitlab-rake delete mode 100755 12.7/scripts/bin/object-storage-backup delete mode 100755 12.7/scripts/bin/object-storage-restore delete mode 100644 12.7/scripts/lib/object_storage_backup.rb delete mode 100755 12.7/scripts/prebuild.sh delete mode 100644 12.8/LICENSE delete mode 100644 12.8/README.md delete mode 100755 12.8/build-scripts/cleanup.sh delete mode 100755 12.8/build-scripts/prepare.sh delete mode 100755 12.8/scripts/bin/backup-utility delete mode 100755 12.8/scripts/bin/entrypoint.sh delete mode 100755 12.8/scripts/bin/gitlab-rails delete mode 100755 12.8/scripts/bin/gitlab-rake delete mode 100755 12.8/scripts/bin/object-storage-backup delete mode 100755 12.8/scripts/bin/object-storage-restore delete mode 100644 12.8/scripts/lib/object_storage_backup.rb delete mode 100755 12.8/scripts/prebuild.sh rename 12.8/Dockerfile => Dockerfile (87%) rename 12.5/LICENSE => LICENSE (100%) rename {12.8/build-scripts => build-scripts}/build.sh (69%) rename {12.5/build-scripts => build-scripts}/cleanup.sh (100%) create mode 100755 build-scripts/prepare.sh create mode 100644 download.json rename {12.7/scripts => scripts}/bin/backup-utility (100%) rename {12.3/scripts => scripts}/bin/entrypoint.sh (100%) rename {12.3/scripts => scripts}/bin/gitlab-rails (100%) rename {12.3/scripts => scripts}/bin/gitlab-rake (100%) rename {12.3/scripts => scripts}/bin/object-storage-backup (100%) rename {12.3/scripts => scripts}/bin/object-storage-restore (100%) rename {12.4/scripts => scripts}/lib/object_storage_backup.rb (100%) diff --git a/12.3/Dockerfile b/12.3/Dockerfile deleted file mode 100644 index 1fe8f53..0000000 --- a/12.3/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -ARG CI_REGISTRY_IMAGE= -ARG FROM_IMAGE=gitlab-rails-ee -ARG TAG=ubi -ARG PYTHON_TAG=gitlab-python:ubi - -ARG RAILS_IMAGE=${FROM_IMAGE}:${TAG} -ARG PYTHON_IMAGE=gitlab-python:ubi - -FROM ${PYTHON_IMAGE} AS python -FROM ${RAILS_IMAGE} - -ARG S3CMD_VERSION=2.0.1 -ARG GSUTIL_VERSION=4.42 -ARG GITLAB_USER=git - -COPY --from=python /usr/local/bin /usr/local/bin/ -COPY --from=python /usr/local/lib /usr/local/lib/ -COPY --from=python /usr/local/include /usr/local/include/ - -RUN dnf --disableplugin=subscription-manager install -yb --nodocs ca-certificates openssl \ - && pip3 install s3cmd==${S3CMD_VERSION} gsutil==${GSUTIL_VERSION} crcmod - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.3/scripts/bin/backup-utility b/12.3/scripts/bin/backup-utility deleted file mode 100755 index 49585b7..0000000 --- a/12.3/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd sync "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd sync ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.3/scripts/lib/object_storage_backup.rb b/12.3/scripts/lib/object_storage_backup.rb deleted file mode 100755 index 7d8986d..0000000 --- a/12.3/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,149 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - # s3cmd treats `-` as a special filename for using stdin, as a result - # we need a slightly different syntax to support syncing the `-` directory (used for system uploads) - if File.basename(source_path) == '-' - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/-/) - else - cmd = %W(s3cmd --stop-on-error sync #{source_path} s3://#{@remote_bucket_name}) - end - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.4/Dockerfile b/12.4/Dockerfile deleted file mode 100644 index a767ed5..0000000 --- a/12.4/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.0 -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.4 - -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-ruby:12.4 -ARG PYTHON_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-python:12.4 - -FROM ${PYTHON_IMAGE} AS python -FROM ${RAILS_IMAGE} - -ARG S3CMD_VERSION=2.0.1 -ARG GSUTIL_VERSION=4.42 -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=python /usr/local/bin /usr/local/bin/ -COPY --from=python /usr/local/lib /usr/local/lib/ -COPY --from=python /usr/local/include /usr/local/include/ - -RUN dnf install -yb --nodocs ca-certificates openssl libedit \ - && pip3 install s3cmd==${S3CMD_VERSION} gsutil==${GSUTIL_VERSION} crcmod - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.4/scripts/bin/backup-utility b/12.4/scripts/bin/backup-utility deleted file mode 100755 index e20f191..0000000 --- a/12.4/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.4/scripts/bin/entrypoint.sh b/12.4/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e..0000000 --- a/12.4/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.4/scripts/bin/gitlab-rails b/12.4/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb..0000000 --- a/12.4/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.4/scripts/bin/gitlab-rake b/12.4/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75d..0000000 --- a/12.4/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.4/scripts/bin/object-storage-backup b/12.4/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e..0000000 --- a/12.4/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.4/scripts/bin/object-storage-restore b/12.4/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958..0000000 --- a/12.4/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.5/Dockerfile b/12.5/Dockerfile deleted file mode 100644 index 5718f13..0000000 --- a/12.5/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.0 -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.5 - -ARG GITLAB_VERSION=v12.5.2-ubi8 - -FROM ${RAILS_IMAGE} as builder - -ARG NEXUS_SERVER -ARG GITLAB_VERSION -ARG VENDOR=gitlab -ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} - -ADD build-scripts/ /build-scripts/ - -RUN /build-scripts/prepare.sh ${PACKAGE_URL} - -FROM ${RAILS_IMAGE} - -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=builder /prepare/dependencies / - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -RUN dnf clean all \ - && rm -r /var/cache/dnf \ - && dnf --disableplugin=subscription-manager --nogpgcheck install -yb --nodocs ca-certificates openssl - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.5/README.md b/12.5/README.md deleted file mode 100644 index 3ef72c5..0000000 --- a/12.5/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab 12.5.0 release - -The hardened containers for GitLab 12.5 require the gitlab 12.5 blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the 12.5 directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.5/build-scripts/build.sh b/12.5/build-scripts/build.sh deleted file mode 100755 index a26f376..0000000 --- a/12.5/build-scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# NOTICE: This script requires `docker`. - -set -euxo pipefail - -TAG=${1:-12.5} -REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} -DOCKER_OPTS=${DOCKER_OPTS:-""} - -imageName() { - printf "${REPOSITORY}${1}:${TAG}" -} - -buildImage() { - IMAGE="${1}" - CONTEXT="${IMAGE%*-ee}" - { - docker build \ - -t "$(imageName ${IMAGE})" . \ - ${DOCKER_OPTS:-} | tee ${CONTEXT}.out - } || { - echo "${CONTEXT}" >> failed.log - } -} - -# Cleanup log outputs from previous build -rm -f *.out failed.log - -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - -DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" -buildImage gitlab-task-runner diff --git a/12.5/build-scripts/prepare.sh b/12.5/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4..0000000 --- a/12.5/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.5/scripts/bin/backup-utility b/12.5/scripts/bin/backup-utility deleted file mode 100755 index e20f191..0000000 --- a/12.5/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.5/scripts/bin/entrypoint.sh b/12.5/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e..0000000 --- a/12.5/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.5/scripts/bin/gitlab-rails b/12.5/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb..0000000 --- a/12.5/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.5/scripts/bin/gitlab-rake b/12.5/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75d..0000000 --- a/12.5/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.5/scripts/bin/object-storage-backup b/12.5/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e..0000000 --- a/12.5/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.5/scripts/bin/object-storage-restore b/12.5/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958..0000000 --- a/12.5/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.5/scripts/lib/object_storage_backup.rb b/12.5/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6..0000000 --- a/12.5/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.5/scripts/prebuild.sh b/12.5/scripts/prebuild.sh deleted file mode 100755 index 170b220..0000000 --- a/12.5/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.5.2-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 8040EEFCCED8C668EF27F7C61DC5606C0C7E9A9B && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.6/Dockerfile b/12.6/Dockerfile deleted file mode 100644 index 7f1792d..0000000 --- a/12.6/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.0 - -ARG GITLAB_VERSION=v12.6.1-ubi8 - -ARG UBI_IMAGE=${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.6 - -FROM ${UBI_IMAGE} AS builder - -ARG NEXUS_SERVER -ARG VENDOR=gitlab -ARG GITLAB_VERSION -ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} - -ADD build-scripts/ /build-scripts/ - -RUN /build-scripts/prepare.sh "${PACKAGE_URL}" - -FROM ${RAILS_IMAGE} - -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=builder /prepare/dependencies / - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -RUN dnf clean all \ - && rm -r /var/cache/dnf \ - && dnf --disableplugin=subscription-manager --nogpgcheck install -yb --nodocs ca-certificates openssl - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.6/LICENSE b/12.6/LICENSE deleted file mode 100644 index 5285f42..0000000 --- a/12.6/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/12.6/README.md b/12.6/README.md deleted file mode 100644 index e363bcf..0000000 --- a/12.6/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab 12.6.0 release - -The hardened containers for GitLab 12.6 require the gitlab 12.6 blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the 12.6 directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby - * gitlab-container-registry -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.6/build-scripts/build.sh b/12.6/build-scripts/build.sh deleted file mode 100755 index 16547a1..0000000 --- a/12.6/build-scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# NOTICE: This script requires `docker`. - -set -euxo pipefail - -TAG=${1:-12.6} -REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} -DOCKER_OPTS=${DOCKER_OPTS:-""} - -imageName() { - printf "${REPOSITORY}${1}:${TAG}" -} - -buildImage() { - IMAGE="${1}" - CONTEXT="${IMAGE%*-ee}" - { - docker build \ - -t "$(imageName ${IMAGE})" . \ - ${DOCKER_OPTS:-} | tee ${CONTEXT}.out - } || { - echo "${CONTEXT}" >> failed.log - } -} - -# Cleanup log outputs from previous build -rm -f *.out failed.log - -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - -DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" -buildImage gitlab-task-runner diff --git a/12.6/build-scripts/cleanup.sh b/12.6/build-scripts/cleanup.sh deleted file mode 100755 index ad291f9..0000000 --- a/12.6/build-scripts/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -rm -f *.tar.gz *.out failed.log diff --git a/12.6/build-scripts/prepare.sh b/12.6/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4..0000000 --- a/12.6/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.6/scripts/bin/backup-utility b/12.6/scripts/bin/backup-utility deleted file mode 100755 index e20f191..0000000 --- a/12.6/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.6/scripts/bin/entrypoint.sh b/12.6/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e..0000000 --- a/12.6/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.6/scripts/bin/gitlab-rails b/12.6/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb..0000000 --- a/12.6/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.6/scripts/bin/gitlab-rake b/12.6/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75d..0000000 --- a/12.6/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.6/scripts/bin/object-storage-backup b/12.6/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e..0000000 --- a/12.6/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.6/scripts/bin/object-storage-restore b/12.6/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958..0000000 --- a/12.6/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.6/scripts/lib/object_storage_backup.rb b/12.6/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6..0000000 --- a/12.6/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.6/scripts/prebuild.sh b/12.6/scripts/prebuild.sh deleted file mode 100755 index 5205bfd..0000000 --- a/12.6/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.6.1-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 8040EEFCCED8C668EF27F7C61DC5606C0C7E9A9B && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.7/Dockerfile b/12.7/Dockerfile deleted file mode 100644 index 766f581..0000000 --- a/12.7/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -ARG GITLAB_VERSION=v12.7.0-ubi8 - -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.1 - -ARG UBI_IMAGE=${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.7 - -FROM ${UBI_IMAGE} AS builder - -ARG NEXUS_SERVER -ARG VENDOR=gitlab -ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} - -ADD build-scripts/ /build-scripts/ - -RUN /build-scripts/prepare.sh "${PACKAGE_URL}" - -FROM ${RAILS_IMAGE} - -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=builder /prepare/dependencies / - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -RUN dnf clean all \ - && rm -r /var/cache/dnf \ - && dnf --disableplugin=subscription-manager --nogpgcheck install -yb --nodocs ca-certificates openssl - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.7/LICENSE b/12.7/LICENSE deleted file mode 100644 index 5285f42..0000000 --- a/12.7/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/12.7/README.md b/12.7/README.md deleted file mode 100644 index 459ff44..0000000 --- a/12.7/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab Build - -The hardened containers for GitLab require the gitlab correctly version assets blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the desired versioned directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby - * gitlab-container-registry -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.7/build-scripts/build.sh b/12.7/build-scripts/build.sh deleted file mode 100755 index 9d6d9e2..0000000 --- a/12.7/build-scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# NOTICE: This script requires `docker`. - -set -euxo pipefail - -TAG=${1:-12.7} -REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} -DOCKER_OPTS=${DOCKER_OPTS:-""} - -imageName() { - printf "${REPOSITORY}${1}:${TAG}" -} - -buildImage() { - IMAGE="${1}" - CONTEXT="${IMAGE%*-ee}" - { - docker build \ - -t "$(imageName ${IMAGE})" . \ - ${DOCKER_OPTS:-} | tee ${CONTEXT}.out - } || { - echo "${CONTEXT}" >> failed.log - } -} - -# Cleanup log outputs from previous build -rm -f *.out failed.log - -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - -DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" -buildImage gitlab-task-runner diff --git a/12.7/build-scripts/cleanup.sh b/12.7/build-scripts/cleanup.sh deleted file mode 100755 index ad291f9..0000000 --- a/12.7/build-scripts/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -rm -f *.tar.gz *.out failed.log diff --git a/12.7/build-scripts/prepare.sh b/12.7/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4..0000000 --- a/12.7/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.7/scripts/bin/entrypoint.sh b/12.7/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e..0000000 --- a/12.7/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.7/scripts/bin/gitlab-rails b/12.7/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb..0000000 --- a/12.7/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.7/scripts/bin/gitlab-rake b/12.7/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75d..0000000 --- a/12.7/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.7/scripts/bin/object-storage-backup b/12.7/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e..0000000 --- a/12.7/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.7/scripts/bin/object-storage-restore b/12.7/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958..0000000 --- a/12.7/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.7/scripts/lib/object_storage_backup.rb b/12.7/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6..0000000 --- a/12.7/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.7/scripts/prebuild.sh b/12.7/scripts/prebuild.sh deleted file mode 100755 index 2b34b87..0000000 --- a/12.7/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.7.0-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 8040EEFCCED8C668EF27F7C61DC5606C0C7E9A9B && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.8/LICENSE b/12.8/LICENSE deleted file mode 100644 index 5285f42..0000000 --- a/12.8/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/12.8/README.md b/12.8/README.md deleted file mode 100644 index 459ff44..0000000 --- a/12.8/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab Build - -The hardened containers for GitLab require the gitlab correctly version assets blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the desired versioned directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby - * gitlab-container-registry -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.8/build-scripts/cleanup.sh b/12.8/build-scripts/cleanup.sh deleted file mode 100755 index ad291f9..0000000 --- a/12.8/build-scripts/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -rm -f *.tar.gz *.out failed.log diff --git a/12.8/build-scripts/prepare.sh b/12.8/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4..0000000 --- a/12.8/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.8/scripts/bin/backup-utility b/12.8/scripts/bin/backup-utility deleted file mode 100755 index 59ac11c..0000000 --- a/12.8/scripts/bin/backup-utility +++ /dev/null @@ -1,262 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db repositories lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - - # Previous versions of the dump failed to mark the repos as skipped, so we additionally check for the directory - if [ -e $backups_path/repositories ]; then - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - fi - - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.8/scripts/bin/entrypoint.sh b/12.8/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e..0000000 --- a/12.8/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.8/scripts/bin/gitlab-rails b/12.8/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb..0000000 --- a/12.8/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.8/scripts/bin/gitlab-rake b/12.8/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75d..0000000 --- a/12.8/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.8/scripts/bin/object-storage-backup b/12.8/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e..0000000 --- a/12.8/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.8/scripts/bin/object-storage-restore b/12.8/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958..0000000 --- a/12.8/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.8/scripts/lib/object_storage_backup.rb b/12.8/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6..0000000 --- a/12.8/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.8/scripts/prebuild.sh b/12.8/scripts/prebuild.sh deleted file mode 100755 index ae8b162..0000000 --- a/12.8/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.8.1-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 5c7738cc4840f93f6e9170ff5a0e20d5f9706778 && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.8/Dockerfile b/Dockerfile similarity index 87% rename from 12.8/Dockerfile rename to Dockerfile index 88e1ad4..c366b65 100644 --- a/12.8/Dockerfile +++ b/Dockerfile @@ -9,15 +9,13 @@ ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.8 FROM ${UBI_IMAGE} AS builder -ARG NEXUS_SERVER -ARG VENDOR=gitlab +ARG GITLAB_VERSION ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} +COPY ${PACKAGE_NAME} /opt/ ADD build-scripts/ /build-scripts/ -RUN /build-scripts/prepare.sh "${PACKAGE_URL}" - +RUN /build-scripts/prepare.sh "/opt/${PACKAGE_NAME}" FROM ${RAILS_IMAGE} diff --git a/12.5/LICENSE b/LICENSE similarity index 100% rename from 12.5/LICENSE rename to LICENSE diff --git a/README.md b/README.md index 6f916b9..3a26688 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,41 @@ # gitlab-task-runner-container +Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. + +## GitLab Build + +The hardened containers for GitLab require the gitlab correctly version assets blob to be available for download. + +There are some included scripts to make this easier for user building images offline on their machine. + + +## Building + +1. Switch to the desired versioned directory +2. Run `./build-scripts/build.sh` + * Runs docker build, and takes care of setting the appropriate build args for users running locally + + +`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. + +## Build Phases + +Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. + +- Phase One + * kubectl + * gitlab-ruby + * gitlab-container-registry +- Phase Two + * git-base + * gitlab-exporter + * gitlab-mailroom + * gitlab-shell + * gitlab-rails + * gitlab-workhorse +- Phase 3 + * gitaly +- Phase 4 + * gitlab-sidekiq + * gitlab-task-runner + * gitlab-unicorn diff --git a/12.8/build-scripts/build.sh b/build-scripts/build.sh similarity index 69% rename from 12.8/build-scripts/build.sh rename to build-scripts/build.sh index 0470f34..f67ccc6 100755 --- a/12.8/build-scripts/build.sh +++ b/build-scripts/build.sh @@ -6,8 +6,6 @@ set -euxo pipefail TAG=${1:-12.8} REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} DOCKER_OPTS=${DOCKER_OPTS:-""} imageName() { @@ -29,13 +27,5 @@ buildImage() { # Cleanup log outputs from previous build rm -f *.out failed.log -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" buildImage gitlab-task-runner diff --git a/12.5/build-scripts/cleanup.sh b/build-scripts/cleanup.sh similarity index 100% rename from 12.5/build-scripts/cleanup.sh rename to build-scripts/cleanup.sh diff --git a/build-scripts/prepare.sh b/build-scripts/prepare.sh new file mode 100755 index 0000000..242b668 --- /dev/null +++ b/build-scripts/prepare.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euxo pipefail + +PACKAGE_PATH=$1 +WORKSPACE="${WORKSPACE:-/prepare}" + +mkdir -p ${WORKSPACE}/dependencies + +# Extract UBI dependencies +tar -xvf "${PACKAGE_PATH}" -C "${WORKSPACE}" + +# Extract the specific depenencies needed for this contianer +tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" +tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/download.json b/download.json new file mode 100644 index 0000000..7f52925 --- /dev/null +++ b/download.json @@ -0,0 +1,7 @@ +{ "resources": + [ + { "url" : "https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-v12.8.1-ubi8.tar", + "filename": "ubi8-build-dependencies-v12.8.1-ubi8.tar", + "sha256": "1d49f47924738132f4d10cb9b32ce6d9c20b151dfc3b056c298607f267c75571" + } +] } diff --git a/12.7/scripts/bin/backup-utility b/scripts/bin/backup-utility similarity index 100% rename from 12.7/scripts/bin/backup-utility rename to scripts/bin/backup-utility diff --git a/12.3/scripts/bin/entrypoint.sh b/scripts/bin/entrypoint.sh similarity index 100% rename from 12.3/scripts/bin/entrypoint.sh rename to scripts/bin/entrypoint.sh diff --git a/12.3/scripts/bin/gitlab-rails b/scripts/bin/gitlab-rails similarity index 100% rename from 12.3/scripts/bin/gitlab-rails rename to scripts/bin/gitlab-rails diff --git a/12.3/scripts/bin/gitlab-rake b/scripts/bin/gitlab-rake similarity index 100% rename from 12.3/scripts/bin/gitlab-rake rename to scripts/bin/gitlab-rake diff --git a/12.3/scripts/bin/object-storage-backup b/scripts/bin/object-storage-backup similarity index 100% rename from 12.3/scripts/bin/object-storage-backup rename to scripts/bin/object-storage-backup diff --git a/12.3/scripts/bin/object-storage-restore b/scripts/bin/object-storage-restore similarity index 100% rename from 12.3/scripts/bin/object-storage-restore rename to scripts/bin/object-storage-restore diff --git a/12.4/scripts/lib/object_storage_backup.rb b/scripts/lib/object_storage_backup.rb similarity index 100% rename from 12.4/scripts/lib/object_storage_backup.rb rename to scripts/lib/object_storage_backup.rb -- GitLab