diff --git a/12.3/Dockerfile b/12.3/Dockerfile deleted file mode 100644 index 1fe8f53e152a67ceb1fe497d08ad9a426c0aca91..0000000000000000000000000000000000000000 --- a/12.3/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -ARG CI_REGISTRY_IMAGE= -ARG FROM_IMAGE=gitlab-rails-ee -ARG TAG=ubi -ARG PYTHON_TAG=gitlab-python:ubi - -ARG RAILS_IMAGE=${FROM_IMAGE}:${TAG} -ARG PYTHON_IMAGE=gitlab-python:ubi - -FROM ${PYTHON_IMAGE} AS python -FROM ${RAILS_IMAGE} - -ARG S3CMD_VERSION=2.0.1 -ARG GSUTIL_VERSION=4.42 -ARG GITLAB_USER=git - -COPY --from=python /usr/local/bin /usr/local/bin/ -COPY --from=python /usr/local/lib /usr/local/lib/ -COPY --from=python /usr/local/include /usr/local/include/ - -RUN dnf --disableplugin=subscription-manager install -yb --nodocs ca-certificates openssl \ - && pip3 install s3cmd==${S3CMD_VERSION} gsutil==${GSUTIL_VERSION} crcmod - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.3/scripts/bin/backup-utility b/12.3/scripts/bin/backup-utility deleted file mode 100755 index 49585b76672e6726b1ee1a527e21fa1db02a2904..0000000000000000000000000000000000000000 --- a/12.3/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd sync "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd sync ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.3/scripts/lib/object_storage_backup.rb b/12.3/scripts/lib/object_storage_backup.rb deleted file mode 100755 index 7d8986dd7cf1f33e152fc6b91853872185451b3d..0000000000000000000000000000000000000000 --- a/12.3/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,149 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - # s3cmd treats `-` as a special filename for using stdin, as a result - # we need a slightly different syntax to support syncing the `-` directory (used for system uploads) - if File.basename(source_path) == '-' - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/-/) - else - cmd = %W(s3cmd --stop-on-error sync #{source_path} s3://#{@remote_bucket_name}) - end - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.4/Dockerfile b/12.4/Dockerfile deleted file mode 100644 index a767ed597d772a7c9ea1b97b74c2426ee6ce280a..0000000000000000000000000000000000000000 --- a/12.4/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.0 -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.4 - -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-ruby:12.4 -ARG PYTHON_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-python:12.4 - -FROM ${PYTHON_IMAGE} AS python -FROM ${RAILS_IMAGE} - -ARG S3CMD_VERSION=2.0.1 -ARG GSUTIL_VERSION=4.42 -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=python /usr/local/bin /usr/local/bin/ -COPY --from=python /usr/local/lib /usr/local/lib/ -COPY --from=python /usr/local/include /usr/local/include/ - -RUN dnf install -yb --nodocs ca-certificates openssl libedit \ - && pip3 install s3cmd==${S3CMD_VERSION} gsutil==${GSUTIL_VERSION} crcmod - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.4/scripts/bin/backup-utility b/12.4/scripts/bin/backup-utility deleted file mode 100755 index e20f19147ace7b696f3b49af53d96f03892345c6..0000000000000000000000000000000000000000 --- a/12.4/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.4/scripts/bin/entrypoint.sh b/12.4/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e6d0b16185a1aca86d829bc68e5843f854..0000000000000000000000000000000000000000 --- a/12.4/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.4/scripts/bin/gitlab-rails b/12.4/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb21b8cdb44197c892390f543d3c12d8ccb..0000000000000000000000000000000000000000 --- a/12.4/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.4/scripts/bin/gitlab-rake b/12.4/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75dd5c5ee55d7831ae29e4bcf0a3b25ab4f4..0000000000000000000000000000000000000000 --- a/12.4/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.4/scripts/bin/object-storage-backup b/12.4/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e5a62c1c2736c7d6cedcdb093759cf1823..0000000000000000000000000000000000000000 --- a/12.4/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.4/scripts/bin/object-storage-restore b/12.4/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958e76339f8abfe00d9d1dd22c22adb1039d..0000000000000000000000000000000000000000 --- a/12.4/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.5/Dockerfile b/12.5/Dockerfile deleted file mode 100644 index 5718f1371b4c30a33a966f88f1b760cd98eafa42..0000000000000000000000000000000000000000 --- a/12.5/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.0 -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.5 - -ARG GITLAB_VERSION=v12.5.2-ubi8 - -FROM ${RAILS_IMAGE} as builder - -ARG NEXUS_SERVER -ARG GITLAB_VERSION -ARG VENDOR=gitlab -ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} - -ADD build-scripts/ /build-scripts/ - -RUN /build-scripts/prepare.sh ${PACKAGE_URL} - -FROM ${RAILS_IMAGE} - -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=builder /prepare/dependencies / - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -RUN dnf clean all \ - && rm -r /var/cache/dnf \ - && dnf --disableplugin=subscription-manager --nogpgcheck install -yb --nodocs ca-certificates openssl - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.5/README.md b/12.5/README.md deleted file mode 100644 index 3ef72c56d571afd8486f36006494ec64e9923e0c..0000000000000000000000000000000000000000 --- a/12.5/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab 12.5.0 release - -The hardened containers for GitLab 12.5 require the gitlab 12.5 blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the 12.5 directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.5/build-scripts/build.sh b/12.5/build-scripts/build.sh deleted file mode 100755 index a26f376c8f91e79883c27c99af8fa64af4ab4018..0000000000000000000000000000000000000000 --- a/12.5/build-scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# NOTICE: This script requires `docker`. - -set -euxo pipefail - -TAG=${1:-12.5} -REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} -DOCKER_OPTS=${DOCKER_OPTS:-""} - -imageName() { - printf "${REPOSITORY}${1}:${TAG}" -} - -buildImage() { - IMAGE="${1}" - CONTEXT="${IMAGE%*-ee}" - { - docker build \ - -t "$(imageName ${IMAGE})" . \ - ${DOCKER_OPTS:-} | tee ${CONTEXT}.out - } || { - echo "${CONTEXT}" >> failed.log - } -} - -# Cleanup log outputs from previous build -rm -f *.out failed.log - -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - -DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" -buildImage gitlab-task-runner diff --git a/12.5/build-scripts/prepare.sh b/12.5/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4d8504ab97ecebf7eed8a00ebf8f7f956d..0000000000000000000000000000000000000000 --- a/12.5/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.5/scripts/bin/backup-utility b/12.5/scripts/bin/backup-utility deleted file mode 100755 index e20f19147ace7b696f3b49af53d96f03892345c6..0000000000000000000000000000000000000000 --- a/12.5/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.5/scripts/bin/entrypoint.sh b/12.5/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e6d0b16185a1aca86d829bc68e5843f854..0000000000000000000000000000000000000000 --- a/12.5/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.5/scripts/bin/gitlab-rails b/12.5/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb21b8cdb44197c892390f543d3c12d8ccb..0000000000000000000000000000000000000000 --- a/12.5/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.5/scripts/bin/gitlab-rake b/12.5/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75dd5c5ee55d7831ae29e4bcf0a3b25ab4f4..0000000000000000000000000000000000000000 --- a/12.5/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.5/scripts/bin/object-storage-backup b/12.5/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e5a62c1c2736c7d6cedcdb093759cf1823..0000000000000000000000000000000000000000 --- a/12.5/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.5/scripts/bin/object-storage-restore b/12.5/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958e76339f8abfe00d9d1dd22c22adb1039d..0000000000000000000000000000000000000000 --- a/12.5/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.5/scripts/lib/object_storage_backup.rb b/12.5/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6361cef6b7e6477d9bd9e565d8f95c4f07..0000000000000000000000000000000000000000 --- a/12.5/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.5/scripts/prebuild.sh b/12.5/scripts/prebuild.sh deleted file mode 100755 index 170b2206cdfdd391223ead07c003cd8e550edeae..0000000000000000000000000000000000000000 --- a/12.5/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.5.2-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 8040EEFCCED8C668EF27F7C61DC5606C0C7E9A9B && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.6/Dockerfile b/12.6/Dockerfile deleted file mode 100644 index 7f1792daac4f497ae576e6c86ca3f39c99801ac5..0000000000000000000000000000000000000000 --- a/12.6/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.0 - -ARG GITLAB_VERSION=v12.6.1-ubi8 - -ARG UBI_IMAGE=${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.6 - -FROM ${UBI_IMAGE} AS builder - -ARG NEXUS_SERVER -ARG VENDOR=gitlab -ARG GITLAB_VERSION -ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} - -ADD build-scripts/ /build-scripts/ - -RUN /build-scripts/prepare.sh "${PACKAGE_URL}" - -FROM ${RAILS_IMAGE} - -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=builder /prepare/dependencies / - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -RUN dnf clean all \ - && rm -r /var/cache/dnf \ - && dnf --disableplugin=subscription-manager --nogpgcheck install -yb --nodocs ca-certificates openssl - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.6/LICENSE b/12.6/LICENSE deleted file mode 100644 index 5285f420ff222526f9afa7acf507362367132f9c..0000000000000000000000000000000000000000 --- a/12.6/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/12.6/README.md b/12.6/README.md deleted file mode 100644 index e363bcfc7913d951f6388793df7fa351234c7d4b..0000000000000000000000000000000000000000 --- a/12.6/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab 12.6.0 release - -The hardened containers for GitLab 12.6 require the gitlab 12.6 blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the 12.6 directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby - * gitlab-container-registry -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.6/build-scripts/build.sh b/12.6/build-scripts/build.sh deleted file mode 100755 index 16547a114767ab38362a4f1059d520f8b2a1707b..0000000000000000000000000000000000000000 --- a/12.6/build-scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# NOTICE: This script requires `docker`. - -set -euxo pipefail - -TAG=${1:-12.6} -REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} -DOCKER_OPTS=${DOCKER_OPTS:-""} - -imageName() { - printf "${REPOSITORY}${1}:${TAG}" -} - -buildImage() { - IMAGE="${1}" - CONTEXT="${IMAGE%*-ee}" - { - docker build \ - -t "$(imageName ${IMAGE})" . \ - ${DOCKER_OPTS:-} | tee ${CONTEXT}.out - } || { - echo "${CONTEXT}" >> failed.log - } -} - -# Cleanup log outputs from previous build -rm -f *.out failed.log - -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - -DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" -buildImage gitlab-task-runner diff --git a/12.6/build-scripts/cleanup.sh b/12.6/build-scripts/cleanup.sh deleted file mode 100755 index ad291f98ba8b4db3c80d564f753a1288550c8f10..0000000000000000000000000000000000000000 --- a/12.6/build-scripts/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -rm -f *.tar.gz *.out failed.log diff --git a/12.6/build-scripts/prepare.sh b/12.6/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4d8504ab97ecebf7eed8a00ebf8f7f956d..0000000000000000000000000000000000000000 --- a/12.6/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.6/scripts/bin/backup-utility b/12.6/scripts/bin/backup-utility deleted file mode 100755 index e20f19147ace7b696f3b49af53d96f03892345c6..0000000000000000000000000000000000000000 --- a/12.6/scripts/bin/backup-utility +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.6/scripts/bin/entrypoint.sh b/12.6/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e6d0b16185a1aca86d829bc68e5843f854..0000000000000000000000000000000000000000 --- a/12.6/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.6/scripts/bin/gitlab-rails b/12.6/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb21b8cdb44197c892390f543d3c12d8ccb..0000000000000000000000000000000000000000 --- a/12.6/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.6/scripts/bin/gitlab-rake b/12.6/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75dd5c5ee55d7831ae29e4bcf0a3b25ab4f4..0000000000000000000000000000000000000000 --- a/12.6/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.6/scripts/bin/object-storage-backup b/12.6/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e5a62c1c2736c7d6cedcdb093759cf1823..0000000000000000000000000000000000000000 --- a/12.6/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.6/scripts/bin/object-storage-restore b/12.6/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958e76339f8abfe00d9d1dd22c22adb1039d..0000000000000000000000000000000000000000 --- a/12.6/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.6/scripts/lib/object_storage_backup.rb b/12.6/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6361cef6b7e6477d9bd9e565d8f95c4f07..0000000000000000000000000000000000000000 --- a/12.6/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.6/scripts/prebuild.sh b/12.6/scripts/prebuild.sh deleted file mode 100755 index 5205bfd9889feb3b61d21a983acb03fc0e699067..0000000000000000000000000000000000000000 --- a/12.6/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.6.1-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 8040EEFCCED8C668EF27F7C61DC5606C0C7E9A9B && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.7/Dockerfile b/12.7/Dockerfile deleted file mode 100644 index 766f58163d2b800bde70a2f03b23c49f3160af04..0000000000000000000000000000000000000000 --- a/12.7/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -ARG GITLAB_VERSION=v12.7.0-ubi8 - -ARG BASE_REGISTRY=registry.access.redhat.com -ARG BASE_IMAGE=ubi8/ubi -ARG BASE_TAG=8.1 - -ARG UBI_IMAGE=${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} -ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.7 - -FROM ${UBI_IMAGE} AS builder - -ARG NEXUS_SERVER -ARG VENDOR=gitlab -ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} - -ADD build-scripts/ /build-scripts/ - -RUN /build-scripts/prepare.sh "${PACKAGE_URL}" - -FROM ${RAILS_IMAGE} - -ARG GITLAB_VERSION -ARG GITLAB_USER=git - -LABEL source="https://gitlab.com/gitlab-org/gitlab" \ - name="GitLab Task Runner" \ - maintainer="GitLab Distribution Team" \ - vendor="GitLab" \ - version=${GITLAB_VERSION} \ - release=${GITLAB_VERSION} \ - summary="Task Runner is an entry point for interaction with other containers in the cluster." \ - description="Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage." - -COPY --from=builder /prepare/dependencies / - -COPY scripts/bin/* /usr/local/bin/ -COPY scripts/lib/* /usr/lib/ruby/vendor_ruby - -RUN dnf clean all \ - && rm -r /var/cache/dnf \ - && dnf --disableplugin=subscription-manager --nogpgcheck install -yb --nodocs ca-certificates openssl - -USER ${GITLAB_USER}:${GITLAB_USER} - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/12.7/LICENSE b/12.7/LICENSE deleted file mode 100644 index 5285f420ff222526f9afa7acf507362367132f9c..0000000000000000000000000000000000000000 --- a/12.7/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/12.7/README.md b/12.7/README.md deleted file mode 100644 index 459ff447a73d537aa82972af65e0b022961c6990..0000000000000000000000000000000000000000 --- a/12.7/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab Build - -The hardened containers for GitLab require the gitlab correctly version assets blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the desired versioned directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby - * gitlab-container-registry -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.7/build-scripts/build.sh b/12.7/build-scripts/build.sh deleted file mode 100755 index 9d6d9e25c281acf491b3b7611976986d73be38dc..0000000000000000000000000000000000000000 --- a/12.7/build-scripts/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# NOTICE: This script requires `docker`. - -set -euxo pipefail - -TAG=${1:-12.7} -REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} -DOCKER_OPTS=${DOCKER_OPTS:-""} - -imageName() { - printf "${REPOSITORY}${1}:${TAG}" -} - -buildImage() { - IMAGE="${1}" - CONTEXT="${IMAGE%*-ee}" - { - docker build \ - -t "$(imageName ${IMAGE})" . \ - ${DOCKER_OPTS:-} | tee ${CONTEXT}.out - } || { - echo "${CONTEXT}" >> failed.log - } -} - -# Cleanup log outputs from previous build -rm -f *.out failed.log - -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - -DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" -buildImage gitlab-task-runner diff --git a/12.7/build-scripts/cleanup.sh b/12.7/build-scripts/cleanup.sh deleted file mode 100755 index ad291f98ba8b4db3c80d564f753a1288550c8f10..0000000000000000000000000000000000000000 --- a/12.7/build-scripts/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -rm -f *.tar.gz *.out failed.log diff --git a/12.7/build-scripts/prepare.sh b/12.7/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4d8504ab97ecebf7eed8a00ebf8f7f956d..0000000000000000000000000000000000000000 --- a/12.7/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.7/scripts/bin/entrypoint.sh b/12.7/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e6d0b16185a1aca86d829bc68e5843f854..0000000000000000000000000000000000000000 --- a/12.7/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.7/scripts/bin/gitlab-rails b/12.7/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb21b8cdb44197c892390f543d3c12d8ccb..0000000000000000000000000000000000000000 --- a/12.7/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.7/scripts/bin/gitlab-rake b/12.7/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75dd5c5ee55d7831ae29e4bcf0a3b25ab4f4..0000000000000000000000000000000000000000 --- a/12.7/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.7/scripts/bin/object-storage-backup b/12.7/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e5a62c1c2736c7d6cedcdb093759cf1823..0000000000000000000000000000000000000000 --- a/12.7/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.7/scripts/bin/object-storage-restore b/12.7/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958e76339f8abfe00d9d1dd22c22adb1039d..0000000000000000000000000000000000000000 --- a/12.7/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.7/scripts/lib/object_storage_backup.rb b/12.7/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6361cef6b7e6477d9bd9e565d8f95c4f07..0000000000000000000000000000000000000000 --- a/12.7/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.7/scripts/prebuild.sh b/12.7/scripts/prebuild.sh deleted file mode 100755 index 2b34b87f8702304f8d988169d1af9d99be8c83bd..0000000000000000000000000000000000000000 --- a/12.7/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.7.0-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 8040EEFCCED8C668EF27F7C61DC5606C0C7E9A9B && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.8/LICENSE b/12.8/LICENSE deleted file mode 100644 index 5285f420ff222526f9afa7acf507362367132f9c..0000000000000000000000000000000000000000 --- a/12.8/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/12.8/README.md b/12.8/README.md deleted file mode 100644 index 459ff447a73d537aa82972af65e0b022961c6990..0000000000000000000000000000000000000000 --- a/12.8/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# gitlab-task-runner-container - -Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. - -## GitLab Build - -The hardened containers for GitLab require the gitlab correctly version assets blob to be available for download. - -There are some included scripts to make this easier for user building images offline on their machine. - - -## Building - -1. Switch to the desired versioned directory -2. Run `./build-scripts/build.sh` - * Runs docker build, and takes care of setting the appropriate build args for users running locally - * Uses the `NEXUS_SERVER` variable from your environment, or full `PACKAGE_URL`. - - -`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. - -## Build Phases - -Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. - -- Phase One - * kubectl - * gitlab-ruby - * gitlab-container-registry -- Phase Two - * git-base - * gitlab-exporter - * gitlab-mailroom - * gitlab-shell - * gitlab-rails - * gitlab-workhorse -- Phase 3 - * gitaly -- Phase 4 - * gitlab-sidekiq - * gitlab-task-runner - * gitlab-unicorn diff --git a/12.8/build-scripts/cleanup.sh b/12.8/build-scripts/cleanup.sh deleted file mode 100755 index ad291f98ba8b4db3c80d564f753a1288550c8f10..0000000000000000000000000000000000000000 --- a/12.8/build-scripts/cleanup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -rm -f *.tar.gz *.out failed.log diff --git a/12.8/build-scripts/prepare.sh b/12.8/build-scripts/prepare.sh deleted file mode 100755 index fac2ea4d8504ab97ecebf7eed8a00ebf8f7f956d..0000000000000000000000000000000000000000 --- a/12.8/build-scripts/prepare.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PACKAGE_URL=$1 -WORKSPACE="${WORKSPACE:-/prepare}" -PACKAGE_NAME="${PACKAGE_NAME:-ubi8-build-dependencies.tar}" - -# Download UBI dependencies package and it sha256 -curl -fLk --create-dirs "${PACKAGE_URL}.sha256" -o "${WORKSPACE}/${PACKAGE_NAME}.sha256" -curl -fLk --create-dirs "${PACKAGE_URL}" -o "${WORKSPACE}/${PACKAGE_NAME}" - -# Verify the package integrity -echo "$(cat ${PACKAGE_NAME}.sha256) ${PACKAGE_NAME}" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ${PACKAGE_NAME} succeeded\n\n"; \ -else printf "SHA256 check for ${PACKAGE_NAME} failed\n\n"; fi - -# Extract UBI dependencies -tar -xvf "${WORKSPACE}/${PACKAGE_NAME}" -C "${WORKSPACE}" - -# Extract the specific depenencies needed for this contianer -mkdir ${WORKSPACE}/dependencies -tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" -tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/12.8/scripts/bin/backup-utility b/12.8/scripts/bin/backup-utility deleted file mode 100755 index 59ac11cd1f827edfea433d17fb8a248d1f22c444..0000000000000000000000000000000000000000 --- a/12.8/scripts/bin/backup-utility +++ /dev/null @@ -1,262 +0,0 @@ -#!/bin/bash -set -e - -ACTION="backup" -export BACKUP_BUCKET_NAME=${BACKUP_BUCKET_NAME-gitlab-backups} -export BACKUP_BACKEND=${BACKUP_BACKEND-s3} - -rails_dir=/srv/gitlab -backups_path=$rails_dir/tmp/backups -backup_tars_path=$rails_dir/tmp/backup_tars -object_storage_backends=( registry uploads artifacts lfs packages ) - -skipping_backup_for=() - -function usage() -{ - cat << HEREDOC - - Usage: backup-utility [--restore] [-f URL] [-t TIMESTAMP] [--skip COMPONENT] [--backend BACKEND] - - Options: - -h, --help Show this help message and exit. - --restore [-t TIMESTAMP | -f URL] When specified, utility restores from an existing backup specified - as url or timestamp in object storage. - -f URL http(s):/ftp:/file: URL with backup location. Use with --restore. - -t TIMESTAMP Timestamp (part before '_gitlab_backup.tar' in archive name), - can be used to specify backup source or target name. - --rsyncable Pass the '--rsyncable' parameter to gzip for artifact compression. - --skip COMPONENT When specified, utility will skip the backup of COMPONENT. - May be defined multiple times. Valid values for COMPONENT are - db, repositories, and any of the object storages (e.g. 'lfs'). - --backend BACKEND Object storage backend to use for backups. - Can be either 's3' or 'gcs'. -HEREDOC -} - -# Checks if provided argument is a url for downloading it -function is_url() { - regex='(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' - - [[ $1 =~ $regex ]] -} - -function fetch_remote_backup(){ - mkdir -p $backups_path - output_path=$backups_path/0_gitlab_backup.tar - - if is_url $1; then - >&2 echo "Downloading from $1"; - curl --progress-bar -o $output_path $1 - else # It's a timestamp - file_name="$1_gitlab_backup.tar" - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd get "s3://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp "gs://$BACKUP_BUCKET_NAME/$file_name" $output_path > /dev/null - else - echo "Unknown backend: ${BACKUP_BACKEND}" - fi - fi - echo $output_path -} - -function unpack_backup(){ - local file_path=$1 - cd $(dirname $file_path) - - echo "Unpacking backup" - - if [ ! -f $file_path ]; then - echo $file_path not found - exit 1 - fi - - tar -xf $file_path -} - -function pack_backup(){ - echo "Packing up backup tar" - local backup_name=$1 - tar -cf ${backup_tars_path}/${backup_name}.tar -C $backups_path . -} - -function get_version(){ - cat $rails_dir/VERSION -} - -function get_backup_name(){ - if [ -n "$BACKUP_TIMESTAMP" ]; then - echo ${BACKUP_TIMESTAMP}_gitlab_backup - else - now_timestamp=$(date +%s_%Y_%m_%d) - gitlab_version=$(get_version) - echo ${now_timestamp}_${gitlab_version}_gitlab_backup - fi -} - -function cleanup(){ - rm -rf $backups_path/* - rm -rf $backup_tars_path/* -} - -function write_backup_info(){ - cat << EOF > $backups_path/backup_information.yml -:db_version: $($rails_dir/bin/rails runner "File.write('/tmp/db_version', ActiveRecord::Migrator.current_version.to_s)" && cat /tmp/db_version) -:backup_created_at: $(date "+%Y-%m-%d %H:%M:%S %z") -:gitlab_version: $(get_version) -:tar_version: $(tar --version | head -n 1) -:installation_type: gitlab-helm-chart -:skipped: $1 -EOF -} - -function get_skipped(){ - all=( artifacts.tar.gz uploads.tar.gz builds.tar.gz db repositories lfs.tar.gz registry.tar.gz pages.tar.gz packages.tar.gz ) - skipped_string="" - - for backup_item in ${all[@]}; do - if [ ! -e $backups_path/$backup_item ]; then - skipped_string="$skipped_string,${backup_item%.tar.gz}"; - fi; - done; - - echo ${skipped_string#,} -} - -function backup(){ - backup_name=$(get_backup_name) - mkdir -p $backup_tars_path - - if ! [[ ${skipping_backup_for[@]} =~ "db" ]]; then - gitlab-rake gitlab:backup:db:create - fi - if ! [[ ${skipping_backup_for[@]} =~ "repositories" ]]; then - gitlab-rake gitlab:backup:repo:create - fi - - for backup_item in ${object_storage_backends[@]}; do - if ! [[ ${skipping_backup_for[@]} =~ $backup_item ]]; then - object-storage-backup $backup_item $backups_path/${backup_item}.tar.gz - fi - done - - skipped=$(get_skipped $backup_name) - write_backup_info $skipped - pack_backup $backup_name - if [ "${BACKUP_BACKEND}" = "s3" ]; then - s3cmd put ${backup_tars_path}/${backup_name}.tar s3://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at s3://$BACKUP_BUCKET_NAME/${backup_name}.tar" - elif [ "${BACKUP_BACKEND}" = "gcs" ]; then - gsutil cp -n ${backup_tars_path}/${backup_name}.tar gs://$BACKUP_BUCKET_NAME > /dev/null - echo "[DONE] Backup can be found at gs://$BACKUP_BUCKET_NAME/${backup_name}.tar" - else - echo "Unknown backend for backup: ${BACKUP_BACKEND}" - fi - - cleanup -} - -function is_skipped() { - [[ $SKIPPED =~ $1 ]] -} - -function restore(){ - if [ -z "$BACKUP_URL" ] && [ -z "$BACKUP_TIMESTAMP" ]; then - echo "You need to set BACKUP_URL or BACKUP_TIMESTAMP variable" - exit 1 - fi - - BACKUP=${BACKUP_URL-} - if [ -z "$BACKUP" ]; then - BACKUP=$BACKUP_TIMESTAMP - fi - - file=$(fetch_remote_backup $BACKUP) - - dir_name=$(dirname $file) - file_name=$(basename $file) - timestamp="${file_name%%_*}" - export BACKUP=$timestamp - unpack_backup $file - - skipped_line=$(grep skipped $(dirname $file)/backup_information.yml) - export SKIPPED=$(echo ${skipped_line#:skipped:}) - - installation_type_line=$(grep installation_type $(dirname $file)/backup_information.yml || echo ":installation_type: unknown") - export INSTALLATION_TYPE=$(echo ${installation_type_line#:installation_type: }) - - ! is_skipped "db" && gitlab-rake gitlab:db:drop_tables - ! is_skipped "db" && gitlab-rake gitlab:backup:db:restore - - # Previous versions of the dump failed to mark the repos as skipped, so we additionally check for the directory - if [ -e $backups_path/repositories ]; then - ! is_skipped "repositories" && gitlab-rake gitlab:backup:repo:restore - fi - - ! is_skipped "builds" && gitlab-rake gitlab:backup:builds:restore - - if [ "$INSTALLATION_TYPE" = "gitlab-helm-chart" ]; then - for restore_item in ${object_storage_backends[@]}; do - if [ -f $backups_path/${restore_item}.tar.gz ]; then - ! is_skipped $restore_item && object-storage-restore $restore_item $backups_path/${restore_item}.tar.gz - fi - done - else - echo "Backup tarball not from a Helm chart based installation. Not processing files in object storage." - fi - - gitlab-rake cache:clear -} - -while [[ $# -gt 0 ]] -do - key="$1" - - case $key in - -h|--help) - usage - ACTION="none" - break - ;; - -f|--file) - BACKUP_URL="$2" - shift - shift - ;; - -t|--timestamp) - BACKUP_TIMESTAMP="$2" - shift - shift - ;; - --backend) - export BACKUP_BACKEND="$2" - shift - shift - ;; - --restore) - ACTION="restore" - shift - ;; - --rsyncable) - export GZIP_RSYNCABLE="yes" - shift - ;; - --skip) - skipping_backup_for+=( "$2" ) - shift - shift - ;; - *) - usage - echo "Unexpected parameter: $key" - exit 1 - ;; - esac -done - -if [ "$ACTION" = "restore" ]; then - restore -elif [ "$ACTION" = "backup" ]; then - backup -fi diff --git a/12.8/scripts/bin/entrypoint.sh b/12.8/scripts/bin/entrypoint.sh deleted file mode 100755 index 3bc0d9e6d0b16185a1aca86d829bc68e5843f854..0000000000000000000000000000000000000000 --- a/12.8/scripts/bin/entrypoint.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -e -/scripts/set-config "${CONFIG_TEMPLATE_DIRECTORY}" "${CONFIG_DIRECTORY:=$CONFIG_TEMPLATE_DIRECTORY}" - -cd /srv/gitlab; -echo "Attempting to run '$@' as a main process"; - -exec "$@"; diff --git a/12.8/scripts/bin/gitlab-rails b/12.8/scripts/bin/gitlab-rails deleted file mode 100755 index 9a2febb21b8cdb44197c892390f543d3c12d8ccb..0000000000000000000000000000000000000000 --- a/12.8/scripts/bin/gitlab-rails +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rails "$@" diff --git a/12.8/scripts/bin/gitlab-rake b/12.8/scripts/bin/gitlab-rake deleted file mode 100755 index 330d75dd5c5ee55d7831ae29e4bcf0a3b25ab4f4..0000000000000000000000000000000000000000 --- a/12.8/scripts/bin/gitlab-rake +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rails_dir=/srv/gitlab -cd $rails_dir -$rails_dir/bin/bundle exec rake -f $rails_dir/Rakefile "$@" diff --git a/12.8/scripts/bin/object-storage-backup b/12.8/scripts/bin/object-storage-backup deleted file mode 100755 index 47c2a2e5a62c1c2736c7d6cedcdb093759cf1823..0000000000000000000000000000000000000000 --- a/12.8/scripts/bin/object-storage-backup +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort ("backup_item and output_tar_path arguments needs to be passed to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).backup diff --git a/12.8/scripts/bin/object-storage-restore b/12.8/scripts/bin/object-storage-restore deleted file mode 100755 index e8a5958e76339f8abfe00d9d1dd22c22adb1039d..0000000000000000000000000000000000000000 --- a/12.8/scripts/bin/object-storage-restore +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby - -require 'object_storage_backup' - -abort("restore_item and tar path needs to be passed as arguments to the script") unless ARGV.length == 2 - -bucket_name = ENV["#{ARGV[0].upcase}_BUCKET_NAME"] || "gitlab-#{ARGV[0]}" -tmp_bucket = ENV['TMP_BUCKET_NAME'] || 'tmp' -backend_type = ENV['BACKUP_BACKEND'] || 's3' -ObjectStorageBackup.new(ARGV[0], ARGV[1], bucket_name, tmp_bucket, backend_type).restore diff --git a/12.8/scripts/lib/object_storage_backup.rb b/12.8/scripts/lib/object_storage_backup.rb deleted file mode 100644 index fd8dae6361cef6b7e6477d9bd9e565d8f95c4f07..0000000000000000000000000000000000000000 --- a/12.8/scripts/lib/object_storage_backup.rb +++ /dev/null @@ -1,144 +0,0 @@ -require 'open3' -require 'fileutils' - -class String - def red; "\e[31m#{self}\e[0m" end - def green; "\e[32m#{self}\e[0m" end - def blue; "\e[34m#{self}\e[0m" end -end - -class ObjectStorageBackup - attr_accessor :name, :local_tar_path, :remote_bucket_name, :tmp_bucket_name, :backend - - def initialize(name, local_tar_path, remote_bucket_name, tmp_bucket_name = 'tmp', backend = 's3') - @name = name - @local_tar_path = local_tar_path - @remote_bucket_name = remote_bucket_name - @tmp_bucket_name = tmp_bucket_name - @backend = backend - end - - def backup - if @backend == "s3" - check_bucket_cmd = %W(s3cmd ls s3://#{@remote_bucket_name}) - cmd = %W(s3cmd --stop-on-error --delete-removed sync s3://#{@remote_bucket_name}/ /srv/gitlab/tmp/#{@name}/) - elsif @backend == "gcs" - check_bucket_cmd = %W(gsutil ls gs://#{@remote_bucket_name}) - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} /srv/gitlab/tmp/#{@name}) - end - - # Check if the bucket exists - output, status = run_cmd(check_bucket_cmd) - unless status.zero? - puts "Bucket not found: #{@remote_bucket_name}. Skipping backup of #{@name} ...".blue - return - end - - puts "Dumping #{@name} ...".blue - - # create the destination: gsutils requires it to exist, s3cmd does not - FileUtils.mkdir_p("/srv/gitlab/tmp/#{@name}", mode: 0700) - - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - # check the destiation for contents. Bucket may have been empty. - if Dir.empty? "/srv/gitlab/tmp/#{@name}" - puts "empty".green - return - end - - # build gzip command used for tar compression - gzip_cmd = 'gzip' + (ENV['GZIP_RSYNCABLE'] == 'yes' ? ' --rsyncable' : '') - - cmd = %W(tar -cf #{@local_tar_path} -I #{gzip_cmd} -C /srv/gitlab/tmp/#{@name} . ) - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - - puts "done".green - end - - def restore - puts "Restoring #{@name} ...".blue - - backup_existing - cleanup - restore_from_backup - puts "done".green - end - - def failure_abort(error_message) - puts "[Error] #{error_message}".red - abort "Restore #{@name} failed" - end - - def upload_to_object_storage(source_path) - if @backend == "s3" - dir_name = File.basename(source_path) - cmd = %W(s3cmd --stop-on-error sync #{source_path}/ s3://#{@remote_bucket_name}/#{dir_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r #{source_path}/ gs://#{@remote_bucket_name}) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def backup_existing - backup_file_name = "#{@name}.#{Time.now.to_i}" - - if @backend == "s3" - cmd = %W(s3cmd sync s3://#{@remote_bucket_name} s3://#{@tmp_bucket_name}/#{backup_file_name}/) - elsif @backend == "gcs" - cmd = %W(gsutil -m rsync -r gs://#{@remote_bucket_name} gs://#{@tmp_bucket_name}/#{backup_file_name}/) - end - - output, status = run_cmd(cmd) - - failure_abort(output) unless status.zero? - end - - def cleanup - if @backend == "s3" - cmd = %W(s3cmd --stop-on-error del --force --recursive s3://#{@remote_bucket_name}) - elsif @backend == "gcs" - # Check if the bucket has any objects - list_objects_cmd = %W(gsutil ls gs://#{@remote_bucket_name}/) - output, status = run_cmd(list_objects_cmd) - failure_abort(output) unless status.zero? - - # There are no objects in the bucket so skip the cleanup - if output.length == 0 - return - end - - cmd = %W(gsutil rm -f -r gs://#{@remote_bucket_name}/*) - end - output, status = run_cmd(cmd) - failure_abort(output) unless status.zero? - end - - def restore_from_backup - extracted_tar_path = File.join(File.dirname(@local_tar_path), "/srv/gitlab/tmp/#{@name}") - FileUtils.mkdir_p(extracted_tar_path, mode: 0700) - - failure_abort("#{@local_tar_path} not found") unless File.exist?(@local_tar_path) - - untar_cmd = %W(tar -xf #{@local_tar_path} -C #{extracted_tar_path}) - - output, status = run_cmd(untar_cmd) - - failure_abort(output) unless status.zero? - - Dir.glob("#{extracted_tar_path}/*").each do |file| - upload_to_object_storage(file) - end - end - - def run_cmd(cmd) - _, stdout, wait_thr = Open3.popen2e(*cmd) - return stdout.read, wait_thr.value.exitstatus - end - -end diff --git a/12.8/scripts/prebuild.sh b/12.8/scripts/prebuild.sh deleted file mode 100755 index ae8b162c3d95a520831419b413bb0425c73cb445..0000000000000000000000000000000000000000 --- a/12.8/scripts/prebuild.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -### Environment Variables ### -GITLAB_VERSION=v12.8.1-ubi8 -VENDOR=gitlab - - -# These three variables are required to push whatever outside binaries your container needs at build time to our Nexus repo -NEXUS_SERVER=${NEXUS_SERVER} -NEXUS_USERNAME=${NEXUS_USERNAME} -NEXUS_PASSWORD=${NEXUS_PASSWORD} - - -## Download variables -DOWNLOAD_DIR=tmp/${VENDOR} -UPLOAD_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/kubectl - -### Download files/dependencies ### -# temporarily place your binaries locally in the download directory -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### GPG Signature Check ### -# GPG signature verification is a requirement in the case that the downloaded files have a GPG signature -# For more information on GPG keys visit https://access.redhat.com/solutions/1541303 or https://gnupg.org/gph/en/manual.html -curl --create-dirs https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -o ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc - -for server in $(shuf -e ha.pool.sks-keyservers.net \ - hkp://p80.pool.sks-keyservers.net:80 \ - keyserver.ubuntu.com \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu) ; -do - gpg --batch --keyserver "$server" --recv-keys 5c7738cc4840f93f6e9170ff5a0e20d5f9706778 && break || : ; \ -done - -gpg --verify ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc ${DOWNLOAD_DIR}/ubi8-build-dependencies-${GITLAB_VERSION}.tar - - -### SHA256 Verification ### -# Verifying the files with the SHA256 is a requirement for all files -# Make sure to not download the SHA256 from the internet, but create it, check it and upload it to the Nexus repo -cd ${DOWNLOAD_DIR} -sha256sum ubi8-build-dependencies-${GITLAB_VERSION}.tar | awk '{print $1}' > ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 \ -&& echo "$(cat ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256) ubi8-build-dependencies-${GITLAB_VERSION}.tar" | sha256sum --check --status \ -&& if [ $? == '0' ]; then printf "\nSHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar succeeded\n\n"; \ -else printf "SHA256 check for ubi8-build-dependencies-${GITLAB_VERSION}.tar failed\n\n"; fi -cd - - -### Nexus Repo Upload ### -# Push whatever binaries you need to ${NEXUS_SERVER}/dsop/vendor/project/ as you see in the example below. Follow the same -# format as in Gitlab. You will also need to push the GPG signature file and SHA256 file -for package in ubi8-build-dependencies-${GITLAB_VERSION}.tar ubi8-build-dependencies-${GITLAB_VERSION}.tar.sha256 ubi8-build-dependencies-${GITLAB_VERSION}.tar.asc -do - curl -kfS -u ${NEXUS_USERNAME}:${NEXUS_PASSWORD} -T ${DOWNLOAD_DIR}/${package} https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${package} -done diff --git a/12.8/Dockerfile b/Dockerfile similarity index 87% rename from 12.8/Dockerfile rename to Dockerfile index 88e1ad464ec356f301257a25bf0607f717308ebc..c366b6511287723fa98eec5bde697642eaeb3440 100644 --- a/12.8/Dockerfile +++ b/Dockerfile @@ -9,15 +9,13 @@ ARG RAILS_IMAGE=${BASE_REGISTRY}/gitlab/gitlab/gitlab-rails:12.8 FROM ${UBI_IMAGE} AS builder -ARG NEXUS_SERVER -ARG VENDOR=gitlab +ARG GITLAB_VERSION ARG PACKAGE_NAME=ubi8-build-dependencies-${GITLAB_VERSION}.tar -ARG PACKAGE_URL=https://${NEXUS_SERVER}/repository/dsop/${VENDOR}/gitlab-task-runner/${PACKAGE_NAME} +COPY ${PACKAGE_NAME} /opt/ ADD build-scripts/ /build-scripts/ -RUN /build-scripts/prepare.sh "${PACKAGE_URL}" - +RUN /build-scripts/prepare.sh "/opt/${PACKAGE_NAME}" FROM ${RAILS_IMAGE} diff --git a/12.5/LICENSE b/LICENSE similarity index 100% rename from 12.5/LICENSE rename to LICENSE diff --git a/README.md b/README.md index 6f916b9601c5cf753fc4be224058cf4be7df9254..3a26688ab4a95d8cbd65b7d239bdd62f29967b95 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,41 @@ # gitlab-task-runner-container +Task Runner is an entry point for interaction with other containers in the cluster. It contains scripts for running Rake tasks, backup, restore, and tools to intract with object storage. + +## GitLab Build + +The hardened containers for GitLab require the gitlab correctly version assets blob to be available for download. + +There are some included scripts to make this easier for user building images offline on their machine. + + +## Building + +1. Switch to the desired versioned directory +2. Run `./build-scripts/build.sh` + * Runs docker build, and takes care of setting the appropriate build args for users running locally + + +`build-script/build.sh` is provided as an example and helpful for building locally. You can also instead call docker build and pass build-args directly. + +## Build Phases + +Some of the GitLab containers are build ontop of previous containers, building the containers in ordered phases is necessary to build all containers. + +- Phase One + * kubectl + * gitlab-ruby + * gitlab-container-registry +- Phase Two + * git-base + * gitlab-exporter + * gitlab-mailroom + * gitlab-shell + * gitlab-rails + * gitlab-workhorse +- Phase 3 + * gitaly +- Phase 4 + * gitlab-sidekiq + * gitlab-task-runner + * gitlab-unicorn diff --git a/12.8/build-scripts/build.sh b/build-scripts/build.sh similarity index 69% rename from 12.8/build-scripts/build.sh rename to build-scripts/build.sh index 0470f34407dab512a474983ec8c2858a0b6a3ee2..f67ccc6725ce661c2bc08aeca21ead0c3320f2ec 100755 --- a/12.8/build-scripts/build.sh +++ b/build-scripts/build.sh @@ -6,8 +6,6 @@ set -euxo pipefail TAG=${1:-12.8} REPOSITORY=${2:-} -NEXUS_SERVER=${NEXUS_SERVER:-} -PACKAGE_URL=${PACKAGE_URL:-} DOCKER_OPTS=${DOCKER_OPTS:-""} imageName() { @@ -29,13 +27,5 @@ buildImage() { # Cleanup log outputs from previous build rm -f *.out failed.log -if [ $NEXUS_SERVER ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg NEXUS_SERVER=${NEXUS_SERVER}" -fi - -if [ $PACKAGE_URL ]; then - DOCKER_OPTS="$DOCKER_OPTS --build-arg PACKAGE_URL=${PACKAGE_URL}" -fi - DOCKER_OPTS="$DOCKER_OPTS --build-arg RAILS_IMAGE=$(imageName gitlab-rails)" buildImage gitlab-task-runner diff --git a/12.5/build-scripts/cleanup.sh b/build-scripts/cleanup.sh similarity index 100% rename from 12.5/build-scripts/cleanup.sh rename to build-scripts/cleanup.sh diff --git a/build-scripts/prepare.sh b/build-scripts/prepare.sh new file mode 100755 index 0000000000000000000000000000000000000000..242b66826bcba130d92bcbb70d7651858307fd73 --- /dev/null +++ b/build-scripts/prepare.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euxo pipefail + +PACKAGE_PATH=$1 +WORKSPACE="${WORKSPACE:-/prepare}" + +mkdir -p ${WORKSPACE}/dependencies + +# Extract UBI dependencies +tar -xvf "${PACKAGE_PATH}" -C "${WORKSPACE}" + +# Extract the specific depenencies needed for this contianer +tar -xvf "${WORKSPACE}/gitlab-python.tar.gz" -C "${WORKSPACE}/dependencies" +tar -xvf "${WORKSPACE}/gitlab-task-runner-ee.tar.gz" -C "${WORKSPACE}/dependencies" diff --git a/download.json b/download.json new file mode 100644 index 0000000000000000000000000000000000000000..7f52925955dba496199b35573be81ddb97857bf8 --- /dev/null +++ b/download.json @@ -0,0 +1,7 @@ +{ "resources": + [ + { "url" : "https://gitlab-ubi.s3.us-east-2.amazonaws.com/ubi8-build-dependencies-v12.8.1-ubi8.tar", + "filename": "ubi8-build-dependencies-v12.8.1-ubi8.tar", + "sha256": "1d49f47924738132f4d10cb9b32ce6d9c20b151dfc3b056c298607f267c75571" + } +] } diff --git a/12.7/scripts/bin/backup-utility b/scripts/bin/backup-utility similarity index 100% rename from 12.7/scripts/bin/backup-utility rename to scripts/bin/backup-utility diff --git a/12.3/scripts/bin/entrypoint.sh b/scripts/bin/entrypoint.sh similarity index 100% rename from 12.3/scripts/bin/entrypoint.sh rename to scripts/bin/entrypoint.sh diff --git a/12.3/scripts/bin/gitlab-rails b/scripts/bin/gitlab-rails similarity index 100% rename from 12.3/scripts/bin/gitlab-rails rename to scripts/bin/gitlab-rails diff --git a/12.3/scripts/bin/gitlab-rake b/scripts/bin/gitlab-rake similarity index 100% rename from 12.3/scripts/bin/gitlab-rake rename to scripts/bin/gitlab-rake diff --git a/12.3/scripts/bin/object-storage-backup b/scripts/bin/object-storage-backup similarity index 100% rename from 12.3/scripts/bin/object-storage-backup rename to scripts/bin/object-storage-backup diff --git a/12.3/scripts/bin/object-storage-restore b/scripts/bin/object-storage-restore similarity index 100% rename from 12.3/scripts/bin/object-storage-restore rename to scripts/bin/object-storage-restore diff --git a/12.4/scripts/lib/object_storage_backup.rb b/scripts/lib/object_storage_backup.rb similarity index 100% rename from 12.4/scripts/lib/object_storage_backup.rb rename to scripts/lib/object_storage_backup.rb