-
Micah Nagel authoredMicah Nagel authored
.gitlab-ci.yml 16.54 KiB
# Bigbang CI pipelines defintiions
# There are 4 different use-cases/pipelines that are supported by this file:
# 1) Build and test changes made from a Merge Reuqest using a docker-in-docker (DIND) K3D cluster deployment inside the
# gitlab runner.
# 2) Build and test a commit to the Master branch (default branch) using a AWS created K3S cluster which is deployed
# using Terraform
# 3) Build and test a new release and/or tagged commit using a docker-in-docker (DIND) K3D cluster deployment inside the
# gitlab runner. Once a successful build and test is completed, package the build and perform a release operation.
# 4) Periodically at a scheduled time, build and test the master branch using a AWS created K3S cluster which is deployed
# using Terraform
# global rules for when pipelines run
workflow:
rules:
# run pipeline for manual tag events such as a new release
- if: $CI_COMMIT_TAG
# run pipeline on merge request events
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
# run pipeline on commits to default branch
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# skip pipeline for branches that start with "docs"
- if: '$CI_COMMIT_REF_NAME =~ /^doc*/i'
when: never
# Enabled CI pipeline testing it commit message contains "test-ci"
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_MESSAGE =~ /.*test-ci.*/i
# Include templates and cluster creation jobs
include:
- local: '/.gitlab-ci/templates.yml'
# Pipeline stages
# - Smoke tests are executed for all pipelines except scheduled nightly runs.
# -
stages:
- smoke tests
- network up
- cluster up
- bigbang up
- test
- bigbang down
- cluster down
- network down
- package
- release
variables:
RELEASE_BUCKET: umbrella-bigbang-releases
IMAGE_LIST: images.txt
IMAGE_PKG: images.tar.gz
REPOS_PKG: repositories.tar.gz
VALUES_FILE: chart/values.yaml
CI_VALUES_FILE: tests/ci/k3d/values.yaml
#-----------------------------------------------------------------------------------------------------------------------
# Pre Stage Jobs. This execute before any job is run.
#
pre vars:
image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3
stage: .pre
extends:
- .bigbang-gitlab-runner-tags
artifacts:
reports:
dotenv: variables.env
script:
# Create the TF_VAR_env variable
- echo "TF_VAR_env=$(echo $CI_COMMIT_REF_SLUG | cut -c 1-7)-$(echo $CI_COMMIT_SHA | cut -c 1-7)" >> variables.env
- cat variables.env
#-----------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------
# Smoke Tests
#
.chart_changes: &chart_changes
changes:
- chart/**/*
- .gitlab-ci.yml
- .gitlab-ci/jobs/**/*
- scripts/**/*
- tests/**/*
- base/flux/*
.deploy_bigbang: &deploy_bigbang
- |
set -e
for deploy_script in $(find ./tests/deploy -type f -name '*.sh' | sort); do
chmod +x ${deploy_script}
echo -e "\e[0Ksection_start:`date +%s`:${deploy_script##*/}[collapsed=true]\r\e[0K\e[33;1m${deploy_script##*/}\e[37m"
./${deploy_script}
echo -e "\e[0Ksection_end:`date +%s`:${deploy_script##*/}\r\e[0K"
done
.test_bigbang: &test_bigbang
- |
set -e
for test_script in $(find ./tests/tests -type f -name '*.sh' | sort); do
echo -e "\e[0Ksection_start:`date +%s`:${test_script##*/}[collapsed=true]\r\e[0K\e[33;1m${test_script##*/}\e[37m"
chmod +x ${test_script}
echo "Executing ${test_script}..."
./${test_script} && export EXIT_CODE=$? || export EXIT_CODE=$?
if [[ ${EXIT_CODE} -ne 0 ]]; then
echo "${test_script} failed, see log output above and cluster debug."
exit ${EXIT_CODE}
fi
echo -e "\e[0Ksection_end:`date +%s`:${test_script##*/}\r\e[0K"
done
clean install:
stage: smoke tests
extends:
- .k3d-ci
variables:
CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}"
rules:
# Always run a clean installation test unless we are deploying the AWS cluster installation during a scheduled test (nightly master test)
- if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
when: never
- *chart_changes
script:
- *deploy_bigbang
# Fetch list of all images ran (retry crictl up to 6x)
- echo -e "\e[0Ksection_start:`date +%s`:images_used[collapsed=true]\r\e[0K\e[33;1mImages Used\e[37m"
- cid=$(docker ps -aqf "name=k3d-${CI_JOB_ID}-server-0")
- images=$(timeout 65 bash -c "until docker exec $cid crictl images -o json; do sleep 10; done;")
- echo $images | jq -r '.images[].repoTags[0] | select(. != null)' | tee images.txt
- echo -e "\e[0Ksection_end:`date +%s`:images_used\r\e[0K"
- *test_bigbang
artifacts:
paths:
- images.txt
- "test-artifacts/"
expire_in: 3 days
when: always
allow_failure:
exit_codes: 123
upgrade: stage: smoke tests
dependencies:
- pre vars
extends:
- .k3d-ci
rules:
# skip job for nightly master and "test-ci::infra" labeled pipelines
- if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
when: never
# skip job when MR title starts with 'SKIP UPGRADE'
- if: '$CI_MERGE_REQUEST_TITLE =~ /SKIP UPGRADE/'
when: never
# run pipeline on merge request events
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
<<: *chart_changes
variables:
CLUSTER_NAME: "upgrade-${CI_COMMIT_SHORT_SHA}"
script:
- echo "Install Big Bang from ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
- echo -e "\e[0Ksection_start:`date +%s`:git_master[collapsed=true]\r\e[0K\e[33;1mGit Fetch Master\e[37m"
- git fetch && git checkout ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}
- echo -e "\e[0Ksection_end:`date +%s`:git_master\r\e[0K"
- *deploy_bigbang
- *test_bigbang
- echo "Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
- echo -e "\e[0Ksection_start:`date +%s`:git_upgrade[collapsed=true]\r\e[0K\e[33;1mGit Upgrade\e[37m"
- git reset --hard && git clean -fd
- git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}
- echo -e "\e[0Ksection_end:`date +%s`:git_upgrade\r\e[0K"
- *deploy_bigbang
- *test_bigbang
artifacts:
paths:
- "test-artifacts/"
expire_in: 3 days
when: always
allow_failure:
exit_codes: 123
#-----------------------------------------------------------------------------------------------------------------------
# Rules for execution of AWS based K3S cluster deployment: Infrastructure jobs
#
# Abstract for job manually triggering infrastructure builds
.infra fork:
stage: network up
rules:
# Run on scheduled jobs OR when `test-ci` label is assigned
- if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
allow_failure: false
# Abstract for jobs responsible for creating infrastructure
.infra create:
rules:
# Run on scheduled jobs OR when `test-ci` label is assigned
- if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
# skip job when branch name starts with "hotfix" or "patch"
- if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^(hotfix|patch)/'
when: never
# Abstract for jobs responsible for cleaning up infrastructure OR when `test-ci` label is assigned
.infra cleanup:
rules:
# Run on scheduled jobs
- if: '($CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "master") || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::infra(,|$)/'
allow_failure: true
when: always
#-----------------------------------------------------------------------------------------------------------------------
# Infrastructure: Networking#
aws/network up:
extends:
- .bigbang-gitlab-runner-tags
- .infra fork
- .network up
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
auto_stop_in: 1 hour
retry:
max: 2
when:
- unknown_failure
- stuck_or_timeout_failure
- runner_system_failure
aws/network down:
extends:
- .bigbang-gitlab-runner-tags
- .infra cleanup
- .network down
stage: network down
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
action: stop
retry:
max: 2
when:
- unknown_failure
- stuck_or_timeout_failure
- runner_system_failure
#-----------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------
# Infrastructure: RKE2
#
# Create RKE2 cluster on AWS
aws/rke2/cluster up:
stage: cluster up
extends:
- .bigbang-gitlab-runner-tags
- .infra create
- .rke2 up
needs:
- job: aws/network up
- job: pre vars
artifacts: true
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
retry:
max: 2
when:
- unknown_failure
- stuck_or_timeout_failure
- runner_system_failure
# Install BigBang on RKE2 cluster on AWS
aws/rke2/bigbang up:
stage: bigbang up
extends:
- .bigbang-gitlab-runner-tags
- .infra create
- .kubectl-output
needs:
- job: aws/rke2/cluster up
artifacts: true
before_script: - mkdir -p ~/.kube
- cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config
# Deploy a default storage class for aws
- kubectl apply -f ${CI_PROJECT_DIR}/.gitlab-ci/jobs/rke2/dependencies/k8s-resources/aws/default-ebs-sc.yaml
script:
- *deploy_bigbang
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
retry:
max: 2
when:
- unknown_failure
- stuck_or_timeout_failure
- runner_system_failure
# Run tests on BigBang on RKE2 cluster on AWS
aws/rke2/bigbang test:
stage: test
extends:
- .bigbang-gitlab-runner-tags
- .infra create
- .kubectl-output
needs:
- job: aws/rke2/cluster up
artifacts: true
- job: aws/rke2/bigbang up
before_script:
- mkdir -p ~/.kube
- cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config
script:
## Move this yum install to the dockerfile for the builder
## putting it here now for a quick way to install dig
- echo -e "\e[0Ksection_start:`date +%s`:host_setup[collapsed=true]\r\e[0K\e[33;1mHost Setup\e[37m"
- yum install bind-utils -y
- ./scripts/hosts.sh
- echo -e "\e[0Ksection_end:`date +%s`:host_setup\r\e[0K"
- *test_bigbang
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
allow_failure:
exit_codes: 123
retry:
max: 2
when:
- unknown_failure
- stuck_or_timeout_failure
- runner_system_failure
artifacts:
paths:
- "test-artifacts/"
expire_in: 3 days
when: always
# Uninstall BigBang on RKE2 cluster on AWS
aws/rke2/bigbang down:
stage: bigbang down
extends:
- .bigbang-gitlab-runner-tags
- .infra cleanup
- .kubectl-output
needs:
- job: aws/rke2/cluster up
artifacts: true
- job: aws/rke2/bigbang test
before_script:
- mkdir -p ~/.kube
- cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config
script:
- helm un -n bigbang bigbang # TODO: Smarter wait
- sleep 180
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
# Destroy RKE2 cluster on AWS
aws/rke2/cluster down:
stage: cluster down
extends:
- .bigbang-gitlab-runner-tags
- .infra cleanup
- .rke2 down
needs:
- job: aws/rke2/bigbang down
- job: pre vars
artifacts: true
environment:
name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
retry:
max: 2
when:
- unknown_failure
- stuck_or_timeout_failure
- runner_system_failure
#-----------------------------------------------------------------------------------------------------------------------
#-----------------------------------------------------------------------------------------------------------------------
# Release Jobs
#
package:
stage: package
image: registry.dso.mil/platform-one/big-bang/bigbang/synker:0.0.3
extends:
- .bigbang-gitlab-runner-tags
rules:
# run job for manual tag events or test-ci::release MRs
- if: '$CI_COMMIT_TAG || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::release(,|$)/'
before_script:
# Set up auth
- mkdir -p /root/.docker
- |
jq -n '{"auths": {"registry.dso.mil": {"auth": $bb_registry_auth}, "registry1.dso.mil": {"auth": $registry1_auth}, "registry.il2.dso.mil": {"auth": $il2_registry_auth} } }' \
--arg bb_registry_auth ${BB_REGISTRY_AUTH} \
--arg registry1_auth ${REGISTRY1_AUTH} \
--arg il2_registry_auth ${IL2_REGISTRY_AUTH} > /root/.docker/config.json
script:
- echo -e "\e[0Ksection_start:`date +%s`:synker_pull[collapsed=true]\r\e[0K\e[33;1mSynker Pull\e[37m"
- cp ./scripts/package/synker.yaml ./synker.yaml
# Populate images list in synker config
- |
for image in $(cat images.txt); do
yq -i e "(.source.images |= . + \"${image}\")" "./synker.yaml"
done
- synker pull -b=1
- echo -e "\e[0Ksection_end:`date +%s`:synker_pull\r\e[0K"
# Create image list from synker, overwrite since ./synker.yaml contains everything at this point
- yq e '.source.images | .[] | ... comments=""' "./synker.yaml" > images.txt
# Tar up synker as well?
- cp /usr/local/bin/synker synker.yaml /var/lib/registry/
# Grab the registry image
- crane pull registry:2 registry.tar
- mv registry.tar /var/lib/registry/
- echo -e "\e[0Ksection_start:`date +%s`:package_synker[collapsed=true]\r\e[0K\e[33;1mPackage Images\e[37m"
- tar -C /var/lib/registry -czvf $IMAGE_PKG .
- echo -e "\e[0Ksection_end:`date +%s`:package_synker\r\e[0K"
- echo -e "\e[0Ksection_start:`date +%s`:unpack_images[collapsed=true]\r\e[0K\e[33;1mUnpack Images\e[37m"
- tar -czvf $IMAGE_PKG /var/lib/registry
- echo -e "\e[0Ksection_end:`date +%s`:unpack_images\r\e[0K" # Package dependent repos
- echo -e "\e[0Ksection_start:`date +%s`:package_repos[collapsed=true]\r\e[0K\e[33;1mPackage Repos\e[37m"
- ./scripts/package/gits.sh
- tar -czf $REPOS_PKG repos/
- echo -e "\e[0Ksection_end:`date +%s`:package_repos\r\e[0K"
# Prep release
- mkdir -p release
- mv $IMAGE_LIST $IMAGE_PKG $REPOS_PKG release/
# Publish packages to s3 release
- |
if [ -z $CI_COMMIT_TAG ]; then
aws s3 sync --quiet release/ s3://umbrella-bigbang-releases/tests/${CI_COMMIT_SHA}
else
aws s3 sync --quiet release/ s3://umbrella-bigbang-releases/umbrella/${CI_COMMIT_TAG}
fi
after_script: []
release:
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
extends:
- .bigbang-gitlab-runner-tags
rules:
# run job for manual tag events or test-ci::release MRs
- if: '$CI_COMMIT_TAG || $CI_MERGE_REQUEST_LABELS =~ /(^|,)test-ci::release(,|$)/'
variables:
RELEASE_ENDPOINT: https://${RELEASE_BUCKET}.s3-${AWS_DEFAULT_REGION}.amazonaws.com/umbrella/${CI_COMMIT_TAG}
script:
# Use release-cli to cut a release in Gitlab or simulate a dry-run & print asset links
- |
if [ -z $CI_COMMIT_TAG ]; then
RELEASE_ENDPOINT="https://${RELEASE_BUCKET}.s3-${AWS_DEFAULT_REGION}.amazonaws.com/tests/${CI_COMMIT_SHA}"
printf "Release will run: \n\
release-cli create --name \"Big Bang \${CI_COMMIT_TAG}\" --tag-name \${CI_COMMIT_TAG} \n\
--description \"Automated release notes are a WIP.\" \n\
--assets-link \"{\"name\":\"${IMAGE_LIST}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_LIST}\"}\" \n\
--assets-link \"{\"name\":\"${IMAGE_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_PKG}\"}\" \n\
--assets-link \"{\"name\":\"${REPOS_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${REPOS_PKG}\"}\"\n"
else
release-cli create --name "Big Bang ${CI_COMMIT_TAG}" --tag-name ${CI_COMMIT_TAG} \
--description "Automated release notes are a WIP." \
--assets-link "{\"name\":\"${IMAGE_LIST}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_LIST}\"}" \
--assets-link "{\"name\":\"${IMAGE_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_PKG}\"}" \
--assets-link "{\"name\":\"${REPOS_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${REPOS_PKG}\"}"
fi
#-----------------------------------------------------------------------------------------------------------------------