# global rules for when pipelines run workflow: rules: # run pipeline for manual tag events - if: $CI_COMMIT_TAG # run pipeline on merge request events - if: $CI_PIPELINE_SOURCE == "merge_request_event" # run pipeline on commits to default branch - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # skip pipeline for branches that start with "docs" - if: '$CI_COMMIT_REF_NAME =~ /^docs/' when: never # include templates include: - local: '/.gitlab-ci/templates.yml' stages: - smoke tests - network up - cluster up - bigbang up - test - bigbang down - cluster down - network down - package - release variables: RELEASE_BUCKET: umbrella-bigbang-releases IMAGE_LIST: images.txt IMAGE_PKG: images.tar.gz REPOS_PKG: repositories.tar.gz #----------------------------------------------------------------------------------------------------------------------- # Pre Stage Jobs # changelog: image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3 stage: .pre allow_failure: true variables: CHANGELOG_FILE: CHANGELOG.md script: - ./scripts/lint_changelog.sh version: image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3 stage: .pre allow_failure: true variables: CHART_FILE: chart/Chart.yaml script: - ./scripts/lint_version.sh commitlint: image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3 stage: .pre allow_failure: true script: - ./scripts/lint_version.sh pre vars: image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3 stage: .pre artifacts: reports: dotenv: variables.env script: # obtain MR and master versions - CHART_MR_VERSION=$(sed -n -e 's/^version. //p' chart/Chart.yaml) - git fetch && git checkout ${CI_DEFAULT_BRANCH} - CHART_MA_VERSION=$(sed -n -e 's/^version. //p' chart/Chart.yaml) - git fetch && git checkout ${CI_COMMIT_REF_NAME} - echo "CHART_MR_VERSION=$CHART_MR_VERSION" >> variables.env - echo "CHART_MA_VERSION=$CHART_MA_VERSION" >> variables.env # obtain semver differences (subtract master version from mr verison) - CHART_VERSION_DIFF=$(./scripts/semver_diff.sh $CHART_MR_VERSION $CHART_MA_VERSION) - IFS=. DIFF_ARR=(${CHART_VERSION_DIFF##*-}) - echo "CHART_VERSION_DIFF=$CHART_VERSION_DIFF" >> variables.env # detect breaking change (first two version sections in semver diff) - CHART_BREAKING_CHANGE="false" - if (( ${DIFF_ARR[0]} > 0 )); then CHART_BREAKING_CHANGE="true"; fi - if (( ${DIFF_ARR[1]} > 0 )); then CHART_BREAKING_CHANGE="true"; fi # store variables - echo "CHART_BREAKING_CHANGE=$CHART_BREAKING_CHANGE" >> variables.env # Create the TF_VAR_env variable - echo "TF_VAR_env=$(echo $CI_COMMIT_REF_SLUG | cut -c 1-7)-$(echo $CI_COMMIT_SHA | cut -c 1-7)" >> variables.env - cat variables.env #----------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------- # Smoke Tests # .bigbang: image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/k3d-builder:0.0.1 after_script: - kubectl get all -A - kubectl get helmrelease -A .deploy_bigbang: &deploy_bigbang - find ./scripts/deploy -type f -name '*.sh' | sort | xargs -r -I {} sh -c 'echo {} && sh {}' .test_bigbang: &test_bigbang - find ./tests -type f -name '*.sh' | sort | xargs -r -I {} sh -c 'echo {} && sh {}' clean install: stage: smoke tests extends: - .k3d variables: CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}" script: - *deploy_bigbang - *test_bigbang - docker exec -i k3d-${CLUSTER_NAME}-server-0 crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)' > images.txt artifacts: paths: - images.txt expire_in: 7 days upgrade: stage: smoke tests dependencies: - pre vars extends: - .k3d rules: # skip job when MR title starts with 'Breaking Change' - if: '$CI_MERGE_REQUEST_TITLE =~ /^Breaking Change/' when: never # run pipeline on merge request events - if: $CI_PIPELINE_SOURCE == "merge_request_event" variables: CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}" script: - if $CHART_BREAKING_CHANGE; then echo "Breaking change detected by chart version difference, skipping job"; exit 0; fi - echo "Install Big Bang from ${CI_DEFAULT_BRANCH}" - git fetch && git checkout ${CI_DEFAULT_BRANCH} - *deploy_bigbang - *test_bigbang - echo "Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" - git reset --hard && git clean -fd - git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} - *deploy_bigbang - *test_bigbang #----------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------- # Infrastructure: Management Jobs # # Abstract for job manually triggering infrastructure builds .infra fork: stage: network up rules: # skip job when branch name starts with "hotfix" or "patch" - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^(hotfix|patch)/' when: never # run job on merge requests when manually activated - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' allow_failure: false when: manual # Abstract for jobs responsible for creating infrastructure .infra create: rules: # skip job when branch name starts with "hotfix" or "patch" - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^(hotfix|patch)/' when: never # run pipeline on commits to default branch - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' # Abstract for jobs responsible for cleaning up infrastructure .infra cleanup: rules: # skip job when branch name starts with "hotfix" or "patch" - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^(hotfix|patch)/' when: never # run job on merge requests regardless of failure - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' allow_failure: true when: always #----------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------- # Infrastructure: Networking # aws/network up: extends: - .infra fork - .network up environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} auto_stop_in: 1 hour aws/network down: extends: - .infra cleanup - .network down stage: network down environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} action: stop #----------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------- # Infrastructure: RKE2 # # Create RKE2 cluster on AWS aws/rke2/cluster up: stage: cluster up extends: - .infra create - .rke2 up needs: - job: aws/network up - job: pre vars artifacts: true environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} # Install BigBang on RKE2 cluster on AWS aws/rke2/bigbang up: stage: bigbang up extends: - .infra create - .bigbang needs: - job: aws/rke2/cluster up artifacts: true before_script: - mkdir -p ~/.kube - cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config # Deploy a default storage class for aws - kubectl apply -f ${CI_PROJECT_DIR}/.gitlab-ci/jobs/rke2/dependencies/k8s-resources/aws/default-ebs-sc.yaml script: - *deploy_bigbang environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} # Run tests on BigBang on RKE2 cluster on AWS aws/rke2/bigbang test: stage: test extends: - .infra create - .bigbang needs: - job: aws/rke2/cluster up artifacts: true - job: aws/rke2/bigbang up before_script: - mkdir -p ~/.kube - cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config script: ## Move this yum install to the dockerfile for the builder ## putting it here now for a quick way to install dig - yum install bind-utils -y - ./scripts/hosts.sh - *test_bigbang environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} # Uninstall BigBang on RKE2 cluster on AWS aws/rke2/bigbang down: stage: bigbang down extends: - .infra cleanup - .bigbang needs: - job: aws/rke2/cluster up artifacts: true - job: aws/rke2/bigbang test before_script: - mkdir -p ~/.kube - cp ${CI_PROJECT_DIR}/rke2.yaml ~/.kube/config script: - helm un -n bigbang bigbang # TODO: Smarter wait - sleep 180 environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} # Destroy RKE2 cluster on AWS aws/rke2/cluster down: stage: cluster down extends: - .infra cleanup - .rke2 down needs: - job: aws/rke2/bigbang down - job: pre vars artifacts: true environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} #----------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------- # Release Jobs # package: stage: package image: registry.dsop.io/platform-one/big-bang/umbrella/synker:0.0.1 rules: # run job for manual tag events - if: $CI_COMMIT_TAG # when: never # # run job on commits to default branch # - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH before_script: # Set up auth - mkdir -p /root/.docker - echo $DOCKER_AUTH_CONFIG > /root/.docker/config.json script: - cp ./scripts/package/synker.yaml ./synker.yaml # Populate images list in synker config # NOTE: This is stupid and hacky, use yq or something better to do this - cat images.txt | sed -e 's/^/ - /' >> synker.yaml - synker pull -b=1 # Tar up synker as well? - cp /usr/local/bin/synker synker.yaml /var/lib/registry/ - tar -C /var/lib/registry -czvf $IMAGE_PKG . - tar -czvf $IMAGE_PKG /var/lib/registry # Package dependent repos - ./scripts/package/gits.sh - tar -czf $REPOS_PKG repos/ # Prep release - mkdir -p release - mv $IMAGE_LIST $IMAGE_PKG $REPOS_PKG release/ # Publish packages to s3 release - aws s3 sync --quiet release/ s3://umbrella-bigbang-releases/umbrella/${CI_COMMIT_TAG} after_script: [] release: stage: release image: registry.gitlab.com/gitlab-org/release-cli:latest rules: # run job for manual tag events - if: $CI_COMMIT_TAG # when: never # # run job on commits to default branch # - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH variables: RELEASE_ENDPOINT: https://${RELEASE_BUCKET}.s3-${AWS_DEFAULT_REGION}.amazonaws.com/umbrella/${CI_COMMIT_TAG} script: - | release-cli create --name "Big Bang v${CI_COMMIT_TAG}" --tag-name ${CI_COMMIT_TAG} \ --description "Automated release notes are a WIP." \ --assets-link "{\"name\":\"${IMAGE_LIST}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_LIST}\"}" \ --assets-link "{\"name\":\"${IMAGE_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_PKG}\"}" \ --assets-link "{\"name\":\"${REPOS_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${REPOS_PKG}\"}" #-----------------------------------------------------------------------------------------------------------------------