diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36145ce6f8e7090bc141c63de8ec6bc17a9c9dd0..754ea1a83f9a87105afea83c29f9cc684e8f3a86 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -86,8 +86,9 @@ pre vars: set -e for deploy_script in $(find ./tests/deploy -type f -name '*.sh' | sort); do chmod +x ${deploy_script} - echo "Executing ${deploy_script}..." + echo -e "\e[0Ksection_start:`date +%s`:${deploy_script##*/}[collapsed=true]\r\e[0K\e[33;1m${deploy_script##*/}\e[37m" ./${deploy_script} + echo -e "\e[0Ksection_end:`date +%s`:${deploy_script##*/}\r\e[0K" done .test_bigbang: &test_bigbang @@ -95,8 +96,9 @@ pre vars: set -e for test_script in $(find ./tests/tests -type f -name '*.sh' | sort); do chmod +x ${test_script} - echo "Executing ${test_script}..." + echo -e "\e[0Ksection_start:`date +%s`:${test_script##*/}[collapsed=true]\r\e[0K\e[33;1m${test_script##*/}\e[37m" ./${test_script} + echo -e "\e[0Ksection_end:`date +%s`:${test_script##*/}\r\e[0K" done clean install: @@ -115,9 +117,11 @@ clean install: - *test_bigbang # Fetch list of all images ran (retry crictl up to 6x) + - echo -e "\e[0Ksection_start:`date +%s`:images_used[collapsed=true]\r\e[0K\e[33;1mImages Used\e[37m" - cid=$(docker ps -aqf "name=k3d-${CI_JOB_ID}-server-0") - images=$(timeout 65 bash -c "until docker exec $cid crictl images -o json; do sleep 10; done;") - echo $images | jq -r '.images[].repoTags[0] | select(. != null)' | tee images.txt + - echo -e "\e[0Ksection_end:`date +%s`:images_used\r\e[0K" artifacts: paths: - images.txt @@ -146,12 +150,16 @@ upgrade: CLUSTER_NAME: "upgrade-${CI_COMMIT_SHORT_SHA}" script: - echo "Install Big Bang from ${CI_DEFAULT_BRANCH}" + - echo -e "\e[0Ksection_start:`date +%s`:git_master[collapsed=true]\r\e[0K\e[33;1mGit Fetch Master\e[37m" - git fetch && git checkout ${CI_DEFAULT_BRANCH} + - echo -e "\e[0Ksection_end:`date +%s`:git_master\r\e[0K" - *deploy_bigbang - *test_bigbang - echo "Upgrade Big Bang from ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" + - echo -e "\e[0Ksection_start:`date +%s`:git_upgrade[collapsed=true]\r\e[0K\e[33;1mGit Upgrade\e[37m" - git reset --hard && git clean -fd - git checkout ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} + - echo -e "\e[0Ksection_end:`date +%s`:git_upgrade\r\e[0K" - *deploy_bigbang - *test_bigbang artifacts: @@ -295,8 +303,10 @@ aws/rke2/bigbang test: script: ## Move this yum install to the dockerfile for the builder ## putting it here now for a quick way to install dig + - echo -e "\e[0Ksection_start:`date +%s`:host_setup[collapsed=true]\r\e[0K\e[33;1mHost Setup\e[37m" - yum install bind-utils -y - ./scripts/hosts.sh + - echo -e "\e[0Ksection_end:`date +%s`:host_setup\r\e[0K" - *test_bigbang environment: name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA} @@ -371,6 +381,7 @@ package: --arg registry1_auth ${REGISTRY1_AUTH} \ --arg il2_registry_auth ${IL2_REGISTRY_AUTH} > /root/.docker/config.json script: + - echo -e "\e[0Ksection_start:`date +%s`:synker_pull[collapsed=true]\r\e[0K\e[33;1mSynker Pull\e[37m" - cp ./scripts/package/synker.yaml ./synker.yaml # Populate images list in synker config - | @@ -378,6 +389,7 @@ package: yq -i e "(.source.images |= . + \"${image}\")" "./synker.yaml" done - synker pull -b=1 + - echo -e "\e[0Ksection_end:`date +%s`:synker_pull\r\e[0K" # Create image list from synker, overwrite since ./synker.yaml contains everything at this point - yq e '.source.images | .[] | ... comments=""' "./synker.yaml" > images.txt # Tar up synker as well? @@ -385,11 +397,17 @@ package: # Grab the registry image - crane pull registry:2 registry.tar - mv registry.tar /var/lib/registry/ + - echo -e "\e[0Ksection_start:`date +%s`:package_synker[collapsed=true]\r\e[0K\e[33;1mPackage Images\e[37m" - tar -C /var/lib/registry -czvf $IMAGE_PKG . + - echo -e "\e[0Ksection_end:`date +%s`:package_synker\r\e[0K" + - echo -e "\e[0Ksection_start:`date +%s`:unpack_images[collapsed=true]\r\e[0K\e[33;1mUnpack Images\e[37m" - tar -czvf $IMAGE_PKG /var/lib/registry + - echo -e "\e[0Ksection_end:`date +%s`:unpack_images\r\e[0K" # Package dependent repos + - echo -e "\e[0Ksection_start:`date +%s`:package_repos[collapsed=true]\r\e[0K\e[33;1mPackage Repos\e[37m" - ./scripts/package/gits.sh - tar -czf $REPOS_PKG repos/ + - echo -e "\e[0Ksection_end:`date +%s`:package_repos\r\e[0K" # Prep release - mkdir -p release - mv $IMAGE_LIST $IMAGE_PKG $REPOS_PKG release/ diff --git a/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml b/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml index 6ceb8509aa10e66bf53d85bb2b297a36d76bad80..3c8d18dfa4e5e3727a5c8df16fd792eee4a1d05d 100644 --- a/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml +++ b/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml @@ -27,14 +27,18 @@ DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" DOCKER_DRIVER: overlay2 before_script: + - echo -e "\e[0Ksection_start:`date +%s`:k3d_up[collapsed=true]\r\e[0K\e[33;1mK3D Cluster Create\e[37m" # Give docker-in-docker time to come alive - i=0; while [ "$i" -lt 12 ]; do docker info &>/dev/null && break; sleep 5; i=$(( i + 1 )) ; done - docker network create ${CI_JOB_ID} --driver=bridge -o "com.docker.network.driver.mtu"="1450" - k3d cluster create ${CI_JOB_ID} --config tests/ci/k3d/config.yaml --network ${CI_JOB_ID} - until kubectl get deployment coredns -n kube-system -o go-template='{{.status.availableReplicas}}' | grep -v -e '<no value>'; do sleep 1s; done - kubectl get all -A + - echo -e "\e[0Ksection_end:`date +%s`:k3d_up\r\e[0K" after_script: + - echo -e "\e[0Ksection_start:`date +%s`:k3d_down[collapsed=true]\r\e[0K\e[33;1mK3D Cluster Delete\e[37m" - kubectl get all -A - kubectl get gitrepository,helmrelease,kustomizations -A - k3d cluster delete ${CI_JOB_ID} - docker network rm ${CI_JOB_ID} + - echo -e "\e[0Ksection_end:`date +%s`:k3d_down\r\e[0K" \ No newline at end of file diff --git a/.gitlab-ci/jobs/networking/aws/.gitlab-ci.yml b/.gitlab-ci/jobs/networking/aws/.gitlab-ci.yml index 4bc54d812672617e99e48c217f2cc703cbea4c92..2f28876a310f82c2e042615f0651a89962877a52 100644 --- a/.gitlab-ci/jobs/networking/aws/.gitlab-ci.yml +++ b/.gitlab-ci/jobs/networking/aws/.gitlab-ci.yml @@ -13,6 +13,7 @@ .network up: extends: .network script: + - echo -e "\e[0Ksection_start:`date +%s`:network_up[collapsed=true]\r\e[0K\e[33;1mNetwork Up\e[37m" - *calc_unique_cidr - echo "Creating network with cidr range ${TF_VAR_vpc_cidr}" # Loop to retry network up terraform apply due to issues locking terraform.state in s3 @@ -29,12 +30,14 @@ echo "Attempt failed to apply will retry in 30 seconds" sleep 30 done + - echo -e "\e[0Ksection_end:`date +%s`:network_up\r\e[0K" .network down: extends: - .network - .terraform destroy workspace script: + - echo -e "\e[0Ksection_start:`date +%s`:network_down[collapsed=true]\r\e[0K\e[33;1mNetwork Down\e[37m" - *calc_unique_cidr - echo "Destroying network" # Loop to retry network terraform destory @@ -51,3 +54,4 @@ echo "Attempt failed to destroy will retry in 30 seconds" sleep 30 done + - echo -e "\e[0Ksection_end:`date +%s`:network_down\r\e[0K" \ No newline at end of file diff --git a/.gitlab-ci/jobs/rke2/.gitlab-ci.yml b/.gitlab-ci/jobs/rke2/.gitlab-ci.yml index 480594baa6915bdb392e106f76414a8299e75097..ad348902b5b7da3c2da14c0f7d91232a113f6e11 100644 --- a/.gitlab-ci/jobs/rke2/.gitlab-ci.yml +++ b/.gitlab-ci/jobs/rke2/.gitlab-ci.yml @@ -6,6 +6,7 @@ .rke2 up: extends: .rke2 tf script: + - echo -e "\e[0Ksection_start:`date +%s`:rke2_up[collapsed=true]\r\e[0K\e[33;1mRKE2 Up\e[37m" # Fetch dependencies - apk add bash aws-cli # Loop to retry rke2 terraform apply @@ -23,6 +24,7 @@ sleep 30 done - mv rke2.yaml ${CI_PROJECT_DIR}/rke2.yaml + - echo -e "\e[0Ksection_end:`date +%s`:rke2_up\r\e[0K" artifacts: paths: - ${CI_PROJECT_DIR}/rke2.yaml @@ -32,6 +34,7 @@ - .rke2 tf - .terraform destroy workspace script: + - echo -e "\e[0Ksection_start:`date +%s`:rke2_down[collapsed=true]\r\e[0K\e[33;1mRKE2 Down\e[37m" # Loop to retry rke2 terraform destory - | set -e @@ -46,3 +49,5 @@ echo "Attempt failed to destroy will retry in 30 seconds" sleep 30 done + - echo -e "\e[0Ksection_end:`date +%s`:rke2_down\r\e[0K" + diff --git a/.gitlab-ci/templates.yml b/.gitlab-ci/templates.yml index 2301faacad62913931546b81a2182a1a22efe563..a8e5046fcf8d1554533587e5258cc38248f7010b 100644 --- a/.gitlab-ci/templates.yml +++ b/.gitlab-ci/templates.yml @@ -22,8 +22,12 @@ include: image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k8s-ci:v1.20.4-bb.3 extends: .bigbang-gitlab-runner-tags after_script: + - echo -e "\e[0Ksection_start:`date +%s`:kubectl_get_all[collapsed=true]\r\e[0K\e[33;1mkubectl get all -A\e[37m" - kubectl get all -A + - echo -e "\e[0Ksection_end:`date +%s`:kubectl_get_all\r\e[0K" + - echo -e "\e[0Ksection_start:`date +%s`:kubectl_get_helmrelease[collapsed=true]\r\e[0K\e[33;1mkubectl get helmrelease -A\e[37m" - kubectl get helmrelease -A + - echo -e "\e[0Ksection_end:`date +%s`:kubectl_get_helmrelease\r\e[0K" .terraformer: image: @@ -36,6 +40,7 @@ include: TF_IN_AUTOMATION: "1" TF_VAR_ci_pipeline_url: $CI_PIPELINE_URL before_script: + - echo -e "\e[0Ksection_start:`date +%s`:terraform_init[collapsed=true]\r\e[0K\e[33;1mTerraform Init\e[37m" - echo "$TF_VAR_env" - cd ${CI_PROJECT_DIR}/${TF_ROOT} - terraform version @@ -54,9 +59,11 @@ include: sleep 5 done - terraform validate + - echo -e "\e[0Ksection_end:`date +%s`:terraform_init\r\e[0K" .terraform destroy workspace: after_script: + - echo -e "\e[0Ksection_start:`date +%s`:terraform_destroy_workspace[collapsed=true]\r\e[0K\e[33;1mTerraform Destroy Workspace\e[37m" - cd ${CI_PROJECT_DIR}/${TF_ROOT} # Loop to retry terraform workspace destory - | @@ -72,3 +79,4 @@ include: echo "Attempt failed to destroy workspace will retry in 30 seconds" sleep 30 done + - echo -e "\e[0Ksection_end:`date +%s`:terraform_destroy_workspace\r\e[0K" \ No newline at end of file