From d65250fb1046b70ed07074bc20c55ed4b4006b02 Mon Sep 17 00:00:00 2001
From: Josh Wolf <josh@joshwolf.dev>
Date: Tue, 26 Jan 2021 15:28:38 -0700
Subject: [PATCH] ci: refactor ci to use dogfooding cluster
---
.gitlab-ci.yml | 47 ++++++++++++++++++-----
.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml | 26 +++++++++++++
.gitlab-ci/jobs/k3d/.gitlab-ci.yml | 32 ---------------
.gitlab-ci/templates.yml | 4 +-
tests/bash/01_virtualservices.sh | 4 ++
5 files changed, 69 insertions(+), 44 deletions(-)
create mode 100644 .gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml
delete mode 100644 .gitlab-ci/jobs/k3d/.gitlab-ci.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b91a8b2b04..29cdb054ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -33,6 +33,18 @@ variables:
IMAGE_PKG: images.tar.gz
REPOS_PKG: repositories.tar.gz
+.bigbang-dogfood:
+ tags:
+ - bigbang
+ - dogfood
+
+.bigbang:
+ image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/k3d-builder:0.0.1
+ extends: .bigbang-dogfood
+ after_script:
+ - kubectl get all -A
+ - kubectl get helmrelease -A
+
#-----------------------------------------------------------------------------------------------------------------------
# Pre Stage Jobs
#
@@ -40,6 +52,8 @@ variables:
changelog:
image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3
stage: .pre
+ extends:
+ - .bigbang-dogfood
allow_failure: true
variables:
CHANGELOG_FILE: CHANGELOG.md
@@ -50,6 +64,8 @@ changelog:
version:
image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3
stage: .pre
+ extends:
+ - .bigbang-dogfood
allow_failure: true
variables:
CHART_FILE: chart/Chart.yaml
@@ -61,6 +77,8 @@ version:
commits:
image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3
stage: .pre
+ extends:
+ - .bigbang-dogfood
allow_failure: true
script:
- dnf module install -y nodejs
@@ -71,6 +89,8 @@ commits:
pre vars:
image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/pre-envs:ubi8.3
stage: .pre
+ extends:
+ - .bigbang-dogfood
artifacts:
reports:
dotenv: variables.env
@@ -102,12 +122,6 @@ pre vars:
# Smoke Tests
#
-.bigbang:
- image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/k3d-builder:0.0.1
- after_script:
- - kubectl get all -A
- - kubectl get helmrelease -A
-
.deploy_bigbang: &deploy_bigbang
- find ./scripts/deploy -type f -name '*.sh' | sort | xargs -r -I {} sh -c 'echo {} && sh {}'
@@ -117,13 +131,15 @@ pre vars:
clean install:
stage: smoke tests
extends:
- - .k3d
+ - .k3s-ci
variables:
CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}"
script:
- *deploy_bigbang
- *test_bigbang
- - docker exec -i k3d-${CLUSTER_NAME}-server-0 crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)' > images.txt
+
+ # Fetch list of all images ran
+ - curl -sfL k3s:8081/images | jq -r '.images[].repoTags[0]' > images.txt
artifacts:
paths:
- images.txt
@@ -134,7 +150,7 @@ upgrade:
dependencies:
- pre vars
extends:
- - .k3d
+ - .k3s-ci
rules:
# skip job when MR title starts with 'Breaking Change'
- if: '$CI_MERGE_REQUEST_TITLE =~ /^Breaking Change/'
@@ -142,7 +158,7 @@ upgrade:
# run pipeline on merge request events
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
variables:
- CLUSTER_NAME: "clean-${CI_COMMIT_SHORT_SHA}"
+ CLUSTER_NAME: "upgrade-${CI_COMMIT_SHORT_SHA}"
script:
- if $CHART_BREAKING_CHANGE; then echo "Breaking change detected by chart version difference, skipping job"; exit 0; fi
- echo "Install Big Bang from ${CI_DEFAULT_BRANCH}"
@@ -201,6 +217,7 @@ upgrade:
aws/network up:
extends:
+ - .bigbang-dogfood
- .infra fork
- .network up
environment:
@@ -209,6 +226,7 @@ aws/network up:
aws/network down:
extends:
+ - .bigbang-dogfood
- .infra cleanup
- .network down
stage: network down
@@ -226,6 +244,7 @@ aws/network down:
aws/rke2/cluster up:
stage: cluster up
extends:
+ - .bigbang-dogfood
- .infra create
- .rke2 up
needs:
@@ -239,6 +258,7 @@ aws/rke2/cluster up:
aws/rke2/bigbang up:
stage: bigbang up
extends:
+ - .bigbang-dogfood
- .infra create
- .bigbang
needs:
@@ -258,6 +278,7 @@ aws/rke2/bigbang up:
aws/rke2/bigbang test:
stage: test
extends:
+ - .bigbang-dogfood
- .infra create
- .bigbang
needs:
@@ -280,6 +301,7 @@ aws/rke2/bigbang test:
aws/rke2/bigbang down:
stage: bigbang down
extends:
+ - .bigbang-dogfood
- .infra cleanup
- .bigbang
needs:
@@ -300,6 +322,7 @@ aws/rke2/bigbang down:
aws/rke2/cluster down:
stage: cluster down
extends:
+ - .bigbang-dogfood
- .infra cleanup
- .rke2 down
needs:
@@ -318,6 +341,8 @@ aws/rke2/cluster down:
package:
stage: package
image: registry.dsop.io/platform-one/big-bang/umbrella/synker:0.0.1
+ extends:
+ - .bigbang-dogfood
rules:
# run job for manual tag events
- if: $CI_COMMIT_TAG
@@ -351,6 +376,8 @@ package:
release:
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
+ extends:
+ - .bigbang-dogfood
rules:
# run job for manual tag events
- if: $CI_COMMIT_TAG
diff --git a/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml b/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml
new file mode 100644
index 0000000000..f37d75867c
--- /dev/null
+++ b/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml
@@ -0,0 +1,26 @@
+.k8s-util:
+ image: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k8s-ci:v1.20.2
+
+.dind-runner:
+ tags:
+ - bigbang
+ - privileged
+ - dogfood
+
+.k3s-ci:
+ extends:
+ - .k8s-util
+ - .dind-runner
+ services:
+ - name: registry.dso.mil/platform-one/big-bang/pipeline-templates/pipeline-templates/k3s-ci:v1.20.2-k3s1-bb.0
+ alias: k3s
+ command:
+ - "server"
+ - "--tls-san=k3s"
+ - "--disable=metrics-server"
+ - "--disable=traefik"
+ before_script:
+ - mkdir -p ~/.kube
+ - curl -sfL k3s:8081?service=k3s > ~/.kube/config
+ - kubectl version
+ - kubectl cluster-info
diff --git a/.gitlab-ci/jobs/k3d/.gitlab-ci.yml b/.gitlab-ci/jobs/k3d/.gitlab-ci.yml
deleted file mode 100644
index 94546061b6..0000000000
--- a/.gitlab-ci/jobs/k3d/.gitlab-ci.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-.k3d_before_script: &k3d_before_script
- # Starting dnsmasq for cluster dns resolution
- - docker run -d -p 53:53/udp -p 53:53 registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/go-dnsmasq:0eddd476
- - echo "nameserver 127.0.0.1" >> /etc/resolv.conf
- # Standup cluster
- - k3d cluster create ${CLUSTER_NAME} --k3s-server-arg "--disable=traefik" --k3s-server-arg "--disable=metrics-server" -p 80:80@loadbalancer -p 443:443@loadbalancer --wait --agents $N_AGENTS --servers $N_SERVERS
- - while ! (kubectl get node | grep "server" > /dev/null); do sleep 3; done
- - kubectl get nodes
- - k3d node list
-
-.k3d_after_script: &k3d_after_script
- - k3d cluster delete ${CLUSTER_NAME}
-
-.k3d:
- image: registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/k3d-builder:045fb1c2
- services:
- - registry.dsop.io/platform-one/big-bang/pipeline-templates/pipeline-templates/docker:dind
- tags:
- - bigbang
- - privileged
- - public
- variables:
- DOCKER_HOST: tcp://localhost:2375/
- DOCKER_DRIVER: overlay2
- DOCKER_TLS_CERTDIR: ""
- CLUSTER_NAME: ${CI_COMMIT_REF_SLUG}
- N_SERVERS: 1
- N_AGENTS: 0
- before_script:
- - *k3d_before_script
- after_script:
- - *k3d_after_script
\ No newline at end of file
diff --git a/.gitlab-ci/templates.yml b/.gitlab-ci/templates.yml
index 736dd64a58..0904d75f4e 100644
--- a/.gitlab-ci/templates.yml
+++ b/.gitlab-ci/templates.yml
@@ -2,8 +2,8 @@ include:
# "Default" AWS Networking
- local: '/.gitlab-ci/jobs/networking/aws/.gitlab-ci.yml'
- # K3D Gitlab CI job
- - local: '/.gitlab-ci/jobs/k3d/.gitlab-ci.yml'
+ # Clusters in CI
+ - local: '/.gitlab-ci/jobs/ci-cluster/.gitlab-ci.yml'
# RKE2 Gitlab CI jobs
- local: '/.gitlab-ci/jobs/rke2/.gitlab-ci.yml'
diff --git a/tests/bash/01_virtualservices.sh b/tests/bash/01_virtualservices.sh
index 4fff8b9027..47267cdba8 100755
--- a/tests/bash/01_virtualservices.sh
+++ b/tests/bash/01_virtualservices.sh
@@ -3,9 +3,13 @@
# exit on error
set -e
+# Populate /etc/hosts
+ip=$(kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
+
echo "Checking "
hosts=`kubectl get vs -A -o jsonpath="{ .items[*].spec.hosts[*] }"`
for host in $hosts; do
+ echo "$ip $host" >> /etc/hosts
curl -svv https://$host/ > /dev/null
done
\ No newline at end of file
--
GitLab