Resolve "Add retry for RKE2 cluster creation/destruction"

Summary

Testing adding retry logic to our nightly infra CI pipelines. So if terraform or the runners have an issue, the stage can attempt to retry before marking everything as a failure.

Part of #713 (closed)

Terraform is NOT happy being re-ran if it encounters an issue creating or verifying a resource the first time: yikes, and will require some better terraform logic from someone smarter than I.

This MR includes some per-stage retry logic for all stages of a master nightly pipeline run so that if the runner has an issue, times-out, or fails to schedule it will retry. So still an attempt to make things more resilient!

.gitlab-ci.yml

@@ -202,6 +202,12 @@ aws/network up:
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
     auto_stop_in: 1 hour
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
 
 aws/network down:
   extends:
@@ -212,6 +218,12 @@ aws/network down:
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
     action: stop
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
 
 #-----------------------------------------------------------------------------------------------------------------------
 
@@ -232,6 +244,12 @@ aws/rke2/cluster up:
       artifacts: true
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
 
 # Install BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang up:
@@ -253,6 +271,12 @@ aws/rke2/bigbang up:
     - *deploy_bigbang
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
 
 # Run tests on BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang test:
@@ -276,6 +300,12 @@ aws/rke2/bigbang test:
     - *test_bigbang
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
 
 # Uninstall BigBang on RKE2 cluster on AWS
 aws/rke2/bigbang down:
@@ -311,6 +341,12 @@ aws/rke2/cluster down:
       artifacts: true
   environment:
     name: review/aws-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
 
 #-----------------------------------------------------------------------------------------------------------------------