rke2 retries

.gitlab-ci/jobs/networking/aws/.gitlab-ci.yml

@@ -15,7 +15,20 @@
   script:
     - *calc_unique_cidr
     - echo "Creating network with cidr range ${TF_VAR_vpc_cidr}"
-    - terraform apply -auto-approve
+    # Loop to retry network up terraform apply due to issues locking terraform.state in s3
+    - |
+      set -e
+      attempt_counter=0
+      max_attempts=2
+      until [ $(terraform apply -auto-approve >/dev/null; echo $?) -eq 0 ]; do
+        if [ ${attempt_counter} == ${max_attempts} ];then
+          echo "Error applying network up terraform"
+          exit 1
+        fi
+        attempt_counter=$(($attempt_counter+1))
+        echo "Attempt failed to apply will retry in 30 seconds"
+        sleep 30
+      done
 
 .network down:
   extends:
@@ -24,4 +37,17 @@
   script:
     - *calc_unique_cidr
     - echo "Destroying network"
-    - terraform destroy -auto-approve
+    # Loop to retry network terraform destory
+    - |
+      set -e
+      attempt_counter=0
+      max_attempts=2
+      until [ $(terraform destroy -auto-approve >/dev/null; echo $?) -eq 0 ]; do
+        if [ ${attempt_counter} == ${max_attempts} ];then
+          echo "Error destroying network terraform"
+          exit 1
+        fi
+        attempt_counter=$(($attempt_counter+1))
+        echo "Attempt failed to destroy will retry in 30 seconds"
+        sleep 30
+      done

.gitlab-ci/jobs/rke2/.gitlab-ci.yml

@@ -8,7 +8,20 @@
   script:
     # Fetch dependencies
     - apk add bash aws-cli
-    - terraform apply -input=false -auto-approve
+    # Loop to retry rke2 terraform apply
+    - |
+      set -e
+      attempt_counter=0
+      max_attempts=2
+      until [ $(terraform apply -input=false -auto-approve >/dev/null; echo $?) -eq 0 ]; do
+        if [ ${attempt_counter} == ${max_attempts} ];then
+          echo "Error applying rke2 cluster up terraform"
+          exit 1
+        fi
+        attempt_counter=$(($attempt_counter+1))
+        echo "Attempt failed to apply will retry in 30 seconds"
+        sleep 30
+      done
     - mv rke2.yaml ${CI_PROJECT_DIR}/rke2.yaml
   artifacts:
     paths:
@@ -19,4 +32,17 @@
     - .rke2 tf
     - .terraform destroy workspace
   script:
-    - terraform destroy -input=false -auto-approve
+    # Loop to retry rke2 terraform destory
+    - |
+      set -e
+      attempt_counter=0
+      max_attempts=2
+      until [ $(terraform destroy -input=false -auto-approve >/dev/null; echo $?) -eq 0 ]; do
+        if [ ${attempt_counter} == ${max_attempts} ];then
+          echo "Error destroying rke2 cluster terraform"
+          exit 1
+        fi
+        attempt_counter=$(($attempt_counter+1))
+        echo "Attempt failed to destroy will retry in 30 seconds"
+        sleep 30
+      done

.gitlab-ci/jobs/rke2/dependencies/terraform/main/main.tf

@@ -31,7 +31,7 @@ EOF
 }
 
 module "rke2" {
-  source = "git::https://repo1.dso.mil/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git?ref=v1.1.8"
+  source = "git::https://repo1.dso.mil/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git?ref=v1.1.9"
 
   cluster_name          = local.name
   vpc_id                = var.vpc_id
@@ -63,7 +63,7 @@ EOF
 }
 
 module "generic_agents" {
-  source = "git::https://repo1.dso.mil/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git//modules/agent-nodepool?ref=v1.1.8"
+  source = "git::https://repo1.dso.mil/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git//modules/agent-nodepool?ref=v1.1.9"
 
   name                = "generic-agent"
   vpc_id              = var.vpc_id

.gitlab-ci/templates.yml

@@ -40,10 +40,35 @@ include:
     - cd ${CI_PROJECT_DIR}/${TF_ROOT}
     - terraform version
     - terraform init -input=false
-    - terraform workspace select $TF_VAR_env || terraform workspace new $TF_VAR_env
+    # Loop to retry terraform workspace creation due to issues locking terraform.state in s3
+    - |
+      set -e
+      attempt_counter=0
+      max_attempts=5
+      until [ $(terraform workspace select $TF_VAR_env >/dev/null || terraform workspace new $TF_VAR_env >/dev/null; echo $?) -eq 0 ]; do
+        if [ ${attempt_counter} -eq ${max_attempts} ];then
+          echo "Error creating workspace"
+          exit 1
+        fi
+        attempt_counter=$(($attempt_counter+1))
+        sleep 5
+      done
     - terraform validate
 
 .terraform destroy workspace:
   after_script:
     - cd ${CI_PROJECT_DIR}/${TF_ROOT}
-    - terraform workspace select default && terraform workspace delete "${TF_VAR_env}"
+    # Loop to retry terraform workspace destory
+    - |
+      set -e
+      attempt_counter=0
+      max_attempts=2
+      until [ $(terraform workspace select default >/dev/null && terraform workspace delete "${TF_VAR_env}" >/dev/null; echo $?) -eq 0 ]; do
+        if [ ${attempt_counter} == ${max_attempts} ];then
+          echo "Error destroying terraform workspace"
+          exit 1
+        fi
+        attempt_counter=$(($attempt_counter+1))
+        echo "Attempt failed to destroy workspace will retry in 30 seconds"
+        sleep 30
+      done