Merge branch 'feature/load-balancer' into 'main'

Migrate load balancer from classic to new

Closes #10, #11, and #12

See merge request !6

terraform/modules/elb/main.tf

@@ -5,112 +5,125 @@
 # - Security group created for other entities to use for ingress from the ELB
 # - Attaching a pool to the load balancer is done outside of this Terraform
 
-# Security group for load balancer
-resource "aws_security_group" "elb" {
-  name_prefix = "${var.name}-elb-"
-  description = "${var.name} Elastic Load Balancer"
-  vpc_id      = "${var.vpc_id}"
+resource "aws_lb" "public_nlb" {
+  name               = "${var.name}-public-nlb"
+  internal           = false
+  load_balancer_type = "network"
+  subnets            = var.subnet_ids
 
-  # Allow all HTTP traffic
-  ingress {
-    description = "HTTP Traffic"
-    from_port   = 80
-    to_port     = 80
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
+  tags = merge({}, var.tags)
+}
+
+resource "aws_lb_target_group" "public_nlb_http" {
+  name     = "${var.name}-public-nlb-http"
+  port     = var.node_port_http
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
+
+  health_check {
+    port           = var.node_port_health_checks
+    path           = "/healthz/ready"
   }
+  lifecycle {
+    create_before_destroy = true
+  }
+  tags = merge({}, var.tags)
+}
 
-  # Allow all HTTPS traffic
-  ingress {
-    description = "HTTPS Traffic"
-    from_port   = 443
-    to_port     = 443
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
+resource "aws_lb_target_group" "public_nlb_https" {
+  name     = "${var.name}-public-nlb-https"
+  port     = var.node_port_https
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
+
+  health_check {
+    port           = var.node_port_health_checks
+    path           = "/healthz/ready"
   }
+  lifecycle {
+    create_before_destroy = true
+  }
+  tags = merge({}, var.tags)
+}
+
+resource "aws_lb_target_group" "public_nlb_sni" {
+  name     = "${var.name}-public-nlb-sni"
+  port     = var.node_port_sni
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
 
-  # Allow all egress
-  egress {
-    description = "All traffic out"
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
+  health_check {
+    port           = var.node_port_health_checks
+    path           = "/healthz/ready"
   }
+  lifecycle {
+    create_before_destroy = true
+  }
+  tags = merge({}, var.tags)
+}
 
-  tags = var.tags
+resource "aws_lb_listener" "public_nlb_http" {
+  load_balancer_arn = aws_lb.public_nlb.arn
+  port              = "80"
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.public_nlb_http.arn
+  }
+}
+
+resource "aws_lb_listener" "public_nlb_https" {
+  load_balancer_arn = aws_lb.public_nlb.arn
+  port              = "443"
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.public_nlb_https.arn
+  }
+}
+
+resource "aws_lb_listener" "public_nlb_sni" {
+  load_balancer_arn = aws_lb.public_nlb.arn
+  port              = "15443"
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.public_nlb_sni.arn
+  }
+}
+
+# Retrieve the IP addresses of the nlb
+data "aws_network_interface" "public_nlb" {
+  for_each = toset(var.subnet_ids)
+
+  filter {
+    name   = "description"
+    values = ["ELB ${aws_lb.public_nlb.arn_suffix}"]
+  }
+
+  filter {
+    name   = "subnet-id"
+    values = [each.value]
+  }
 }
 
 # Security group for server pool to allow traffic from load balancer
-resource "aws_security_group" "elb_pool" {
-  name_prefix = "${var.name}-elb-pool-"
-  description = "${var.name} Traffic to Elastic Load Balancer server pool"
+resource "aws_security_group" "public_nlb_pool" {
+  name_prefix = "${var.name}-public-nlb-to-pool-"
+  description = "${var.name} Traffic from public Network Load Balancer to server pool"
   vpc_id      = "${var.vpc_id}"
 
   # Allow all traffic from load balancer
   ingress {
-    description       = "Allow Load Balancer Traffic"
+    description       = "Allow public Network Load Balancer traffic"
     from_port         = 0
     to_port           = 0
-    protocol          = "-1"
-    security_groups = [aws_security_group.elb.id]
+    protocol          = -1
+    cidr_blocks       = formatlist("%s/32", [for eni in data.aws_network_interface.public_nlb : eni.private_ip])
   }
 
   tags = var.tags
-}
-
-# Create Elastic Load Balancer
-module "elb" {
-  source          = "terraform-aws-modules/elb/aws"
-  version         = "~> 3.0"
-  name = "${var.name}-elb"
-  subnets         = var.subnet_ids
-  security_groups = [aws_security_group.elb.id]
-  internal        = false
-
-  # Port: Description
-  # 80: HTTP for applications
-  # 443: HTTPS for applications
-  # 15021: Istio Health Checks
-  # 15443: Istio SNI Routing in multi-cluster environment
-  listener = [
-    {
-      instance_port     = var.node_port_http
-      instance_protocol = "TCP"
-      lb_port           = 80
-      lb_protocol       = "tcp"
-    },
-    {
-      instance_port     = var.node_port_https
-      instance_protocol = "TCP"
-      lb_port           = 443
-      lb_protocol       = "tcp"
-    },
-    {
-      instance_port     = var.node_port_health_checks
-      instance_protocol = "TCP"
-      lb_port           = 15021
-      lb_protocol       = "tcp"
-    },
-    {
-      instance_port     = var.node_port_sni
-      instance_protocol = "TCP"
-      lb_port           = 15443
-      lb_protocol       = "tcp"
-    },
-  ]
-
-  health_check = {
-    target              = "TCP:${var.node_port_health_checks}"
-    interval            = 10
-    healthy_threshold   = 2
-    unhealthy_threshold = 6
-    timeout             = 5
-  }
-
-  access_logs = {}
-
-  tags = merge({
-    "kubernetes.io/cluster/${var.name}" = "shared"
-  }, var.tags)
 }
\ No newline at end of file

terraform/modules/elb/outputs.tf

-output "elb_id" {
-  description = "The Elastic Load Balancer (ELB) ID"
-  value = module.elb.elb_id
+output "pool_sg_id" {
+  description = "The ID of the security group used as an inbound rule for load balancer's back-end server pool"
+  value = aws_security_group.public_nlb_pool.id
 }
 
-output "pool_sg_id" {
-  description = "The ID of the security group used as an inbound rule for load balancer's back-end application instances"
-  value = aws_security_group.elb_pool.id
+output "elb_target_group_arns" {
+  description = "The load balancer target group ARNs"
+  value = [aws_lb_target_group.public_nlb_http.arn, aws_lb_target_group.public_nlb_https.arn, aws_lb_target_group.public_nlb_sni.arn]
 }
\ No newline at end of file

terraform/modules/elb/variables.tf

@@ -14,28 +14,27 @@ variable "subnet_ids" {
   type = list(string)
 }
 
+variable "node_port_health_checks" {
+  description = "The node port to use for Istio health check traffic"
+  type = string
+  default = "30000"
+}
 variable "node_port_http" {
   description = "The node port to use for HTTP traffic"
   type = string
-  default = "30080"
+  default = "30001"
 }
 
 variable "node_port_https" {
   description = "The node port to use for HTTPS traffic"
   type = string
-  default = "30443"
-}
-
-variable "node_port_health_checks" {
-  description = "The node port to use for Istio health check traffic"
-  type = string
-  default = "32021"
+  default = "30002"
 }
 
 variable "node_port_sni" {
   description = "The node port to use for Istio SNI traffic"
   type = string
-  default = "32443"
+  default = "30003"
 }
 
 variable "tags" {

terraform/modules/k8s/variables.tf

-variable "kubeconfig_path" {
-  description = "Remote path to kubeconfig"
-  type = string
-}
\ No newline at end of file

terraform/modules/pool/main.tf

 # Connects an Elastic Load Balancer to a pool of servers
+# NOTE: RKE2 already sets the lifecycle of the auto scale group to ignore changes in load balancers and target groups
+# See https://repo1.dso.mil/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform/-/blob/master/modules/nodepool/main.tf#L113
 
 resource "aws_autoscaling_attachment" "pool" {
-  elb                    = var.elb_id
+  for_each = toset(var.elb_target_group_arns)
   autoscaling_group_name = var.pool_asg_id
-}
+  alb_target_group_arn = each.value
+}
\ No newline at end of file

terraform/modules/pool/variables.tf

-variable "elb_id" {
-  description = "The load balancer ID to attach the pool"
-  type = string
+variable "name" {
+  description = "The name to apply to resources"
+  type    = string
+  default = "bigbang-dev"
+}
+
+variable "elb_target_group_arns" {
+  description = "The load balancer's target group ARNs to attach to the autoscale group"
+  type = list(string)
 }
 
 variable "pool_asg_id" {
-  description = "The autoscale group IDs that make up the pool to attach to the load balancer"
+  description = "The pool's autoscale group ID"
   type = string
 }
\ No newline at end of file

terraform/modules/k8s/main.tf → terraform/modules/s3/main.tf

-# After the cluster is setup, this script will retrieve the Kubeconfig
-# file from S3 storage and merge in the local ~/.kube/config
+# After the cluster is setup, these scripts will ...
+# - Retrieve the Kuberntes config file from S3 and merge it with the local ~/.kube/config
+# - Upload the SSH private key to S3
 
 # Retrieves kubeconfig
 resource "null_resource" "kubeconfig" {
@@ -15,9 +16,14 @@ resource "null_resource" "kubeconfig" {
       # Merge new config into existing
       export KUBECONFIGBAK=$KUBECONFIG
       export KUBECONFIG=~/.kube/new:~/.kube/config
+      # Replace default with cluster name
+      sed -ri "s/: default$/: ${var.name}/g" ~/.kube/new
+      # Update user only with more info
+      sed -ri "s/(user|- name): ${var.name}$/\1: clusterUser_${var.name}/g" ~/.kube/new
       # Do not redirect to ~/.kube/config or you may truncate the results
       kubectl config view --flatten > ~/.kube/merged
       mv -f ~/.kube/merged ~/.kube/config
+      chmod 0600 ~/.kube/config
 
       # Cleanup
       rm -f ~/.kube/new
@@ -25,4 +31,13 @@ resource "null_resource" "kubeconfig" {
       unset KUBECONFIGBAK
     EOF
   }
-}
\ No newline at end of file
+}
+
+# Upload SSH private key
+resource "aws_s3_bucket_object" "sshkey" {
+  key                    = "ssh-private-key.pem"
+  # Get bucket name in middle of s3://<bucket name>/rke2.yaml
+  bucket                 = replace(replace(var.kubeconfig_path, "/\\/[^/]*$/", ""), "/^[^/]*\\/\\//", "")
+  source                 = pathexpand("${var.private_key_path}/${var.name}.pem")
+  server_side_encryption = "aws:kms"
+}

terraform/modules/s3/variables.tf

+variable "name" {
+  description = "The name of the SSH key"
+  type    = string
+  default = "bigbang-dev"
+}
+
+variable "kubeconfig_path" {
+  description = "Remote path to kubeconfig"
+  type = string
+}
+
+variable "private_key_path" {
+  description = "Local path to SSH private key"
+  type = string
+  default = "~/.ssh"
+}
\ No newline at end of file

terraform/terragrunt.hcl

@@ -21,7 +21,7 @@ remote_state {
 
   generate = {
     path      = "backend.tf"
-    if_exists = "overwrite_terragrunt"
+    if_exists = "overwrite"
   }
 
   config = {

terraform/us-gov-west-1/prod/agent/terragrunt.hcl

@@ -8,7 +8,7 @@ locals {
 }
 
 terraform {
-  source = "git::https://repo1.dsop.io/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git//modules/agent-nodepool?ref=v1.1.8"
+  source = "git::https://repo1.dsop.io/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git//modules/agent-nodepool?ref=v1.1.9"
 }
 
 include {
@@ -80,5 +80,4 @@ inputs = {
   pre_userdata = local.env.cluster.init_script
 
   tags = merge(local.env.region_tags, local.env.tags, {})
-}
-
+}
\ No newline at end of file

terraform/us-gov-west-1/prod/main/terragrunt.hcl

 # This file performs post-cluster actions, like downloading the kubeconfig
+locals {
+  env = merge(
+    yamldecode(file(find_in_parent_folders("region.yaml"))),
+    yamldecode(file(find_in_parent_folders("env.yaml")))
+  )
+}
 
 terraform {
-  source = "${path_relative_from_include()}//modules/k8s"
+  source = "${path_relative_from_include()}//modules/s3"
 }
 
 include {
@@ -16,5 +22,6 @@ dependency "server" {
 }
 
 inputs = {
+  name = local.env.name
   kubeconfig_path = dependency.server.outputs.kubeconfig_path
 }
\ No newline at end of file

terraform/us-gov-west-1/prod/pool/terragrunt.hcl

@@ -16,7 +16,7 @@ include {
 dependency "elb" {
   config_path = "../elb"
   mock_outputs = {
-    elb_id = "mock_elb_id"
+    elb_target_group_arns = ["mock_elb_id"]
   }
 }
 
@@ -28,7 +28,7 @@ dependency "agent" {
 }
 
 inputs = {
-  elb_id = dependency.elb.outputs.elb_id
+  elb_target_group_arns = dependency.elb.outputs.elb_target_group_arns
   pool_asg_id = dependency.agent.outputs.nodepool_id
   tags = merge(local.env.region_tags, local.env.tags, {})
 }
\ No newline at end of file

terraform/us-gov-west-1/prod/server/terragrunt.hcl

@@ -8,7 +8,7 @@ locals {
 }
 
 terraform {
-  source = "git::https://repo1.dsop.io/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git//?ref=v1.1.8"
+  source = "git::https://repo1.dsop.io/platform-one/distros/rancher-federal/rke2/rke2-aws-terraform.git//?ref=v1.1.9"
 }
 
 include {