UNCLASSIFIED - NO CUI

Skip to content
Snippets Groups Projects

[k3d-dev.sh] Rebuild K3D cluster without instance re-creation

Merged [k3d-dev.sh] Rebuild K3D cluster without instance re-creation
reinstall-k3d-feature into master
Merged Danny Gershman requested to merge reinstall-k3d-feature into master
Compare
and
1 file
+ 221
− 175
Compare changes
  • Side-by-side
  • Inline
docs/assets/scripts/developer/k3d-dev.sh
+ 221
− 175
@@ -65,7 +65,7 @@ KeyName="${AWSUSERNAME}-dev"
SGname="${AWSUSERNAME}-dev"
# Identify which VPC to create the spot instance in
VPC="${VPC_ID}" # default VPC
RESET_K3D=false
while [ -n "$1" ]; do # while loop starts
@@ -81,7 +81,7 @@ while [ -n "$1" ]; do # while loop starts
-m) echo "-m option passed to install MetalLB"
METAL_LB=true
;;
;;
-d) echo "-d option passed to destroy the AWS resources"
AWSINSTANCEIDs=$( aws ec2 describe-instances \
@@ -124,89 +124,134 @@ while [ -n "$1" ]; do # while loop starts
exit 0
;;
*) echo "Option $1 not recognized" ;; # In case a non-existant option is submitted
*) echo "Option $1 not recognized" ;; # In case a non-existent option is submitted
esac
shift
done
if [[ "$BIG_INSTANCE" == true ]]
then
echo "Will use large m5a.4xlarge spot instance"
InstSize="m5a.4xlarge"
SpotPrice="0.69"
else
echo "Will use standard t3a.2xlarge spot instance"
InstSize="t3a.2xlarge"
SpotPrice="0.35"
echo "Checking for existing cluster for ${AWSUSERNAME}."
InstId=`aws ec2 describe-instances \
--output text \
--query "Reservations[].Instances[].InstanceId" \
--filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" "Name=instance-state-name,Values=running"`
if [[ ! -z "${InstId}" ]]; then
PublicIP=`aws ec2 describe-instances --output text --no-cli-pager --instance-id ${InstId} --query "Reservations[].Instances[].PublicIpAddress"`
echo "Existing cluster found running on instance ${InstId} on ${PublicIP}"
echo "💣 Big Bang Cluster Management 💣"
PS3="Please select an option: "
options=("Reinstall K3D" "Recreate the EC2 instance from scratch" "Quit")
select opt in "${options[@]}"
do
case $REPLY in
1)
read -p "Are you sure you want to re-install K3D on this instance (y/n)? " -r
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
echo
exit 1
fi
RESET_K3D=true
run "k3d cluster delete"
break;;
2)
read -p "Are you sure you want to destroy this instance ${InstId}, and create a new one in its place (y/n)? " -r
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
echo
exit 1
fi
aws ec2 terminate-instances --instance-ids ${InstId} &>/dev/null
echo -n "Instance is being terminated..."
break;;
3)
echo "Bye."
exit 0;;
*)
echo "Option $1 not recognized";;
esac
done
fi
if [[ "${RESET_K3D}" == false ]]; then
if [[ "$BIG_INSTANCE" == true ]]
then
echo "Will use large m5a.4xlarge spot instance"
InstSize="m5a.4xlarge"
SpotPrice="0.69"
else
echo "Will use standard t3a.2xlarge spot instance"
InstSize="t3a.2xlarge"
SpotPrice="0.35"
fi
#### SSH Key Pair
# Create SSH key if it doesn't exist
echo -n Checking if key pair ${KeyName} exists ...
aws ec2 describe-key-pairs --output json --no-cli-pager --key-names ${KeyName} > /dev/null 2>&1 || keypair=missing
if [ "${keypair}" == "missing" ]; then
echo -n -e "missing\nCreating key pair ${KeyName} ... "
aws ec2 create-key-pair --output json --no-cli-pager --key-name ${KeyName} | jq -r '.KeyMaterial' > ~/.ssh/${KeyName}.pem
chmod 600 ~/.ssh/${KeyName}.pem
echo done
else
echo found
fi
#### SSH Key Pair
# Create SSH key if it doesn't exist
echo -n Checking if key pair ${KeyName} exists ...
aws ec2 describe-key-pairs --output json --no-cli-pager --key-names ${KeyName} > /dev/null 2>&1 || keypair=missing
if [ "${keypair}" == "missing" ]; then
echo -n -e "missing\nCreating key pair ${KeyName} ... "
aws ec2 create-key-pair --output json --no-cli-pager --key-name ${KeyName} | jq -r '.KeyMaterial' > ~/.ssh/${KeyName}.pem
chmod 600 ~/.ssh/${KeyName}.pem
echo done
else
echo found
fi
#### Security Group
# Create security group if it doesn't exist
echo -n "Checking if security group ${SGname} exists ..."
aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} > /dev/null 2>&1 || secgrp=missing
if [ "${secgrp}" == "missing" ]; then
echo -e "missing\nCreating security group ${SGname} ... "
aws ec2 create-security-group --output json --no-cli-pager --description "IP based filtering for ${SGname}" --group-name ${SGname} --vpc-id ${VPC}
echo done
else
echo found
fi
# Lookup the security group created to get the ID
echo -n Retrieving ID for security group ${SGname} ...
SecurityGroupId=$(aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} --query "SecurityGroups[0].GroupId" --output text)
echo done
# Add name tag to security group
aws ec2 create-tags --resources ${SecurityGroupId} --tags Key=Name,Value=${SGname} &> /dev/null
# Add rule for IP based filtering
WorkstationIP=`curl http://checkip.amazonaws.com/ 2> /dev/null`
echo -n Checking if ${WorkstationIP} is authorized in security group ...
aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} | grep ${WorkstationIP} > /dev/null || ipauth=missing
if [ "${ipauth}" == "missing" ]; then
echo -e "missing\nAdding ${WorkstationIP} to security group ${SGname} ..."
if [[ "$PRIVATE_IP" == true ]];
then
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol tcp --port 22 --cidr ${WorkstationIP}/32
else # all protocols to all ports is the default
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol all --cidr ${WorkstationIP}/32
fi
#### Security Group
# Create security group if it doesn't exist
echo -n "Checking if security group ${SGname} exists ..."
aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} > /dev/null 2>&1 || secgrp=missing
if [ "${secgrp}" == "missing" ]; then
echo -e "missing\nCreating security group ${SGname} ... "
aws ec2 create-security-group --output json --no-cli-pager --description "IP based filtering for ${SGname}" --group-name ${SGname} --vpc-id ${VPC}
echo done
else
echo found
fi
# Lookup the security group created to get the ID
echo -n Retrieving ID for security group ${SGname} ...
SecurityGroupId=$(aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} --query "SecurityGroups[0].GroupId" --output text)
echo done
else
echo found
fi
# Add name tag to security group
aws ec2 create-tags --resources ${SecurityGroupId} --tags Key=Name,Value=${SGname} &> /dev/null
# Add rule for IP based filtering
WorkstationIP=`curl http://checkip.amazonaws.com/ 2> /dev/null`
echo -n Checking if ${WorkstationIP} is authorized in security group ...
aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} | grep ${WorkstationIP} > /dev/null || ipauth=missing
if [ "${ipauth}" == "missing" ]; then
echo -e "missing\nAdding ${WorkstationIP} to security group ${SGname} ..."
if [[ "$PRIVATE_IP" == true ]];
then
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol tcp --port 22 --cidr ${WorkstationIP}/32
else # all protocols to all ports is the default
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol all --cidr ${WorkstationIP}/32
fi
echo done
else
echo found
fi
##### Launch Specification
# Typical settings for Big Bang development
InstanceType="${InstSize}"
VolumeSize=120
##### Launch Specification
# Typical settings for Big Bang development
InstanceType="${InstSize}"
VolumeSize=120
echo "Using AMI image id ${AMI_ID}"
ImageId="${AMI_ID}"
echo "Using AMI image id ${AMI_ID}"
ImageId="${AMI_ID}"
# Create userdata.txt
mkdir -p ~/aws
cat << EOF > ~/aws/userdata.txt
# Create userdata.txt
mkdir -p ~/aws
cat << EOF > ~/aws/userdata.txt
MIME-Version: 1.0
Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
@@ -215,39 +260,39 @@ Content-Type: text/x-shellscript; charset="us-ascii"
#!/bin/bash
sudo -- bash -c 'sysctl -w vm.max_map_count=524288; \
echo "vm.max_map_count=524288" > /etc/sysctl.d/vm-max_map_count.conf; \
sysctl -w fs.nr_open=13181252; \
echo "fs.nr_open=13181252" > /etc/sysctl.d/fs-nr_open.conf; \
sysctl -w fs.file-max=13181250; \
echo "fs.file-max=13181250" > /etc/sysctl.d/fs-file-max.conf; \
echo "fs.inotify.max_user_instances=1024" > /etc/sysctl.d/fs-inotify-max_user_instances.conf; \
sysctl -w fs.inotify.max_user_instances=1024; \
echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/fs-inotify-max_user_watches.conf; \
sysctl -w fs.inotify.max_user_watches=1048576; \
echo "fs.may_detach_mounts=1" >> /etc/sysctl.d/fs-may_detach_mounts.conf; \
sysctl -w fs.may_detach_mounts=1; \
sysctl -p; \
echo "* soft nofile 13181250" >> /etc/security/limits.d/ulimits.conf; \
echo "* hard nofile 13181250" >> /etc/security/limits.d/ulimits.conf; \
echo "* soft nproc 13181250" >> /etc/security/limits.d/ulimits.conf; \
echo "* hard nproc 13181250" >> /etc/security/limits.d/ulimits.conf; \
modprobe br_netfilter; \
modprobe nf_nat_redirect; \
modprobe xt_REDIRECT; \
modprobe xt_owner; \
modprobe xt_statistic; \
echo "br_netfilter" >> /etc/modules-load.d/istio-iptables.conf; \
echo "nf_nat_redirect" >> /etc/modules-load.d/istio-iptables.conf; \
echo "xt_REDIRECT" >> /etc/modules-load.d/istio-iptables.conf; \
echo "xt_owner" >> /etc/modules-load.d/istio-iptables.conf; \
echo "xt_statistic" >> /etc/modules-load.d/istio-iptables.conf'
echo "vm.max_map_count=524288" > /etc/sysctl.d/vm-max_map_count.conf; \
sysctl -w fs.nr_open=13181252; \
echo "fs.nr_open=13181252" > /etc/sysctl.d/fs-nr_open.conf; \
sysctl -w fs.file-max=13181250; \
echo "fs.file-max=13181250" > /etc/sysctl.d/fs-file-max.conf; \
echo "fs.inotify.max_user_instances=1024" > /etc/sysctl.d/fs-inotify-max_user_instances.conf; \
sysctl -w fs.inotify.max_user_instances=1024; \
echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/fs-inotify-max_user_watches.conf; \
sysctl -w fs.inotify.max_user_watches=1048576; \
echo "fs.may_detach_mounts=1" >> /etc/sysctl.d/fs-may_detach_mounts.conf; \
sysctl -w fs.may_detach_mounts=1; \
sysctl -p; \
echo "* soft nofile 13181250" >> /etc/security/limits.d/ulimits.conf; \
echo "* hard nofile 13181250" >> /etc/security/limits.d/ulimits.conf; \
echo "* soft nproc 13181250" >> /etc/security/limits.d/ulimits.conf; \
echo "* hard nproc 13181250" >> /etc/security/limits.d/ulimits.conf; \
modprobe br_netfilter; \
modprobe nf_nat_redirect; \
modprobe xt_REDIRECT; \
modprobe xt_owner; \
modprobe xt_statistic; \
echo "br_netfilter" >> /etc/modules-load.d/istio-iptables.conf; \
echo "nf_nat_redirect" >> /etc/modules-load.d/istio-iptables.conf; \
echo "xt_REDIRECT" >> /etc/modules-load.d/istio-iptables.conf; \
echo "xt_owner" >> /etc/modules-load.d/istio-iptables.conf; \
echo "xt_statistic" >> /etc/modules-load.d/istio-iptables.conf'
EOF
# Create the device mapping and spot options JSON files
echo "Creating device_mappings.json ..."
mkdir -p ~/aws
# Create the device mapping and spot options JSON files
echo "Creating device_mappings.json ..."
mkdir -p ~/aws
cat << EOF > ~/aws/device_mappings.json
cat << EOF > ~/aws/device_mappings.json
[
{
"DeviceName": "/dev/sda1",
@@ -260,8 +305,8 @@ cat << EOF > ~/aws/device_mappings.json
]
EOF
echo "Creating spot_options.json ..."
cat << EOF > ~/aws/spot_options.json
echo "Creating spot_options.json ..."
cat << EOF > ~/aws/spot_options.json
{
"MarketType": "spot",
"SpotOptions": {
@@ -271,44 +316,81 @@ cat << EOF > ~/aws/spot_options.json
}
EOF
#### Request a Spot Instance
# Location of your private SSH key created during setup
PEM=~/.ssh/${KeyName}.pem
# Run a spot instance with our launch spec for the max. of 6 hours
# NOTE: t3a.2xlarge spot price is 0.35 m5a.4xlarge is 0.69
echo "Running spot instance ..."
InstId=`aws ec2 run-instances \
--output json --no-paginate \
--count 1 --image-id "${ImageId}" \
--instance-type "${InstanceType}" \
--key-name "${KeyName}" \
--security-group-ids "${SecurityGroupId}" \
--instance-initiated-shutdown-behavior "terminate" \
--user-data file://$HOME/aws/userdata.txt \
--block-device-mappings file://$HOME/aws/device_mappings.json \
--instance-market-options file://$HOME/aws/spot_options.json \
| jq -r '.Instances[0].InstanceId'`
# Check if spot instance request was not created
if [ -z ${InstId} ]; then
exit 1;
fi
#### Request a Spot Instance
# Location of your private SSH key created during setup
PEM=~/.ssh/${KeyName}.pem
# Run a spot instance with our launch spec for the max. of 6 hours
# NOTE: t3a.2xlarge spot price is 0.35 m5a.4xlarge is 0.69
echo "Running spot instance ..."
InstId=`aws ec2 run-instances \
--output json --no-paginate \
--count 1 --image-id "${ImageId}" \
--instance-type "${InstanceType}" \
--key-name "${KeyName}" \
--security-group-ids "${SecurityGroupId}" \
--instance-initiated-shutdown-behavior "terminate" \
--user-data file://$HOME/aws/userdata.txt \
--block-device-mappings file://$HOME/aws/device_mappings.json \
--instance-market-options file://$HOME/aws/spot_options.json \
| jq -r '.Instances[0].InstanceId'`
# Check if spot instance request was not created
if [ -z ${InstId} ]; then
exit 1;
fi
# Add name tag to spot instance
aws ec2 create-tags --resources ${InstId} --tags Key=Name,Value=${AWSUSERNAME}-dev &> /dev/null
# Add name tag to spot instance
aws ec2 create-tags --resources ${InstId} --tags Key=Name,Value=${AWSUSERNAME}-dev &> /dev/null
# Request was created, now you need to wait for it to be filled
echo "Waiting for instance ${InstId} to be ready ..."
aws ec2 wait instance-running --output json --no-cli-pager --instance-ids ${InstId} &> /dev/null
# Request was created, now you need to wait for it to be filled
echo "Waiting for instance ${InstId} to be ready ..."
aws ec2 wait instance-running --output json --no-cli-pager --instance-ids ${InstId} &> /dev/null
# allow some extra seconds for the instance to be fully initiallized
echo "Wait a little longer..."
sleep 15
# allow some extra seconds for the instance to be fully initiallized
echo "Wait a little longer..."
sleep 15
# Get the public IP address of our instance
PublicIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PublicIpAddress'`
# Get the public IP address of our instance
PublicIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PublicIpAddress'`
##### Configure Instance
## TODO: replace these individual commands with userdata when the spot instance is created?
echo
echo
echo "starting instance config"
echo "Instance will automatically terminate 8 hours from now unless you alter the root crontab"
run "sudo bash -c 'echo \"\$(date -u -d \"+8 hours\" +\"%M %H\") * * * /usr/sbin/shutdown -h now\" | crontab -'"
echo
echo
echo "updating packages"
run "sudo apt-get -y update"
echo
echo "installing docker"
# install dependencies
run "sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release gnupg-agent software-properties-common"
# Add the Docker repository, we are installing from Docker and not the Ubuntu APT repo.
run 'sudo mkdir -m 0755 -p /etc/apt/keyrings'
run 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg'
run 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null'
run "sudo apt-get update && sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin"
echo
echo
# Add your base user to the Docker group so that you do not need sudo to run docker commands
run "sudo usermod -aG docker ubuntu"
echo
# install kubectl
echo Installing kubectl...
run 'curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"'
run 'sudo mv /home/ubuntu/kubectl /usr/local/bin/'
run 'sudo chmod +x /usr/local/bin/kubectl'
fi
# Get the private IP address of our instance
PrivateIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PrivateIpAddress'`
@@ -330,42 +412,6 @@ until run "hostname"; do
done
echo
##### Configure Instance
## TODO: replace these individual commands with userdata when the spot instance is created?
echo
echo
echo "starting instance config"
echo "Instance will automatically terminate 8 hours from now unless you alter the root crontab"
run "sudo bash -c 'echo \"\$(date -u -d \"+8 hours\" +\"%M %H\") * * * /usr/sbin/shutdown -h now\" | crontab -'"
echo
echo
echo "updating packages"
run "sudo apt-get -y update"
echo
echo "installing docker"
# install dependencies
run "sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release gnupg-agent software-properties-common"
# Add the Docker repository, we are installing from Docker and not the Ubuntu APT repo.
run 'sudo mkdir -m 0755 -p /etc/apt/keyrings'
run 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg'
run 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null'
run "sudo apt-get update && sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin"
echo
echo
# Add your base user to the Docker group so that you do not need sudo to run docker commands
run "sudo usermod -aG docker ubuntu"
echo
# install kubectl
echo Installing kubectl...
run 'curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"'
run 'sudo mv /home/ubuntu/kubectl /usr/local/bin/'
run 'sudo chmod +x /usr/local/bin/kubectl'
echo
echo
# install k3d on instance
@@ -499,7 +545,7 @@ then
echo "The IPs to use come from the istio-system services of type LOADBALANCER EXTERNAL-IP that are created when Istio is deployed."
echo "You must use Firefox browser with with manual SOCKs v5 proxy configuration to localhost with port 12345."
echo "Also ensure 'Proxy DNS when using SOCKS v5' is checked."
echo "Or, with other browsers like Chrome you could use a browser plugin like foxyproxy to do the same thing as Firefox."
echo "Or, with other browsers like Chrome you could use a browser plugin like foxyproxy to do the same thing as Firefox."
else # using MetalLB and public IP
echo "OPTION 1: ACCESS APPLICATIONS WITH WEB BROWSER ONLY"
echo "To access apps from browser only start ssh with application-level port forwarding:"
@@ -515,7 +561,7 @@ then
echo "OPTION 2: ACCESS APPLICATIONS WITH WEB BROWSER AND COMMAND LINE"
echo "To access apps from browser and from the workstation command line start sshuttle in a separate terminal window."
echo " sshuttle --dns -vr ubuntu@${PublicIP} 172.20.1.0/24 --ssh-cmd 'ssh -i ~/.ssh/${KeyName}.pem'"
echo "Edit your workstation /etc/hosts to add the LOADBALANCER EXTERNAL-IPs from the istio-sytem servcies with application hostnames."
echo "Edit your workstation /etc/hosts to add the LOADBALANCER EXTERNAL-IPs from the istio-system servcies with application hostnames."
echo "Here is an example. You might have to change this depending on the number of gateways you configure for k8s cluster."
echo " # METALLB ISTIO INGRESS IPs"
echo " 172.20.1.240 keycloak.bigbang.dev vault.bigbang.dev"
@@ -528,8 +574,8 @@ then # Not using MetalLB and using private IP
echo
echo "To access apps from a browser edit your /etc/hosts to add the private IP of your EC2 instance with application hostnames. Example:"
echo " ${PrivateIP} gitlab.bigbang.dev prometheus.bigbang.dev kibana.bigbang.dev"
echo
else # Not using MetalLB and using pubilc IP. This is the default
echo
else # Not using MetalLB and using public IP. This is the default
echo "To access apps from a browser edit your /etc/hosts to add the public IP of your EC2 instance with application hostnames."
echo "Example:"
echo " ${PublicIP} gitlab.bigbang.dev prometheus.bigbang.dev kibana.bigbang.dev"

UNCLASSIFIED - NO CUI