diff --git a/README.md b/README.md
index 2088b8429ea2b43172ae50647f8ff96d1ad81acd..ebc27f857282fb8179d7e56a469072eec928d58a 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,8 @@ Big Bang is a declarative, continuous delivery tool for deploying Department of
> If viewing this from Github, note that this is a mirror of a government repo hosted on [Repo1](https://repo1.dso.mil/) by [DoD Platform One](http://p1.dso.mil/). Please direct all code changes, issues and comments to [https://repo1.dso.mil/big-bang/bigbang](https://repo1.dso.mil/big-bang/bigbang)
+[Click here to view the Big Bang Quick Start Guide](docs/guides/deployment-scenarios/quickstart.md)
+
## Usage & Scope
Big Bang's scope is to provide publicly available installation manifests for packages required to adhere to the DoD DevSecOps Reference Architecture and additional useful utilities. Big Bang packages are broken into three categories:
diff --git a/docs/assets/scripts/developer/k3d-dev.sh b/docs/assets/scripts/developer/k3d-dev.sh
index 528be6b511433e35c838074436edc555fd80ac9e..eacea2875873112bb9ccb7321c2c11af314602d1 100755
--- a/docs/assets/scripts/developer/k3d-dev.sh
+++ b/docs/assets/scripts/developer/k3d-dev.sh
@@ -1,27 +1,382 @@
#!/bin/bash
+#### Global variables
+
+### Initialized globals
+
K3D_VERSION="5.7.3"
DEFAULT_K3S_TAG="v1.31.4-k3s1"
+CLOUDPROVIDER="aws"
+SSHUSER=ubuntu
+action=create_instances
+ATTACH_SECONDARY_IP=${ATTACH_SECONDARY_IP:=false}
+BIG_INSTANCE=false
+METAL_LB=false
+PRIVATE_IP=false
+PROJECTTAG=default
+RESET_K3D=false
+USE_WEAVE=false
+TERMINATE_INSTANCE=true
+QUIET=false
+
+### Uninitialized globals
+
+CLOUD_RECREATE_INSTANCE=false
+INIT_SCRIPT=""
+RUN_BATCH_FILE=""
+CURRENT_EPOCH=0
+KUBECONFIG=""
+KUBECTL_CHECKSUM=""
+KUBECTL_VERSION=""
+PrivateIP=""
+PublicIP=""
+SSHKEY=""
+
+### AWS Cloud provider globals
+AMI_ID=${AMI_ID:-""}
+VPC_ID=${VPC_ID:-""}
+AWSUSERNAME=""
+EIP1_ASSOCIATION_ID=""
+EIP1_ID=""
+EIP2_ASSOCIATION_ID=""
+EIP2_ID=""
+EXISTING_VPC=""
+SUBNET_ID=""
+SecondaryIP=""
# get the current script dir
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+function process_arguments {
+ while [ -n "$1" ]; do # while loop starts
+
+ case "$1" in
+
+ -t)
+ echo "-t option passed to use additional tags on instance"
+ shift
+ PROJECTTAG=$1
+ ;;
+
+ -b)
+ echo "-b option passed for big k3d cluster using M5 instance"
+ BIG_INSTANCE=true
+ ;;
+
+ -p)
+ echo "-p option passed to create k3d cluster with private ip"
+ if [[ "${ATTACH_SECONDARY_IP}" = false ]]; then
+ PRIVATE_IP=true
+ else
+ echo "Disabling -p option because -a was specified."
+ fi
+ ;;
+
+ -m)
+ echo "-m option passed to install MetalLB"
+ if [[ "${ATTACH_SECONDARY_IP}" = false ]]; then
+ METAL_LB=true
+ else
+ echo "Disabling -m option because -a was specified."
+ fi
+ ;;
+
+ -a)
+ echo "-a option passed to create secondary public IP (-p and -m flags are skipped if set)"
+ PRIVATE_IP=false
+ METAL_LB=false
+ ATTACH_SECONDARY_IP=true
+ ;;
+
+ -d)
+ echo "-d option passed to destroy the AWS resources"
+ action=destroy_instances
+ ;;
+
+ -i)
+ echo "-i option passed to send init script to k3d instance"
+ shift
+ INIT_SCRIPT=$1
+ ;;
+
+ -H)
+ echo "-H option passed to use existing system public IP"
+ shift
+ PublicIP=$1
+ ;;
+
+ -P)
+ echo "-P option passed to use existing system private IP"
+ shift
+ PrivateIP=$1
+ ;;
+
+ -U)
+ echo "-U option passed to provide username for connecting to existing system"
+ shift
+ SSHUSER=$1
+ ;;
+
+ -k)
+ echo "-k option passed to provide SSH key for connecting to existing system"
+ shift
+ SSHKEY=$1
+ ;;
+
+ -c)
+ echo "-c option passed to specify cloud provider"
+ shift
+ CLOUDPROVIDER=$1
+ ;;
+
+ -T)
+ echo "-T option passed to prevent instance termination"
+ TERMINATE_INSTANCE=false
+ ;;
+ -q)
+ QUIET=true
+ ;;
+
+ -h)
+ echo "Usage:"
+ echo "k3d-dev.sh [options]"
+ echo ""
+ echo " -c CLOUD Use the given CLOUD for cloud infra provisioning [aws]"
+ echo
+ echo "========= The following options ONLY APPLY with [-c aws] =================="
+ echo
+ echo " -b use BIG M5 instance. Default is m5a.4xlarge"
+ echo " -a attach secondary Public IP (overrides -p and -m flags)"
+ echo " -d destroy related AWS resources"
+ echo " -R recreate the EC2 instance (shortcut for -d and running again with same flags)"
+ echo " -r Report on all instances owned by your user"
+ echo " -u Update security group for instances"
+ echo
+ echo "========= These options apply regardless of cloud provider ================"
+ echo
+ echo " -K recreate the k3d cluster on the host"
+ echo " -m create k3d cluster with metalLB load balancer"
+ echo " -p use private IP for security group and k3d cluster"
+ echo " -t Set the project tag on the instance (for managing multiple instances)"
+ echo " -w install the weave CNI instead of the default flannel CNI"
+ echo " -i /path/to/script initialization script to pass to instance before configuring it"
+ echo " -U username username to use when connecting to existing system in -P"
+ echo " -T Don't terminate the instance after 8 hours"
+ echo " -q suppress the final completion message"
+ echo
+ echo "========= These options override -c and use your own infrastructure ======="
+ echo
+ echo " -H xxx.xxx.xxx.xxx Public IP address of existing system to configure"
+ echo " -P xxx.xxx.xxx.xxx private IP address of existing system to configure (if not provided and -H is set, the value of -H is assumed)"
+ echo " -k /path/to/key/file SSH key to use when connecting to cluster instance"
+ echo
+ echo " -h output help"
+ exit 0
+ ;;
+ -K)
+ RESET_K3D=true
+ ;;
+ -R)
+ CLOUD_RECREATE_INSTANCE=true
+ ;;
+ -u)
+ action=update_instances
+ ;;
+
+ -r)
+ action=report_instances
+ ;;
+
+ -w)
+ echo "-w option passed to use Weave CNI"
+ USE_WEAVE=true
+ ;;
+
+ *) echo "Option $1 not recognized" ;; # In case a non-existent option is submitted
+
+ esac
+ shift
+ done
+
+ # Command argument post-processing
+ if [[ "$PRIVATE_IP" == "true" ]] && [[ "$PublicIP" != "" ]] && [[ "$PrivateIP" == "" ]]; then
+ echo "When providing -p and -H you MUST provide -P (private IP address to use)." >&2
+ exit 1
+ fi
+
+ if [[ "${PublicIP}" != "" ]] && [[ "$PrivateIP" == "" ]]; then
+ PrivateIP=$PublicIP
+ fi
+
+}
+
+function check_missing_tools {
+ # check for tools
+ tooldependencies=(jq sed ssh ssh-keygen scp kubectl tr base64 $@)
+ for tooldependency in "${tooldependencies[@]}"; do
+ command -v $tooldependency >/dev/null 2>&1 || {
+ echo >&2 " $tooldependency is not installed."
+ missingtool=1
+ }
+ done
+ sed_gsed="sed"
+ # verify sed version if mac
+ # alias prohibited, symlinks permitted
+ uname="$(uname -s)"
+ if [[ "${uname}" == "Darwin" ]]; then
+ if [[ $(command -v gsed) ]]; then
+ sed_gsed="gsed"
+ else
+ missingtool=1
+ echo ' gnu-sed is not installed. "brew install gnu-sed"'
+ fi
+ fi
+ # if tool missing, exit
+ if [[ "${missingtool}" == 1 ]]; then
+ echo " Please install required tools. Aborting."
+ exit 1
+ fi
+}
+
+function cloud_aws_toolnames {
+ echo aws
+}
+
+function cloud_aws_configure {
+ # getting AWS user name
+ AWSUSERNAME=$(aws sts get-caller-identity --query Arn --output text | cut -f 2 -d '/')
+
+ SGname="${AWSUSERNAME}-dev-${PROJECTTAG}"
+ KeyName="${AWSUSERNAME}-dev-${PROJECTTAG}"
+ SSHKEY=~/.ssh/${KeyName}.pem
+
+ # check for aws username environment variable. If not found then terminate script
+ if [[ -z "${AWSUSERNAME}" ]]; then
+ echo "You must configure your AWS credentials. Your AWS user name is used to name resources in AWS. Example:"
+ echo " aws configure"
+ exit 1
+ else
+ echo "AWS User Name: ${AWSUSERNAME}"
+ fi
+
+ if [[ -z "${VPC_ID}" ]]; then
+ VPC_ID="$(aws ec2 describe-vpcs --filters Name=is-default,Values=true | jq -j .Vpcs[0].VpcId)"
+ if [[ -z "${VPC_ID}" ]]; then
+ echo "AWS account has no default VPC - please provide VPC_ID"
+ exit 1
+ fi
+ fi
+
+ if [[ -n "${SUBNET_ID}" ]]; then
+ if [[ "$(aws ec2 describe-subnets --subnet-id "${SUBNET_ID}" --filters "Name=vpc-id,Values=${VPC_ID}" | jq -j .Subnets[0])" == "null" ]]; then
+ echo "SUBNET_ID ${SUBNET_ID} does not belong to VPC ${VPC_ID}"
+ exit 1
+ fi
+ else
+ SUBNET_ID="$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=${VPC_ID}" "Name=default-for-az,Values=true" | jq -j .Subnets[0].SubnetId)"
+ if [[ "${SUBNET_ID}" == "null" ]]; then
+ echo "VPC ${VPC_ID} has no default subnets - please provide SUBNET_ID"
+ exit 1
+ fi
+ fi
+
+ # If the user is using her own AMI, then respect that and do not update it.
+ APT_UPDATE="true"
+ if [[ $AMI_ID ]]; then
+ APT_UPDATE="false"
+ else
+ # default
+ AMI_ID=$(getDefaultAmi)
+ fi
+
+ #### Preflight Checks
+ # Check that the VPC is available
+ EXISTING_VPC=$(aws ec2 describe-vpcs | grep ${VPC_ID})
+ if [[ -z "${EXISTING_VPC}" ]]; then
+ echo "VPC is not available in the current AWS_PROFILE - Update VPC_ID"
+ exit 1
+ fi
+}
+
+function report_instances {
+ if [[ "${CLOUDPROVIDER}" == "" ]]; then
+ echo "Cannot report instances, no cloud provider set"
+ exit 1
+ fi
+ cloud_${CLOUDPROVIDER}_report_instances
+}
+
+function cloud_aws_report_instances {
+ aws ec2 describe-instances \
+ --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" \
+ --query 'Reservations[*].Instances[*].[InstanceId,State.Name,PublicIpAddress,SecurityGroups[0].GroupId,Tags[?Key==`Project`].Value | [0]]' \
+ --output text
+}
+
+function set_kubeconfig {
+ if [[ "$PublicIP" != "" ]]; then
+ KUBECONFIG=${PublicIP}-dev-${PROJECTTAG}-config
+ elif [[ "${AWSUSERNAME}" != "" ]]; then
+ KUBECONFIG=${AWSUSERNAME}-dev-${PROJECTTAG}-config
+ fi
+}
+
+function run_batch_new() {
+ if [[ "$RUN_BATCH_FILE" != "" ]]; then
+ echo "Can't manage more than one run batch at once" >&2
+ exit 1
+ fi
+ RUN_BATCH_FILE=$(mktemp k3d_dev_run_batchXXX)
+ echo '#!/bin/bash' >> ${RUN_BATCH_FILE}
+ echo 'set -xue' >> ${RUN_BATCH_FILE}
+}
+
+function run_batch_add() {
+ if [[ "$RUN_BATCH_FILE" == "" ]]; then
+ echo "No run batch configured" >&2
+ exit 1
+ fi
+ echo "$@" >> ${RUN_BATCH_FILE}
+}
+
+function run_batch_execute() {
+ if [[ "$RUN_BATCH_FILE" == "" ]]; then
+ echo "No run batch configured" >&2
+ exit 1
+ fi
+ batch_basename=$(basename ${RUN_BATCH_FILE})
+ cat ${RUN_BATCH_FILE} | run "sudo cat > /tmp/${batch_basename}"
+ $(k3dsshcmd) -t "bash /tmp/${batch_basename}"
+ if [[ $? -ne 0 ]]; then
+ echo "Batch file /tmp/${batch_basename} failed on target system." >&2
+ echo "You can debug it by logging into the system:" >&2
+ echo
+ k3dsshcmd >&2
+ exit 1
+ fi
+ rm -f ${RUN_BATCH_FILE}
+ RUN_BATCH_FILE=""
+}
+
+function k3dsshcmd() {
+ echo "ssh -i ${SSHKEY} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${SSHUSER}@${PublicIP}"
+}
function run() {
- ssh -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ubuntu@${PublicIP} "$@"
+ $(k3dsshcmd) "$@"
}
function runwithexitcode() {
- ssh -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ubuntu@${PublicIP} "$@"
- exitcode=$?
- [ $exitcode -eq 0 ]
+ $(k3dsshcmd) "$@"
+ return $?
}
function runwithreturn() {
- echo $(ssh -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ubuntu@${PublicIP} "$@")
+ $(k3sshcmd) "$@"
}
function getPrivateIP2() {
- echo `aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].NetworkInterfaces[0].PrivateIpAddresses[] | select(.Primary==false) | .PrivateIpAddress'`
+ echo $(aws ec2 describe-instances --output json --no-cli-pager --instance-ids "${InstId}" | jq -r '.Reservations[0].Instances[0].NetworkInterfaces[0].PrivateIpAddresses[] | select(.Primary==false) | .PrivateIpAddress')
}
function getDefaultAmi() {
@@ -32,12 +387,12 @@ function getDefaultAmi() {
partition=$(aws sts get-caller-identity --query 'Arn' --output text | awk -F ":" '{print $2}')
# Select the correct AWS Account ID for Ubuntu Server AMI based on the partition
if [[ "${partition}" == "aws-us-gov" ]]; then
- ubuntu_account_id="513442679011"
+ ubuntu_account_id="513442679011"
elif [[ "${partition}" == "aws" ]]; then
- ubuntu_account_id="099720109477"
+ ubuntu_account_id="099720109477"
else
- echo "Unrecognized AWS partition"
- exit 1
+ echo "Unrecognized AWS partition"
+ exit 1
fi
# Filter on latest 22.04 jammy server
image_id=$(aws ec2 describe-images \
@@ -47,14 +402,13 @@ function getDefaultAmi() {
--output text)
if [ $? -ne 0 ] || [ "${image_id}" == "None" ]; then
- echo "Error getting AMI ID"
- exit 1
+ echo "Error getting AMI ID"
+ exit 1
fi
echo "${image_id}"
}
-function update_ec2_security_group
-{
+function update_ec2_security_group {
# Look up the security group created to get the ID
echo -n Retrieving ID for security group ${SGname} ...
#### SecurityGroupId=$(aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} --query "SecurityGroups[0].GroupId" --output text)
@@ -62,23 +416,22 @@ function update_ec2_security_group
echo done
# Add name tag to security group
- aws ec2 create-tags --resources ${SecurityGroupId} --tags Key=Name,Value=${SGname} &> /dev/null
- aws ec2 create-tags --resources ${SecurityGroupId} --tags Key=Project,Value=${PROJECTTAG} &> /dev/null
-
+ aws ec2 create-tags --resources ${SecurityGroupId} --tags Key=Name,Value=${SGname} &>/dev/null
+ aws ec2 create-tags --resources ${SecurityGroupId} --tags Key=Project,Value=${PROJECTTAG} &>/dev/null
+
# Add rule for IP based filtering
- WorkstationIP=`curl http://checkip.amazonaws.com/ 2> /dev/null`
+ WorkstationIP=$(curl http://checkip.amazonaws.com/ 2>/dev/null)
echo -n Checking if ${WorkstationIP} is authorized in security group ...
#### aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} | grep ${WorkstationIP} > /dev/null || ipauth=missing
- aws ec2 describe-security-groups --filter Name=vpc-id,Values=$VPC_ID Name=group-name,Values=$SGname | grep ${WorkstationIP} > /dev/null || ipauth=missing
+ aws ec2 describe-security-groups --filter Name=vpc-id,Values=$VPC_ID Name=group-name,Values=$SGname | grep ${WorkstationIP} >/dev/null || ipauth=missing
if [ "${ipauth}" == "missing" ]; then
echo -e "missing\nAdding ${WorkstationIP} to security group ${SGname} ..."
- if [[ "$PRIVATE_IP" == true ]];
- then
+ if [[ "$PRIVATE_IP" == true ]]; then
#### aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol tcp --port 22 --cidr ${WorkstationIP}/32
#### aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol tcp --port 6443 --cidr ${WorkstationIP}/32
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-id ${SecurityGroupId} --protocol tcp --port 22 --cidr ${WorkstationIP}/32
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-id ${SecurityGroupId} --protocol tcp --port 6443 --cidr ${WorkstationIP}/32
- else # all protocols to all ports is the default
+ else # all protocols to all ports is the default
#### aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-name ${SGname} --protocol all --cidr ${WorkstationIP}/32
aws ec2 authorize-security-group-ingress --output json --no-cli-pager --group-id ${SecurityGroupId} --protocol all --cidr ${WorkstationIP}/32
fi
@@ -88,375 +441,73 @@ function update_ec2_security_group
fi
}
-function destroy_instances
-{
- AWSINSTANCEIDs=$( aws ec2 describe-instances \
+function destroy_instances {
+ if [[ "$PublicIP" != "" ]] && [[ "${CLOUD_RECREATE_INSTANCE}" != "true" ]]; then
+ echo "Public IP of instance has been provided; assuming instance was not provisioned by me."
+ echo "Nothing to do."
+ exit 1
+ fi
+
+ AWSINSTANCEIDs=$(aws ec2 describe-instances \
--output text \
--query "Reservations[].Instances[].InstanceId" \
- --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" "Name=tag:Project,Values=${PROJECTTAG}" "Name=instance-state-name,Values=running" )
- # If instance exists then terminate it
- if [[ $AWSINSTANCEIDs ]]; then
+ --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" "Name=tag:Project,Values=${PROJECTTAG}" "Name=instance-state-name,Values=running")
+ # If instance exists then terminate it
+ if [[ $AWSINSTANCEIDs ]]; then
echo "aws instances being terminated: ${AWSINSTANCEIDs}"
-
- read -p "Are you sure you want to delete these instances (y/n)? " -r
- if [[ ! $REPLY =~ ^[Yy]$ ]]
- then
- echo
- exit 1
+
+ # Don't prompt the user if they already told us to do it
+ if [[ "${CLOUD_RECREATE_INSTANCE}" != "true" ]]; then
+ read -p "Are you sure you want to delete these instances (y/n)? " -r
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ echo
+ exit 1
+ fi
fi
aws ec2 terminate-instances --instance-ids ${AWSINSTANCEIDs} &>/dev/null
echo -n "Waiting for instance termination..."
- aws ec2 wait instance-terminated --instance-ids ${AWSINSTANCEIDs} &> /dev/null
+ aws ec2 wait instance-terminated --instance-ids ${AWSINSTANCEIDs} &>/dev/null
echo "done"
else
echo "You had no running instances."
fi
echo "SecurityGroup name to be deleted: ${SGname}"
- aws ec2 delete-security-group --group-name=${SGname} &> /dev/null
+ aws ec2 delete-security-group --group-name=${SGname} &>/dev/null
echo "KeyPair to be deleted: ${KeyName}"
- aws ec2 delete-key-pair --key-name ${KeyName}-${PROJECT} &> /dev/null
- ALLOCATIONIDs=(`aws ec2 describe-addresses --output text --filters "Name=tag:Owner,Values=${AWSUSERNAME}" "Name=tag:Project,Values=${PROJECTTAG}" --query "Addresses[].AllocationId"`)
- for i in "${ALLOCATIONIDs[@]}"
- do
- echo -n "Releasing Elastic IP $i ..."
- aws ec2 release-address --allocation-id $i
- echo "done"
+ aws ec2 delete-key-pair --key-name ${KeyName}-${PROJECT} &>/dev/null
+ ALLOCATIONIDs=($(aws ec2 describe-addresses --output text --filters "Name=tag:Owner,Values=${AWSUSERNAME}" "Name=tag:Project,Values=${PROJECTTAG}" --query "Addresses[].AllocationId"))
+ for i in "${ALLOCATIONIDs[@]}"; do
+ echo -n "Releasing Elastic IP $i ..."
+ aws ec2 release-address --allocation-id $i
+ echo "done"
done
- exit 0
}
-function update_instances
-{
+function update_instances {
update_ec2_security_group
}
+function install_docker {
+ echo "installing docker"
+ # install dependencies
+ run_batch_new
+ run_batch_add "sudo DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release gnupg-agent software-properties-common"
+ # Add the Docker repository, we are installing from Docker and not the Ubuntu APT repo.
+ run_batch_add 'sudo mkdir -m 0755 -p /etc/apt/keyrings'
+ # gpg won't overwrite the file if we're rebuilding the cluster, we have to clear it
+ run_batch_add 'sudo rm -f /etc/apt/keyrings/docker.gpg'
+ run_batch_add 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --dearmor -o /etc/apt/keyrings/docker.gpg'
+ run_batch_add 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null'
+ run_batch_add "sudo DEBIAN_FRONTEND=noninteractive apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin"
+ # Add your base user to the Docker group so that you do not need sudo to run docker commands
+ run_batch_add "sudo usermod -aG docker ubuntu"
+ run_batch_execute
+}
-function create_instances
-{
- echo "Checking for existing cluster for ${AWSUSERNAME}."
- InstId=`aws ec2 describe-instances \
- --output text \
- --query "Reservations[].Instances[].InstanceId" \
- --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" "Name=tag:Project,Values=${PROJECTTAG}" "Name=instance-state-name,Values=running"`
- if [[ $InstId ]]; then
- PublicIP=`aws ec2 describe-instances --output text --no-cli-pager --instance-id ${InstId} --query "Reservations[].Instances[].PublicIpAddress"`
- PrivateIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PrivateIpAddress'`
- echo "Existing cluster found running on instance ${InstId} on ${PublicIP} / ${PrivateIP}"
- echo "💣 Big Bang Cluster Management 💣"
- PS3="Please select an option: "
- options=("Re-create K3D cluster" "Recreate the EC2 instance from scratch" "Quit")
-
- select opt in "${options[@]}"
- do
- case $REPLY in
- 1)
- read -p "Are you sure you want to re-create a K3D cluster on this instance (y/n)? " -r
- if [[ ! $REPLY =~ ^[Yy]$ ]]
- then
- echo
- exit 1
- fi
- RESET_K3D=true
- SecondaryIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].NetworkInterfaces[0].PrivateIpAddresses[] | select(.Primary==false) | .Association.PublicIp'`
- PrivateIP2=$(getPrivateIP2)
- if [[ "${ATTACH_SECONDARY_IP}" == true && -z "${SecondaryIP}" ]]; then
- echo "Secondary IP didn't exist at the time of creation of the instance, so cannot attach one without re-creating it with the -a flag selected."
- exit 1
- fi
- run "k3d cluster delete"
- run "docker ps -aq | xargs docker stop | xargs docker rm"
- break;;
- 2)
- read -p "Are you sure you want to destroy this instance ${InstId}, and create a new one in its place (y/n)? " -r
- if [[ ! $REPLY =~ ^[Yy]$ ]]
- then
- echo
- exit 1
- fi
-
- aws ec2 terminate-instances --instance-ids ${InstId} &>/dev/null
- echo -n "Instance is being terminated..."
- if [[ "${ATTACH_SECONDARY_IP}" == true ]]; then
- echo -n "Waiting for instance termination..."
- aws ec2 wait instance-terminated --instance-ids ${InstId} &> /dev/null
- echo "done"
- fi
- break;;
- 3)
- echo "Bye."
- exit 0;;
- *)
- echo "Option $1 not recognized";;
- esac
- done
- fi
-
- if [[ "${RESET_K3D}" == false ]]; then
- if [[ "$BIG_INSTANCE" == true ]]
- then
- echo "Will use large m5a.4xlarge spot instance"
- InstSize="m5a.4xlarge"
- SpotPrice="0.69"
- else
- echo "Will use standard t3a.2xlarge spot instance"
- InstSize="t3a.2xlarge"
- SpotPrice="0.35"
- fi
-
-
- #### Cleaning up unused Elastic IPs
- ALLOCATIONIDs=(`aws ec2 describe-addresses --filters "Name=tag:Owner,Values=${AWSUSERNAME}" "Name=tag:Project,Values=${PROJECTTAG}" --query "Addresses[?AssociationId==null]" | jq -r '.[].AllocationId'`)
- for i in "${ALLOCATIONIDs[@]}"
- do
- echo -n "Releasing Elastic IP $i ..."
- aws ec2 release-address --allocation-id $i
- echo "done"
- done
-
- #### SSH Key Pair
- # Create SSH key if it doesn't exist
- echo -n Checking if key pair ${KeyName} exists ...
- aws ec2 describe-key-pairs --output json --no-cli-pager --key-names ${KeyName} > /dev/null 2>&1 || keypair=missing
- if [ "${keypair}" == "missing" ]; then
- echo -n -e "missing\nCreating key pair ${KeyName} ... "
- aws ec2 create-key-pair --output json --no-cli-pager --key-name ${KeyName} | jq -r '.KeyMaterial' > ~/.ssh/${KeyName}.pem
- chmod 600 ~/.ssh/${KeyName}.pem
- echo done
- else
- echo found
- fi
-
-
- #### Security Group
- # Create security group if it doesn't exist
- echo -n "Checking if security group ${SGname} exists ..."
- aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} > /dev/null 2>&1 || secgrp=missing
- if [ "${secgrp}" == "missing" ]; then
- echo -e "missing\nCreating security group ${SGname} ... "
- aws ec2 create-security-group --output json --no-cli-pager --description "IP based filtering for ${SGname}" --group-name ${SGname} --vpc-id ${VPC}
- echo done
- else
- echo found
- fi
-
- update_ec2_security_group
-
- ##### Launch Specification
- # Typical settings for Big Bang development
- InstanceType="${InstSize}"
- VolumeSize=120
-
- echo "Using AMI image id ${AMI_ID}"
- ImageId="${AMI_ID}"
-
- # Create userdata.txt
- tmpdir=$(mktemp -d)
- cat << EOF > "$tmpdir/userdata.txt"
-#!/usr/bin/env bash
-
-echo "* soft nofile 13181250" >> /etc/security/limits.d/ulimits.conf
-echo "* hard nofile 13181250" >> /etc/security/limits.d/ulimits.conf
-echo "* soft nproc 13181250" >> /etc/security/limits.d/ulimits.conf
-echo "* hard nproc 13181250" >> /etc/security/limits.d/ulimits.conf
-
-echo "vm.max_map_count=524288" > /etc/sysctl.d/vm-max_map_count.conf
-echo "fs.nr_open=13181252" > /etc/sysctl.d/fs-nr_open.conf
-echo "fs.file-max=13181250" > /etc/sysctl.d/fs-file-max.conf
-echo "fs.inotify.max_user_instances=1024" > /etc/sysctl.d/fs-inotify-max_user_instances.conf
-echo "fs.inotify.max_user_watches=1048576" > /etc/sysctl.d/fs-inotify-max_user_watches.conf
-echo "fs.may_detach_mounts=1" >> /etc/sysctl.d/fs-may_detach_mounts.conf
-
-sysctl --system
-
-echo "br_netfilter" >> /etc/modules-load.d/istio-iptables.conf
-echo "nf_nat_redirect" >> /etc/modules-load.d/istio-iptables.conf
-echo "xt_REDIRECT" >> /etc/modules-load.d/istio-iptables.conf
-echo "xt_owner" >> /etc/modules-load.d/istio-iptables.conf
-echo "xt_statistic" >> /etc/modules-load.d/istio-iptables.conf
-
-systemctl restart systemd-modules-load.service
-EOF
-
- # Create the device mapping and spot options JSON files
- echo "Creating device_mappings.json ..."
-
- # gp3 volumes are 20% cheaper than gp2 and comes with 3000 Iops baseline and 125 MiB/s baseline throughput for free.
- cat << EOF > "$tmpdir/device_mappings.json"
-[
- {
- "DeviceName": "/dev/sda1",
- "Ebs": {
- "DeleteOnTermination": true,
- "VolumeType": "gp3",
- "VolumeSize": ${VolumeSize},
- "Encrypted": true
- }
- }
-]
-EOF
-
- echo "Creating spot_options.json ..."
- cat << EOF > "$tmpdir/spot_options.json"
- {
- "MarketType": "spot",
- "SpotOptions": {
- "MaxPrice": "${SpotPrice}",
- "SpotInstanceType": "one-time"
- }
- }
-EOF
-
- #### Request a Spot Instance
- # Location of your private SSH key created during setup
- PEM=~/.ssh/${KeyName}.pem
-
- # Run a spot instance with our launch spec for the max. of 6 hours
- # NOTE: t3a.2xlarge spot price is 0.35 m5a.4xlarge is 0.69
- echo "Running spot instance ..."
-
- if [[ "${ATTACH_SECONDARY_IP}" == true ]]; then
- # If we are using a secondary IP, we don't want to assign public IPs at launch time. Instead, the script will attach both public IPs after the instance is launched.
- additional_create_instance_options="--no-associate-public-ip-address --secondary-private-ip-address-count 1"
- else
- additional_create_instance_options="--associate-public-ip-address"
- fi
-
- InstId=`aws ec2 run-instances \
- --output json --no-paginate \
- --count 1 --image-id "${ImageId}" \
- --instance-type "${InstanceType}" \
- --subnet-id "${SUBNET_ID}" \
- --key-name "${KeyName}" \
- --security-group-ids "${SecurityGroupId}" \
- --instance-initiated-shutdown-behavior "terminate" \
- --user-data "file://$tmpdir/userdata.txt" \
- --block-device-mappings "file://$tmpdir/device_mappings.json" \
- --instance-market-options "file://$tmpdir/spot_options.json" \
- ${additional_create_instance_options} \
- | jq -r '.Instances[0].InstanceId'`
-
- # Check if spot instance request was not created
- if [ -z ${InstId} ]; then
- exit 1;
- fi
-
- # Add name tag to spot instance
- aws ec2 create-tags --resources ${InstId} --tags Key=Name,Value=${AWSUSERNAME}-dev &> /dev/null
- aws ec2 create-tags --resources ${InstId} --tags Key=Project,Value=${PROJECTTAG} &> /dev/null
-
- # Request was created, now you need to wait for it to be filled
- echo "Waiting for instance ${InstId} to be ready ..."
- aws ec2 wait instance-running --output json --no-cli-pager --instance-ids ${InstId} &> /dev/null
-
- # allow some extra seconds for the instance to be fully initialized
- echo "Almost there, 15 seconds to go..."
- sleep 15
-
- ## IP Address Allocation and Attachment
- CURRENT_EPOCH=`date +'%s'`
-
- # Get the private IP address of our instance
- PrivateIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PrivateIpAddress'`
-
- # Use Elastic IPs if a Secondary IP is required, instead of the auto assigned one.
- if [[ "${ATTACH_SECONDARY_IP}" == false ]]; then
- PublicIP=`aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PublicIpAddress'`
- else
- echo "Checking to see if an Elastic IP is already allocated and not attached..."
- PublicIP=`aws ec2 describe-addresses --filters "Name=tag:Name,Values=${AWSUSERNAME}-EIP1" "Name=tag:Project,Values=${PROJECTTAG}" --query 'Addresses[?AssociationId==null]' | jq -r '.[0].PublicIp // ""'`
- if [[ -z "${PublicIP}" ]]; then
- echo "Allocating a new/another primary elastic IP..."
- PublicIP=`aws ec2 allocate-address --output json --no-cli-pager --tag-specifications="ResourceType=elastic-ip,Tags=[{Key=Name,Value=${AWSUSERNAME}-EIP1},{Key=Owner,Value=${AWSUSERNAME}}]" | jq -r '.PublicIp'`
- else
- echo "Previously allocated primary Elastic IP ${PublicIP} found."
- fi
-
- echo -n "Associating IP ${PublicIP} address to instance ${InstId} ..."
- EIP1_ASSOCIATION_ID=`aws ec2 associate-address --output json --no-cli-pager --instance-id ${InstId} --private-ip ${PrivateIP} --public-ip $PublicIP | jq -r '.AssociationId'`
- echo "${EIP1_ASSOCIATION_ID}"
- EIP1_ID=`aws ec2 describe-addresses --public-ips ${PublicIP} | jq -r '.Addresses[].AllocationId'`
- aws ec2 create-tags --resources ${EIP1_ID} --tags Key="lastused",Value="${CURRENT_EPOCH}"
- aws ec2 create-tags --resources ${EIP1_ID} --tags Key="Project",Value="${PROJECTTAG}"
-
- PrivateIP2=$(getPrivateIP2)
- echo "Checking to see if a Secondary Elastic IP is already allocated and not attached..."
- SecondaryIP=`aws ec2 describe-addresses --filters "Name=tag:Name,Values=${AWSUSERNAME}-EIP2" "Name=tag:Project,Values=${PROJECTTAG}" --query 'Addresses[?AssociationId==null]' | jq -r '.[0].PublicIp // ""'`
- if [[ -z "${SecondaryIP}" ]]; then
- echo "Allocating a new/another secondary elastic IP..."
- SecondaryIP=`aws ec2 allocate-address --output json --no-cli-pager --tag-specifications="ResourceType=elastic-ip,Tags=[{Key=Name,Value=${AWSUSERNAME}-EIP2},{Key=Owner,Value=${AWSUSERNAME}}]" | jq -r '.PublicIp'`
- else
- echo "Previously allocated secondary Elastic IP ${SecondaryIP} found."
- fi
- echo -n "Associating Secondary IP ${SecondaryIP} address to instance ${InstId}..."
- EIP2_ASSOCIATION_ID=`aws ec2 associate-address --output json --no-cli-pager --instance-id ${InstId} --private-ip ${PrivateIP2} --public-ip $SecondaryIP | jq -r '.AssociationId'`
- echo "${EIP2_ASSOCIATION_ID}"
- EIP2_ID=`aws ec2 describe-addresses --public-ips ${SecondaryIP} | jq -r '.Addresses[].AllocationId'`
- aws ec2 create-tags --resources ${EIP2_ID} --tags Key="lastused",Value="${CURRENT_EPOCH}"
- aws ec2 create-tags --resources ${EIP2_ID} --tags Key="Project",Value="${PROJECTTAG}"
- echo "Secondary public IP is ${SecondaryIP}"
- fi
-
- echo
- echo "Instance ${InstId} is ready!"
- echo "Instance Public IP is ${PublicIP}"
- echo "Instance Private IP is ${PrivateIP}"
- echo
-
- # Remove previous keys related to this IP from your SSH known hosts so you don't end up with a conflict
- ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "${PublicIP}"
-
- echo "ssh init"
- # this is a do-nothing remote ssh command just to initialize ssh and make sure that the connection is working
- until run "hostname"; do
- sleep 5
- echo "Retry ssh command.."
- done
- echo
-
- ##### Configure Instance
- ## TODO: replace these individual commands with userdata when the spot instance is created?
- echo
- echo
- echo "starting instance config"
-
- echo "Instance will automatically terminate 8 hours from now unless you alter the root crontab"
- run "sudo bash -c 'echo \"\$(date -u -d \"+8 hours\" +\"%M %H\") * * * /usr/sbin/shutdown -h now\" | crontab -'"
- echo
-
- if [[ $APT_UPDATE = "true" ]]; then
- echo
- echo "updating packages"
- run "sudo apt-get update && sudo apt-get upgrade -y"
- fi
-
- echo
- echo "installing docker"
- # install dependencies
- run "sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release gnupg-agent software-properties-common"
- # Add the Docker repository, we are installing from Docker and not the Ubuntu APT repo.
- run 'sudo mkdir -m 0755 -p /etc/apt/keyrings'
- run 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg'
- run 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null'
- run "sudo apt-get update && sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin"
-
- echo
- echo
- # Add your base user to the Docker group so that you do not need sudo to run docker commands
- run "sudo usermod -aG docker ubuntu"
- echo
- fi
-
+function install_k3d {
# install k3d on instance
echo "Installing k3d on instance"
- run "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v${K3D_VERSION} bash"
- echo
- echo "k3d version"
- run "k3d version"
- echo
-
- echo "creating k3d cluster"
-
# Shared k3d settings across all options
# 1 server, 3 agents
k3d_command="export K3D_FIX_MOUNTS=1; k3d cluster create --servers 1 --agents 3 --verbose"
@@ -476,13 +527,7 @@ EOF
echo "Using custom K3S image tag $K3S_IMAGE_TAG..."
k3d_command+=" --image docker.io/rancher/k3s:$K3S_IMAGE_TAG"
fi
-
- # create docker network for k3d cluster
- echo "creating docker network for k3d cluster"
- run "docker network remove k3d-network"
- run "docker network create k3d-network --driver=bridge --subnet=172.20.0.0/16 --gateway 172.20.0.1"
k3d_command+=" --network k3d-network"
-
# Add MetalLB specific k3d config
if [[ "$METAL_LB" == true || "$ATTACH_SECONDARY_IP" == true ]]; then
k3d_command+=" --k3s-arg \"--disable=servicelb@server:0\""
@@ -496,22 +541,17 @@ EOF
echo "using public ip for k3d"
k3d_command+=" --k3s-arg \"--tls-san=${PublicIP}@server:0\""
fi
-
# use weave instead of flannel -- helps with large installs
# we match the 172.x subnets used by CI for consistency
if [[ "$USE_WEAVE" == true ]]; then
-
run "if [[ ! -f /opt/cni/bin/loopback ]]; then sudo mkdir -p /opt/cni/bin && sudo curl -s -L https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz | sudo tar xvz -C /opt/cni/bin; fi"
-
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${SCRIPT_DIR}/weave/* ubuntu@${PublicIP}:/tmp/
-
- # network settings
+ scp -i ${SSHKEY} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${SCRIPT_DIR}/weave/* ${SSHUSER}@${PublicIP}:/tmp/
+ # network settings
k3d_command+=" --k3s-arg \"--flannel-backend=none@server:*\""
k3d_command+=" --k3s-arg \"--disable-network-policy@server:*\""
k3d_command+=" --k3s-arg \"--cluster-cidr=172.21.0.0/16@server:*\""
k3d_command+=" --k3s-arg \"--service-cidr=172.20.0.0/16@server:*\""
k3d_command+=" --k3s-arg \"--cluster-dns=172.20.0.10@server:*\""
-
# volume mounts
k3d_command+=" --volume \"/tmp/weave.yaml:/var/lib/rancher/k3s/server/manifests/weave.yaml@server:*\""
k3d_command+=" --volume /tmp/machine-id-server-0:/etc/machine-id@server:0"
@@ -521,10 +561,18 @@ EOF
k3d_command+=" --volume /opt/cni/bin:/opt/cni/bin@all:*"
fi
- # Create k3d cluster
- echo "Creating k3d cluster with command: ${k3d_command}"
- run "${k3d_command}"
+ run_batch_new
+ run_batch_add "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v${K3D_VERSION} bash"
+ run_batch_add "k3d version"
+ # We may be recreating k3d on an existing cluster, so clean house first
+ run_batch_add "k3d cluster delete --all"
+ run_batch_add "(docker network list --filter name=k3d-network --quiet | grep -E '[A-Za-z0-9]+') && docker network remove k3d-network"
+ run_batch_add "docker network create k3d-network --driver=bridge --subnet=172.20.0.0/16 --gateway 172.20.0.1"
+ run_batch_add "${k3d_command}"
+ run_batch_execute
+}
+function install_kubectl {
# install kubectl
echo Installing kubectl based on k8s version...
K3S_IMAGE_TAG=${K3S_IMAGE_TAG:="${DEFAULT_K3S_TAG}"}
@@ -535,196 +583,187 @@ EOF
KUBECTL_VERSION=$(runwithreturn "k3d version -o json" | jq -r '.k3s' | cut -d'-' -f1)
echo "Using k3d default k8s version $KUBECTL_VERSION."
fi
- KUBECTL_CHECKSUM=`curl -sL https://dl.k8s.io/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256`
- run "curl -LO https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
- runwithexitcode "echo ${KUBECTL_CHECKSUM} kubectl | sha256sum --check" && echo "Good checksum" || { echo "Bad checksum" ; exit 1; }
- run 'sudo mv /home/ubuntu/kubectl /usr/local/bin/'
- run 'sudo chmod +x /usr/local/bin/kubectl'
-
- run "kubectl config use-context k3d-k3s-default"
- run "kubectl cluster-info && kubectl get nodes"
+ KUBECTL_CHECKSUM=$(curl -sL https://dl.k8s.io/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256)
+ run_batch_new
+ run_batch_add "curl -LO https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
+ run_batch_add "echo ${KUBECTL_CHECKSUM} kubectl | sha256sum --check"
+ run_batch_add "sudo mv /home/${SSHUSER}/kubectl /usr/local/bin/"
+ run_batch_add 'sudo chmod +x /usr/local/bin/kubectl'
+ run_batch_add "kubectl config use-context k3d-k3s-default"
+ run_batch_add "kubectl cluster-info && kubectl get nodes"
+ run_batch_execute
echo "copying kubeconfig to workstation..."
mkdir -p ~/.kube
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ubuntu@${PublicIP}:/home/ubuntu/.kube/config ~/.kube/${AWSUSERNAME}-dev-${PROJECTTAG}-config
+ scp -i ${SSHKEY} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${SSHUSER}@${PublicIP}:/home/${SSHUSER}/.kube/config ~/.kube/${KUBECONFIG}
if [[ "$PRIVATE_IP" == true ]]; then
- $sed_gsed -i "s/0\.0\.0\.0/${PrivateIP}/g" ~/.kube/${AWSUSERNAME}-dev-${PROJECTTAG}-config
- else # default is to use public ip
- $sed_gsed -i "s/0\.0\.0\.0/${PublicIP}/g" ~/.kube/${AWSUSERNAME}-dev-${PROJECTTAG}-config
+ $sed_gsed -i "s/0\.0\.0\.0/${PrivateIP}/g" ~/.kube/${KUBECONFIG}
+ else # default is to use public ip
+ $sed_gsed -i "s/0\.0\.0\.0/${PublicIP}/g" ~/.kube/${KUBECONFIG}
fi
+}
+function install_metallb {
# Handle MetalLB cluster resource creation
if [[ "${METAL_LB}" == true || "${ATTACH_SECONDARY_IP}" == true ]]; then
echo "Installing MetalLB..."
-
+ run_batch_new
+
until [[ ${REGISTRY_USERNAME} ]]; do
read -p "Please enter your Registry1 username: " REGISTRY_USERNAME
done
until [[ ${REGISTRY_PASSWORD} ]]; do
read -s -p "Please enter your Registry1 password: " REGISTRY_PASSWORD
done
- run "kubectl create namespace metallb-system"
- run "kubectl create secret docker-registry registry1 \
+ scp -i ${SSHKEY} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${SCRIPT_DIR}/metallb/* ${SSHUSER}@${PublicIP}:/tmp/metallb
+
+ run_batch_add "kubectl create namespace metallb-system"
+ run_batch_add "kubectl create secret docker-registry registry1 \
--docker-server=registry1.dso.mil \
--docker-username=${REGISTRY_USERNAME} \
--docker-password=${REGISTRY_PASSWORD} \
-n metallb-system"
- run "mkdir /tmp/metallb"
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${SCRIPT_DIR}/metallb/* ubuntu@${PublicIP}:/tmp/metallb
- run "kubectl apply -k /tmp/metallb"
-
+ run_batch_add "mkdir /tmp/metallb"
+ run_batch_add "kubectl apply -k /tmp/metallb"
# Wait for controller to be live so that validating webhooks function when we apply the config
- echo "Waiting for MetalLB controller..."
- run "kubectl wait --for=condition=available --timeout 120s -n metallb-system deployment controller"
- echo "MetalLB is installed."
+ run_batch_add 'echo "Waiting for MetalLB controller..."'
+ run_batch_add "kubectl wait --for=condition=available --timeout 120s -n metallb-system deployment controller"
+ run_batch_add 'echo "MetalLB is installed"'
+
if [[ "$METAL_LB" == true ]]; then
echo "Building MetalLB configuration for -m mode."
- cat << EOF > "$tmpdir/metallb-config.yaml"
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
- name: default
- namespace: metallb-system
-spec:
- addresses:
- - 172.20.1.240-172.20.1.243
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
- name: l2advertisement1
- namespace: metallb-system
-spec:
- ipAddressPools:
- - default
+ run_batch_add <<-'ENDSSH'
+ #run this command on remote
+ cat << EOF > metallb-config.yaml
+ apiVersion: metallb.io/v1beta1
+ kind: IPAddressPool
+ metadata:
+ name: default
+ namespace: metallb-system
+ spec:
+ addresses:
+ - 172.20.1.240-172.20.1.243
+ ---
+ apiVersion: metallb.io/v1beta1
+ kind: L2Advertisement
+ metadata:
+ name: l2advertisement1
+ namespace: metallb-system
+ spec:
+ ipAddressPools:
+ - default
EOF
-
+ENDSSH
elif [[ "$ATTACH_SECONDARY_IP" == true ]]; then
echo "Building MetalLB configuration for -a mode."
- cat << EOF > "$tmpdir/metallb-config.yaml"
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
- name: primary
- namespace: metallb-system
- labels:
- privateIp: "$PrivateIP"
- publicIp: "$PublicIP"
-spec:
- addresses:
- - "172.20.1.241/32"
- serviceAllocation:
- priority: 100
- namespaces:
- - istio-system
- serviceSelectors:
- - matchExpressions:
- - {key: app, operator: In, values: [public-ingressgateway]}
----
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
- name: secondary
- namespace: metallb-system
- labels:
- privateIp: "$PrivateIP2"
- publicIp: "$SecondaryIP"
-spec:
- addresses:
- - "172.20.1.240/32"
- serviceAllocation:
- priority: 100
- namespaces:
- - istio-system
- serviceSelectors:
- - matchExpressions:
- - {key: app, operator: In, values: [passthrough-ingressgateway]}
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
- name: primary
- namespace: metallb-system
-spec:
- ipAddressPools:
- - primary
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
- name: secondary
- namespace: metallb-system
-spec:
- ipAddressPools:
- - secondary
+ run_batch_add <<ENDSSH
+ #run this command on remote
+ cat <<EOF > metallb-config.yaml
+ ---
+ apiVersion: metallb.io/v1beta1
+ kind: IPAddressPool
+ metadata:
+ name: primary
+ namespace: metallb-system
+ labels:
+ privateIp: "$PrivateIP"
+ publicIp: "$PublicIP"
+ spec:
+ addresses:
+ - "172.20.1.241/32"
+ serviceAllocation:
+ priority: 100
+ namespaces:
+ - istio-system
+ serviceSelectors:
+ - matchExpressions:
+ - {key: app, operator: In, values: [public-ingressgateway]}
+ ---
+ apiVersion: metallb.io/v1beta1
+ kind: IPAddressPool
+ metadata:
+ name: secondary
+ namespace: metallb-system
+ labels:
+ privateIp: "$PrivateIP2"
+ publicIp: "$SecondaryIP"
+ spec:
+ addresses:
+ - "172.20.1.240/32"
+ serviceAllocation:
+ priority: 100
+ namespaces:
+ - istio-system
+ serviceSelectors:
+ - matchExpressions:
+ - {key: app, operator: In, values: [passthrough-ingressgateway]}
+ ---
+ apiVersion: metallb.io/v1beta1
+ kind: L2Advertisement
+ metadata:
+ name: primary
+ namespace: metallb-system
+ spec:
+ ipAddressPools:
+ - primary
+ ---
+ apiVersion: metallb.io/v1beta1
+ kind: L2Advertisement
+ metadata:
+ name: secondary
+ namespace: metallb-system
+ spec:
+ ipAddressPools:
+ - secondary
EOF
-
- cat << EOF > "$tmpdir/primary-proxy.yaml"
-ports:
- 443.tcp:
- - 172.20.1.241
-settings:
- workerConnections: 1024
+ENDSSH
+
+ run_batch_add <<ENDSSH
+ cat <<EOF > primaryProxy.yaml
+ ports:
+ 443.tcp:
+ - 172.20.1.241
+ settings:
+ workerConnections: 1024
EOF
-
- cat << EOF > "$tmpdir/secondary-proxy.yaml"
-ports:
- 443.tcp:
- - 172.20.1.240
-settings:
- workerConnections: 1024
+ENDSSH
+
+ run_batch_add <<ENDSSH
+ cat <<EOF > secondaryProxy.yaml
+ ports:
+ 443.tcp:
+ - 172.20.1.240
+ settings:
+ workerConnections: 1024
EOF
+ENDSSH
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes "$tmpdir/primary-proxy.yaml" ubuntu@${PublicIP}:/home/ubuntu/primary-proxy.yaml
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes "$tmpdir/secondary-proxy.yaml" ubuntu@${PublicIP}:/home/ubuntu/secondary-proxy.yaml
-
- run "docker run -d --name=primaryProxy --network=k3d-network -p $PrivateIP:443:443 -v /home/ubuntu/primary-proxy.yaml:/etc/confd/values.yaml ghcr.io/k3d-io/k3d-proxy:$K3D_VERSION"
- run "docker run -d --name=secondaryProxy --network=k3d-network -p $PrivateIP2:443:443 -v /home/ubuntu/secondary-proxy.yaml:/etc/confd/values.yaml ghcr.io/k3d-io/k3d-proxy:$K3D_VERSION"
+ run_batch_add "docker run -d --name=primaryProxy --network=k3d-network -p $PrivateIP:443:443 -v /home/${SSHUSER}/primaryProxy.yaml:/etc/confd/values.yaml ghcr.io/k3d-io/k3d-proxy:$K3D_VERSION"
+ run_batch_add "docker run -d --name=secondaryProxy --network=k3d-network -p $PrivateIP2:443:443 -v /home/${SSHUSER}//secondaryProxy.yaml:/etc/confd/values.yaml ghcr.io/k3d-io/k3d-proxy:$K3D_VERSION"
fi
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes "$tmpdir/metallb-config.yaml" ubuntu@${PublicIP}:/home/ubuntu/metallb-config.yaml
-
- run "kubectl create -f metallb-config.yaml"
- fi
-
- cat << EOF > "$tmpdir/dns-update.sh"
-#!/usr/bin/env bash
-mkdir -p /root/.kube
-cp /home/ubuntu/.kube/config /root/.kube/config
-sed -i '/dev.bigbang.mil/d' /etc/hosts
-EOF
-
- if [[ "$METAL_LB" == true ]]; then
- cat << EOF >> "$tmpdir/dns-update.sh"
-echo '## begin dev.bigbang.mil section (METAL_LB)' >> /etc/hosts
-echo '172.20.1.240 keycloak.dev.bigbang.mil vault.dev.bigbang.mil' >> /etc/hosts
-echo '172.20.1.241 anchore-api.dev.bigbang.mil anchore.dev.bigbang.mil argocd.dev.bigbang.mil gitlab.dev.bigbang.mil registry.dev.bigbang.mil tracing.dev.bigbang.mil kiali.dev.bigbang.mil kibana.dev.bigbang.mil chat.dev.bigbang.mil minio.dev.bigbang.mil minio-api.dev.bigbang.mil alertmanager.dev.bigbang.mil grafana.dev.bigbang.mil prometheus.dev.bigbang.mil nexus.dev.bigbang.mil sonarqube.dev.bigbang.mil tempo.dev.bigbang.mil twistlock.dev.bigbang.mil' >> /etc/hosts
-echo '## end dev.bigbang.mil section' >> /etc/hosts
-
-kubectl get configmap -n kube-system coredns -o yaml | sed '/^ 172.20.0.1 host.k3d.internal$/a\ \ \ \ 172.20.1.240 keycloak.dev.bigbang.mil vault.dev.bigbang.mil' | kubectl apply -f -
-EOF
- elif [[ "$ATTACH_SECONDARY_IP" == true ]]; then
-
- cat << EOF >> "$tmpdir/dns-update.sh"
-echo '## begin dev.bigbang.mil section (ATTACH_SECONDARY_IP)' >> /etc/hosts
-echo '$PrivateIP2 keycloak.dev.bigbang.mil vault.dev.bigbang.mil' >> /etc/hosts
-echo '$PrivateIP anchore-api.dev.bigbang.mil anchore.dev.bigbang.mil argocd.dev.bigbang.mil gitlab.dev.bigbang.mil registry.dev.bigbang.mil tracing.dev.bigbang.mil kiali.dev.bigbang.mil kibana.dev.bigbang.mil chat.dev.bigbang.mil minio.dev.bigbang.mil minio-api.dev.bigbang.mil alertmanager.dev.bigbang.mil grafana.dev.bigbang.mil prometheus.dev.bigbang.mil nexus.dev.bigbang.mil sonarqube.dev.bigbang.mil tempo.dev.bigbang.mil twistlock.dev.bigbang.mil' >> /etc/hosts
-echo '## end dev.bigbang.mil section' >> /etc/hosts
-
-kubectl get configmap -n kube-system coredns -o yaml | sed '/^ .* host.k3d.internal$/a\ \ \ \ $PrivateIP2 keycloak.dev.bigbang.mil vault.dev.bigbang.mil' | kubectl apply -f -
-EOF
+ run_batch_add "kubectl create -f metallb-config.yaml"
+ if [[ "$METAL_LB" == "true" ]]; then
+ run_batch_add <<-'ENDSSH'
+ # run this command on remote
+ # fix /etc/hosts for new cluster
+ sudo sed -i '/dev.bigbang.mil/d' /etc/hosts
+ sudo bash -c "echo '## begin dev.bigbang.mil section (METAL_LB)' >> /etc/hosts"
+ sudo bash -c "echo 172.20.1.240 keycloak.dev.bigbang.mil vault.dev.bigbang.mil >> /etc/hosts"
+ sudo bash -c "echo 172.20.1.241 anchore-api.dev.bigbang.mil anchore.dev.bigbang.mil argocd.dev.bigbang.mil gitlab.dev.bigbang.mil registry.dev.bigbang.mil tracing.dev.bigbang.mil kiali.dev.bigbang.mil kibana.dev.bigbang.mil chat.dev.bigbang.mil minio.dev.bigbang.mil minio-api.dev.bigbang.mil alertmanager.dev.bigbang.mil grafana.dev.bigbang.mil prometheus.dev.bigbang.mil nexus.dev.bigbang.mil sonarqube.dev.bigbang.mil tempo.dev.bigbang.mil twistlock.dev.bigbang.mil >> /etc/hosts"
+ sudo bash -c "echo '## end dev.bigbang.mil section' >> /etc/hosts"
+ # run kubectl to add keycloak and vault's hostname/IP to the configmap for coredns, restart coredns
+ kubectl get configmap -n kube-system coredns -o yaml | sed '/^ 172.20.0.1 host.k3d.internal$/a\ \ \ \ 172.20.1.240 keycloak.dev.bigbang.mil vault.dev.bigbang.mil' | kubectl apply -f -
+ kubectl delete pod -n kube-system -l k8s-app=kube-dns
+ENDSSH
+ fi
+ run_batch_execute
fi
+}
- cat << EOF >> "$tmpdir/dns-update.sh"
-kubectl delete pod -n kube-system -l k8s-app=kube-dns
-EOF
-
- scp -i ~/.ssh/${KeyName}.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes "$tmpdir/dns-update.sh" ubuntu@${PublicIP}:/home/ubuntu/dns-update.sh
- run "sudo bash /home/ubuntu/dns-update.sh"
-
+function print_instructions {
echo
echo "================================================================================"
echo "====================== DEPLOYMENT FINISHED ====================================="
@@ -732,23 +771,24 @@ EOF
# ending instructions
echo
echo "SAVE THE FOLLOWING INSTRUCTIONS INTO A TEMPORARY TEXT DOCUMENT SO THAT YOU DON'T LOSE THEM"
- echo "NOTE: The EC2 instance will automatically terminate 8 hours from the time of creation unless you delete the root cron job"
+ if [[ "$TERMINATE_INSTANCE" != "false" ]]; then
+ echo "NOTE: The EC2 instance will automatically terminate 8 hours from the time of creation unless you delete the root cron job"
+ fi
echo
echo "ssh to instance:"
- echo " ssh -i ~/.ssh/${KeyName}.pem -o IdentitiesOnly=yes ubuntu@${PublicIP}"
+ echo " ssh -i ${SSHKEY} -o IdentitiesOnly=yes ${SSHUSER}@${PublicIP}"
echo
echo "To use kubectl from your local workstation you must set the KUBECONFIG environment variable:"
- echo " export KUBECONFIG=~/.kube/${AWSUSERNAME}-dev-${PROJECTTAG}-config"
- if [[ "$PRIVATE_IP" == true ]]
- then
+ echo " export KUBECONFIG=~/.kube/${KUBECONFIG}"
+ if [[ "$PRIVATE_IP" == true ]]; then
echo "The cluster connection will not work until you start sshuttle as described below."
fi
echo
- if [[ "$METAL_LB" == true ]]; then # using MetalLB
+ if [[ "$METAL_LB" == true ]]; then # using MetalLB
if [[ "$PRIVATE_IP" == true ]]; then # using MetalLB and private IP
echo "Start sshuttle in a separate terminal window:"
- echo " sshuttle --dns -vr ubuntu@${PublicIP} 172.31.0.0/16 --ssh-cmd 'ssh -i ~/.ssh/${KeyName}.pem -D 127.0.0.1:12345'"
+ echo " sshuttle --dns -vr ${SSHUSER}@${PublicIP} 172.31.0.0/16 --ssh-cmd 'ssh -i ${SSHKEY} -D 127.0.0.1:12345'"
echo "Do not edit /etc/hosts on your local workstation."
echo "Edit /etc/hosts on the EC2 instance. Sample /etc/host entries have already been added there."
echo "Manually add more hostnames as needed."
@@ -756,10 +796,10 @@ EOF
echo "You must use Firefox browser with with manual SOCKs v5 proxy configuration to localhost with port 12345."
echo "Also ensure 'Proxy DNS when using SOCKS v5' is checked."
echo "Or, with other browsers like Chrome you could use a browser plugin like foxyproxy to do the same thing as Firefox."
- else # using MetalLB and public IP
+ else # using MetalLB and public IP
echo "OPTION 1: ACCESS APPLICATIONS WITH WEB BROWSER ONLY"
echo "To access apps from browser only start ssh with application-level port forwarding:"
- echo " ssh -i ~/.ssh/${KeyName}.pem ubuntu@${PublicIP} -D 127.0.0.1:12345"
+ echo " ssh -i ${SSHKEY} ${SSHUSER}@${PublicIP} -D 127.0.0.1:12345"
echo "Do not edit /etc/hosts on your local workstation."
echo "Edit /etc/hosts on the EC2 instance. Sample /etc/host entries have already been added there."
echo "Manually add more hostnames as needed."
@@ -770,22 +810,22 @@ EOF
echo
echo "OPTION 2: ACCESS APPLICATIONS WITH WEB BROWSER AND COMMAND LINE"
echo "To access apps from browser and from the workstation command line start sshuttle in a separate terminal window."
- echo " sshuttle --dns -vr ubuntu@${PublicIP} 172.20.1.0/24 --ssh-cmd 'ssh -i ~/.ssh/${KeyName}.pem'"
+ echo " sshuttle --dns -vr ${SSHUSER}@${PublicIP} 172.20.1.0/24 --ssh-cmd 'ssh -i ${SSHKEY}'"
echo "Edit your workstation /etc/hosts to add the LOADBALANCER EXTERNAL-IPs from the istio-system services with application hostnames."
echo "Here is an example. You might have to change this depending on the number of gateways you configure for k8s cluster."
echo " # METALLB ISTIO INGRESS IPs"
echo " 172.20.1.240 keycloak.dev.bigbang.mil vault.dev.bigbang.mil"
echo " 172.20.1.241 sonarqube.dev.bigbang.mil prometheus.dev.bigbang.mil nexus.dev.bigbang.mil gitlab.dev.bigbang.mil"
fi
- elif [[ "$PRIVATE_IP" == true ]]; then # not using MetalLB
+ elif [[ "$PRIVATE_IP" == true ]]; then # not using MetalLB
# Not using MetalLB and using private IP
echo "Start sshuttle in a separate terminal window:"
- echo " sshuttle --dns -vr ubuntu@${PublicIP} 172.31.0.0/16 --ssh-cmd 'ssh -i ~/.ssh/${KeyName}.pem'"
+ echo " sshuttle --dns -vr ${SSHUSER}@${PublicIP} 172.31.0.0/16 --ssh-cmd 'ssh -i ${SSHKEY}'"
echo
echo "To access apps from a browser edit your /etc/hosts to add the private IP of your EC2 instance with application hostnames. Example:"
echo " ${PrivateIP} gitlab.dev.bigbang.mil prometheus.dev.bigbang.mil kibana.dev.bigbang.mil"
- echo
- else # Not using MetalLB and using public IP. This is the default
+ echo
+ else # Not using MetalLB and using public IP. This is the default
echo "To access apps from a browser edit your /etc/hosts to add the public IP of your EC2 instance with application hostnames."
echo "Example:"
echo " ${PublicIP} gitlab.dev.bigbang.mil prometheus.dev.bigbang.mil kibana.dev.bigbang.mil"
@@ -798,173 +838,404 @@ EOF
fi
}
-function report_instances
-{
- aws ec2 describe-instances \
- --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" \
- --query 'Reservations[*].Instances[*].[InstanceId,State.Name,PublicIpAddress,SecurityGroups[0].GroupId,Tags[?Key==`Project`].Value | [0]]' \
- --output text
-}
+function initialize_instance {
+ ##### Configure Instance
+ ## TODO: replace these individual commands with userdata when the spot instance is created?
+ echo
+ echo
+ echo "starting instance config"
-#### Global variables - These allow the script to be run by non-bigbang devs easily - Update VPC_ID here or export environment variable for it if not default VPC
-if [[ -z "${VPC_ID}" ]]; then
- VPC_ID="$(aws ec2 describe-vpcs --filters Name=is-default,Values=true | jq -j .Vpcs[0].VpcId)"
- if [[ -z "${VPC_ID}" ]]; then
- echo "AWS account has no default VPC - please provide VPC_ID"
+ runwithexitcode "grep '^DISTRIB_ID=Ubuntu' /etc/lsb-release"
+ if [[ $? -ne 0 ]]; then
+ echo "This script only knows how to provision Ubuntu systems. Sorry!"
exit 1
fi
-fi
-if [[ -n "${SUBNET_ID}" ]]; then
- if [[ "$(aws ec2 describe-subnets --subnet-id "${SUBNET_ID}" --filters "Name=vpc-id,Values=${VPC_ID}" | jq -j .Subnets[0])" == "null" ]]; then
- echo "SUBNET_ID ${SUBNET_ID} does not belong to VPC ${VPC_ID}"
- exit 1
+ run_batch_new
+ run_batch_add "sudo -S -- bash -c 'echo \"$SSHUSER ALL=(ALL:ALL) NOPASSWD: ALL\" | sudo tee /etc/sudoers.d/dont-prompt-$SSHUSER-for-sudo-password;'"
+ run_batch_add "sudo -- bash -c \"sysctl -w vm.max_map_count=524288; \
+ echo vm.max_map_count=524288 > /etc/sysctl.d/vm-max_map_count.conf; \
+ sysctl -w fs.nr_open=13181252; \
+ echo fs.nr_open=13181252 > /etc/sysctl.d/fs-nr_open.conf; \
+ sysctl -w fs.file-max=13181250; \
+ echo fs.file-max=13181250 > /etc/sysctl.d/fs-file-max.conf; \
+ echo fs.inotify.max_user_instances=1024 > /etc/sysctl.d/fs-inotify-max_user_instances.conf; \
+ sysctl -w fs.inotify.max_user_instances=1024; \
+ echo fs.inotify.max_user_watches=1048576 > /etc/sysctl.d/fs-inotify-max_user_watches.conf; \
+ sysctl -w fs.inotify.max_user_watches=1048576; \
+ echo fs.may_detach_mounts=1 >> /etc/sysctl.d/fs-may_detach_mounts.conf; \
+ sysctl -w fs.may_detach_mounts=1; \
+ sysctl -p; \
+ echo '* soft nofile 13181250' >> /etc/security/limits.d/ulimits.conf; \
+ echo '* hard nofile 13181250' >> /etc/security/limits.d/ulimits.conf; \
+ echo '* soft nproc 13181250' >> /etc/security/limits.d/ulimits.conf; \
+ echo '* hard nproc 13181250' >> /etc/security/limits.d/ulimits.conf; \
+ modprobe br_netfilter; \
+ modprobe nf_nat_redirect; \
+ modprobe xt_REDIRECT; \
+ modprobe xt_owner; \
+ modprobe xt_statistic; \
+ echo br_netfilter >> /etc/modules-load.d/istio-iptables.conf; \
+ echo nf_nat_redirect >> /etc/modules-load.d/istio-iptables.conf; \
+ echo xt_REDIRECT >> /etc/modules-load.d/istio-iptables.conf; \
+ echo xt_owner >> /etc/modules-load.d/istio-iptables.conf; \
+ echo xt_statistic >> /etc/modules-load.d/istio-iptables.conf\" "
+
+ if [[ "$INIT_SCRIPT" != "" ]]; then
+ echo "Running init script"
+ if [[ ! -e ${INIT_SCRIPT} ]]; then
+ echo "Init script does not exist"
+ exit 1
+ fi
+ cat ${INIT_SCRIPT} | run "sudo cat > /tmp/k3d-dev-initscript"
+ run_batch_add "sudo bash /tmp/k3d-dev-initscript"
fi
-else
- SUBNET_ID="$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=${VPC_ID}" "Name=default-for-az,Values=true" | jq -j .Subnets[0].SubnetId)"
- if [[ "${SUBNET_ID}" == "null" ]]; then
- echo "VPC ${VPC_ID} has no default subnets - please provide SUBNET_ID"
- exit 1
+
+ if [[ "$TERMINATE_INSTANCE" != "false" ]]; then
+ echo "Instance will automatically terminate 8 hours from now unless you alter the root crontab"
+ run_batch_add "sudo bash -c 'echo \"\$(date -u -d \"+8 hours\" +\"%M %H\") * * * /usr/sbin/shutdown -h now\" | crontab -'"
+ echo
fi
-fi
-# If the user is using her own AMI, then respect that and do not update it.
-APT_UPDATE="true"
-if [[ $AMI_ID ]]; then
- APT_UPDATE="false"
-else
- # default
- AMI_ID=$(getDefaultAmi)
-fi
+ if [[ $APT_UPDATE = "true" ]]; then
+ echo
+ run_batch_add 'echo "updating packages"'
+ run_batch_add "sudo DEBIAN_FRONTEND=noninteractive apt-get update"
+ run_batch_add "sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y"
+ fi
+ run_batch_execute
+}
-#### Preflight Checks
-# Check that the VPC is available
-EXISTING_VPC=$(aws ec2 describe-vpcs | grep ${VPC_ID})
-if [[ -z "${EXISTING_VPC}" ]]; then
- echo "VPC is not available in the current AWS_PROFILE - Update VPC_ID"
- exit 1
-fi
-# check for tools
-tooldependencies=(jq sed aws ssh ssh-keygen scp kubectl tr base64)
-for tooldependency in "${tooldependencies[@]}"
- do
- command -v $tooldependency >/dev/null 2>&1 || { echo >&2 " $tooldependency is not installed."; missingtool=1; }
+function cloud_aws_prep_objects {
+ #### Cleaning up unused Elastic IPs
+ ALLOCATIONIDs=($(aws ec2 describe-addresses --filters "Name=tag:Owner,Values=${AWSUSERNAME}" "Name=tag:Project,Values=${PROJECTTAG}" --query "Addresses[?AssociationId==null]" | jq -r '.[].AllocationId'))
+ for i in "${ALLOCATIONIDs[@]}"; do
+ echo -n "Releasing Elastic IP $i ..."
+ aws ec2 release-address --allocation-id $i
+ echo "done"
done
-sed_gsed="sed"
-# verify sed version if mac
-# alias prohibited, symlinks permitted
-uname="$(uname -s)"
-if [[ "${uname}" == "Darwin" ]]; then
- if [[ $(command -v gsed) ]]; then
- sed_gsed="gsed"
+
+ #### SSH Key Pair
+ # Create SSH key if it doesn't exist
+ echo -n Checking if key pair ${KeyName} exists ...
+ aws ec2 describe-key-pairs --output json --no-cli-pager --key-names ${KeyName} >/dev/null 2>&1 || keypair=missing
+ if [ "${keypair}" == "missing" ]; then
+ echo -n -e "missing\nCreating key pair ${KeyName} ... "
+ aws ec2 create-key-pair --output json --no-cli-pager --key-name ${KeyName} | jq -r '.KeyMaterial' >${SSHKEY}
+ chmod 600 ${SSHKEY}
+ echo done
else
- missingtool=1
- echo ' gnu-sed is not installed. "brew install gnu-sed"'
+ echo found
+ fi
+ if [[ ! -f ${SSHKEY} ]]; then
+ echo "Local key file ${SSHKEY} does not exist. Cannot continue." >&2
+ exit 1
fi
-fi
-# if tool missing, exit
-if [[ "${missingtool}" == 1 ]]; then
- echo " Please install required tools. Aborting."
- exit 1
-fi
-# getting AWS user name
-AWSUSERNAME=$( aws sts get-caller-identity --query Arn --output text | cut -f 2 -d '/' )
+ #### Security Group
+ # Create security group if it doesn't exist
+ echo -n "Checking if security group ${SGname} exists ..."
+ aws ec2 describe-security-groups --output json --no-cli-pager --group-names ${SGname} >/dev/null 2>&1 || secgrp=missing
+ if [ "${secgrp}" == "missing" ]; then
+ echo -e "missing\nCreating security group ${SGname} ... "
+ aws ec2 create-security-group --output json --no-cli-pager --description "IP based filtering for ${SGname}" --group-name ${SGname} --vpc-id ${VPC_ID}
+ echo done
+ else
+ echo found
+ fi
+}
-# check for aws username environment variable. If not found then terminate script
-if [[ -z "${AWSUSERNAME}" ]]; then
- echo "You must configure your AWS credentials. Your AWS user name is used to name resources in AWS. Example:"
- echo " aws configure"
- exit 1
-else
- echo "AWS User Name: ${AWSUSERNAME}"
-fi
+function cloud_aws_request_spot_instance
+{
+ ##### Launch Specification
+ # Typical settings for Big Bang development
+ InstanceType="${InstSize}"
+ VolumeSize=120
+ echo "Using AMI image id ${AMI_ID}"
+ ImageId="${AMI_ID}"
-#### Configure Environment
-# Identify which VPC to create the spot instance in
-VPC="${VPC_ID}" # default VPC
-RESET_K3D=false
-ATTACH_SECONDARY_IP=${ATTACH_SECONDARY_IP:=false}
-PROJECTTAG="default"
-# Assign a name for your Security Group. Typically, people use their username to make it easy to identify
-SGname="${AWSUSERNAME}-dev-${PROJECTTAG}"
-# Assign a name for your SSH Key Pair. Typically, people use their username to make it easy to identify
-KeyName="${AWSUSERNAME}-dev-${PROJECTTAG}"
-# The action to perform
-action=create_instances
+ # Create userdata.txt
+ mkdir -p ~/aws
+ cat <<EOF >~/aws/userdata.txt
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="==MYBOUNDARY=="
-while [ -n "$1" ]; do # while loop starts
+--==MYBOUNDARY==
+Content-Type: text/x-shellscript; charset="us-ascii"
- case "$1" in
+#!/bin/bash
+'
+EOF
- -t) echo "-t option passed to use additional tags on instance"
- shift
- PROJECTTAG=$1
- SGname="${AWSUSERNAME}-dev-${PROJECTTAG}"
- KeyName="${AWSUSERNAME}-dev-${PROJECTTAG}"
- ;;
+ # Create the device mapping and spot options JSON files
+ echo "Creating device_mappings.json ..."
+ mkdir -p ~/aws
- -b) echo "-b option passed for big k3d cluster using M5 instance"
- BIG_INSTANCE=true
- ;;
+ # gp3 volumes are 20% cheaper than gp2 and comes with 3000 Iops baseline and 125 MiB/s baseline throughput for free.
+ cat <<EOF >~/aws/device_mappings.json
+[
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeType": "gp3",
+ "VolumeSize": ${VolumeSize},
+ "Encrypted": true
+ }
+ }
+]
+EOF
- -p) echo "-p option passed to create k3d cluster with private ip"
- if [[ "${ATTACH_SECONDARY_IP}" = false ]]; then
- PRIVATE_IP=true
- else
- echo "Disabling -p option because -a was specified."
- fi
- ;;
+ echo "Creating spot_options.json ..."
+ cat <<EOF >~/aws/spot_options.json
+{
+ "MarketType": "spot",
+ "SpotOptions": {
+ "MaxPrice": "${SpotPrice}",
+ "SpotInstanceType": "one-time"
+ }
+}
+EOF
- -m) echo "-m option passed to install MetalLB"
- if [[ "${ATTACH_SECONDARY_IP}" = false ]]; then
- METAL_LB=true
- else
- echo "Disabling -m option because -a was specified."
- fi
- ;;
+ #### Request a Spot Instance
- -a) echo "-a option passed to create secondary public IP (-p and -m flags are skipped if set)"
- PRIVATE_IP=false
- METAL_LB=false
- ATTACH_SECONDARY_IP=true
- ;;
+ # Run a spot instance with our launch spec for the max. of 6 hours
+ # NOTE: t3a.2xlarge spot price is 0.35 m5a.4xlarge is 0.69
+ echo "Running spot instance ..."
- -d) echo "-d option passed to destroy the AWS resources"
- action=destroy_instances
- ;;
+ if [[ "${ATTACH_SECONDARY_IP}" == true ]]; then
+ # If we are using a secondary IP, we don't want to assign public IPs at launch time. Instead, the script will attach both public IPs after the instance is launched.
+ additional_create_instance_options="--no-associate-public-ip-address --secondary-private-ip-address-count 1"
+ else
+ additional_create_instance_options="--associate-public-ip-address"
+ fi
- -h) echo "Usage: $0 [argument ...]"
- echo ""
- echo "arguments:"
- echo " -a attach secondary Public IP (overrides -p and -m flags)"
- echo " -b use BIG M5 instance. Default is m5a.4xlarge"
- echo " -d destroy related AWS resources"
- echo " -h output help"
- echo " -m create k3d cluster with metalLB"
- echo " -p use private IP for security group and k3d cluster"
- echo " -r Report on all instances owned by your user"
- echo " -t Set the project tag on the instance"
- echo " -u Update security group for instances"
- echo " -w install the weave CNI instead of the default flannel CNI"
+ InstId=$(aws ec2 run-instances \
+ --output json --no-paginate \
+ --count 1 --image-id "${ImageId}" \
+ --instance-type "${InstanceType}" \
+ --subnet-id "${SUBNET_ID}" \
+ --key-name "${KeyName}" \
+ --security-group-ids "${SecurityGroupId}" \
+ --instance-initiated-shutdown-behavior "terminate" \
+ --user-data file://$HOME/aws/userdata.txt \
+ --block-device-mappings file://$HOME/aws/device_mappings.json \
+ --instance-market-options file://$HOME/aws/spot_options.json \
+ ${additional_create_instance_options} |
+ jq -r '.Instances[0].InstanceId')
+
+ # Check if spot instance request was not created
+ if [ -z ${InstId} ]; then
+ exit 1
+ fi
+
+ # Add name tag to spot instance
+ aws ec2 create-tags --resources ${InstId} --tags Key=Name,Value=${AWSUSERNAME}-dev &>/dev/null
+ aws ec2 create-tags --resources ${InstId} --tags Key=Project,Value=${PROJECTTAG} &>/dev/null
+
+ # Request was created, now you need to wait for it to be filled
+ echo "Waiting for instance ${InstId} to be ready ..."
+ aws ec2 wait instance-running --output json --no-cli-pager --instance-ids ${InstId} &>/dev/null
+
+ # allow some extra seconds for the instance to be fully initialized
+ echo "Almost there, 15 seconds to go..."
+ sleep 15
+}
+
+function cloud_aws_assign_ip_addresses
+{
+ ## IP Address Allocation and Attachment
+ CURRENT_EPOCH=$(date +'%s')
+
+ # Get the private IP address of our instance
+ PrivateIP=$(aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PrivateIpAddress')
+
+ # Use Elastic IPs if a Secondary IP is required, instead of the auto assigned one.
+ if [[ "${ATTACH_SECONDARY_IP}" == false ]]; then
+ PublicIP=$(aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PublicIpAddress')
+ else
+ echo "Checking to see if an Elastic IP is already allocated and not attached..."
+ PublicIP=$(aws ec2 describe-addresses --filters "Name=tag:Name,Values=${AWSUSERNAME}-EIP1" "Name=tag:Project,Values=${PROJECTTAG}" --query 'Addresses[?AssociationId==null]' | jq -r '.[0].PublicIp // ""')
+ if [[ -z "${PublicIP}" ]]; then
+ echo "Allocating a new/another primary elastic IP..."
+ PublicIP=$(aws ec2 allocate-address --output json --no-cli-pager --tag-specifications="ResourceType=elastic-ip,Tags=[{Key=Name,Value=${AWSUSERNAME}-EIP1},{Key=Owner,Value=${AWSUSERNAME}}]" | jq -r '.PublicIp')
+ else
+ echo "Previously allocated primary Elastic IP ${PublicIP} found."
+ fi
+
+ echo -n "Associating IP ${PublicIP} address to instance ${InstId} ..."
+ EIP1_ASSOCIATION_ID=$(aws ec2 associate-address --output json --no-cli-pager --instance-id ${InstId} --private-ip ${PrivateIP} --public-ip $PublicIP | jq -r '.AssociationId')
+ echo "${EIP1_ASSOCIATION_ID}"
+ EIP1_ID=$(aws ec2 describe-addresses --public-ips ${PublicIP} | jq -r '.Addresses[].AllocationId')
+ aws ec2 create-tags --resources ${EIP1_ID} --tags Key="lastused",Value="${CURRENT_EPOCH}"
+ aws ec2 create-tags --resources ${EIP1_ID} --tags Key="Project",Value="${PROJECTTAG}"
+
+ PrivateIP2=$(getPrivateIP2)
+ echo "Checking to see if a Secondary Elastic IP is already allocated and not attached..."
+ SecondaryIP=$(aws ec2 describe-addresses --filters "Name=tag:Name,Values=${AWSUSERNAME}-EIP2" "Name=tag:Project,Values=${PROJECTTAG}" --query 'Addresses[?AssociationId==null]' | jq -r '.[0].PublicIp // ""')
+ if [[ -z "${SecondaryIP}" ]]; then
+ echo "Allocating a new/another secondary elastic IP..."
+ SecondaryIP=$(aws ec2 allocate-address --output json --no-cli-pager --tag-specifications="ResourceType=elastic-ip,Tags=[{Key=Name,Value=${AWSUSERNAME}-EIP2},{Key=Owner,Value=${AWSUSERNAME}}]" | jq -r '.PublicIp')
+ else
+ echo "Previously allocated secondary Elastic IP ${SecondaryIP} found."
+ fi
+ echo -n "Associating Secondary IP ${SecondaryIP} address to instance ${InstId}..."
+ EIP2_ASSOCIATION_ID=$(aws ec2 associate-address --output json --no-cli-pager --instance-id ${InstId} --private-ip ${PrivateIP2} --public-ip $SecondaryIP | jq -r '.AssociationId')
+ echo "${EIP2_ASSOCIATION_ID}"
+ EIP2_ID=$(aws ec2 describe-addresses --public-ips ${SecondaryIP} | jq -r '.Addresses[].AllocationId')
+ aws ec2 create-tags --resources ${EIP2_ID} --tags Key="lastused",Value="${CURRENT_EPOCH}"
+ aws ec2 create-tags --resources ${EIP2_ID} --tags Key="Project",Value="${PROJECTTAG}"
+ echo "Secondary public IP is ${SecondaryIP}"
+ fi
+}
+
+function cluster_mgmt_select_action_for_existing {
+ echo "💣 Big Bang Cluster Management 💣"
+ PS3="Please select an option: "
+ options=("Re-create K3D cluster" "Recreate the EC2 instance from scratch" "Do Nothing")
+
+ select opt in "${options[@]}"; do
+ case $opt in
+ "Re-create K3D cluster")
+ read -p "Are you sure you want to re-create a K3D cluster on this instance (y/n)? " -r
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ echo
+ exit 1
+ fi
+ RESET_K3D=true
+ SecondaryIP=$(aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].NetworkInterfaces[0].PrivateIpAddresses[] | select(.Primary==false) | .Association.PublicIp')
+ PrivateIP2=$(getPrivateIP2)
+ if [[ "${ATTACH_SECONDARY_IP}" == true && -z "${SecondaryIP}" ]]; then
+ echo "Secondary IP didn't exist at the time of creation of the instance, so cannot attach one without re-creating it with the -a flag selected."
+ exit 1
+ fi
+ break
+ ;;
+ "Recreate the instance from scratch")
+ # Code for recreating the EC2 instance from scratch
+ CLOUD_RECREATE_INSTANCE=true
+ break
+ ;;
+ "Do Nothing")
+ echo "Doing nothing..."
exit 0
- ;;
+ ;;
+ *)
+ echo "Invalid option. Please try again."
+ ;;
+ esac
+ done
+}
- -u) action=update_instances
- ;;
+function cloud_aws_create_instances {
+ echo "Checking for existing cluster for ${AWSUSERNAME}."
+ InstId=$(aws ec2 describe-instances \
+ --output text \
+ --query "Reservations[].Instances[].InstanceId" \
+ --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" "Name=tag:Project,Values=${PROJECTTAG}" "Name=instance-state-name,Values=running")
+ if [[ $InstId ]]; then
+ PublicIP=$(aws ec2 describe-instances --output text --no-cli-pager --instance-id ${InstId} --query "Reservations[].Instances[].PublicIpAddress")
+ PrivateIP=$(aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${InstId} | jq -r '.Reservations[0].Instances[0].PrivateIpAddress')
+ echo "Existing cluster found running on instance ${InstId} on ${PublicIP} / ${PrivateIP}"
+ if [[ "${RESET_K3D}" != "true" ]] && [[ "${CLOUD_RECREATE_INSTANCE}" != "true" ]]; then
+ cluster_mgmt_select_action_for_existing
+ fi
+ if [[ "${CLOUD_RECREATE_INSTANCE}" == "true" ]]; then
+ destroy_instances
+ fi
+ fi
- -r) action=report_instances
- ;;
+ if [[ "${RESET_K3D}" == false ]] ; then
+ if [[ "$BIG_INSTANCE" == true ]]; then
+ echo "Will use large m5a.4xlarge spot instance"
+ InstSize="m5a.4xlarge"
+ SpotPrice="0.69"
+ else
+ echo "Will use standard t3a.2xlarge spot instance"
+ InstSize="t3a.2xlarge"
+ SpotPrice="0.35"
+ fi
- -w) echo "-w option passed to use Weave CNI"
- USE_WEAVE=true
- ;;
+ cloud_aws_prep_objects
+ update_ec2_security_group
+ cloud_aws_request_spot_instance
+ cloud_aws_assign_ip_addresses
- *) echo "Option $1 not recognized" ;; # In case a non-existent option is submitted
+ echo
+ echo "Instance ${InstId} is ready!"
+ echo "Instance Public IP is ${PublicIP}"
+ echo "Instance Private IP is ${PrivateIP}"
+ echo
- esac
- shift
-done
+ # Remove previous keys related to this IP from your SSH known hosts so you don't end up with a conflict
+ ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "${PublicIP}"
+
+ echo "ssh init"
+ # this is a do-nothing remote ssh command just to initialize ssh and make sure that the connection is working
+ until run "hostname"; do
+ sleep 5
+ echo "Retry ssh command.."
+ done
+ echo
+ fi
+}
+
+function fix_etc_hosts {
+ if [[ "$ATTACH_SECONDARY_IP" == true ]] && [[ "$METAL_LB" != "true" ]]; then
+ run <<ENDSSH
+ # run this command on remote
+ # fix /etc/hosts for new cluster
+ sudo sed -i '/dev.bigbang.mil/d' /etc/hosts
+ sudo bash -c "echo '## begin dev.bigbang.mil section (ATTACH_SECONDARY_IP)' >> /etc/hosts"
+ sudo bash -c "echo $PrivateIP2 keycloak.dev.bigbang.mil vault.dev.bigbang.mil >> /etc/hosts"
+ sudo bash -c "echo $PrivateIP anchore-api.dev.bigbang.mil anchore.dev.bigbang.mil argocd.dev.bigbang.mil gitlab.dev.bigbang.mil registry.dev.bigbang.mil tracing.dev.bigbang.mil kiali.dev.bigbang.mil kibana.dev.bigbang.mil chat.dev.bigbang.mil minio.dev.bigbang.mil minio-api.dev.bigbang.mil alertmanager.dev.bigbang.mil grafana.dev.bigbang.mil prometheus.dev.bigbang.mil nexus.dev.bigbang.mil sonarqube.dev.bigbang.mil tempo.dev.bigbang.mil twistlock.dev.bigbang.mil >> /etc/hosts"
+ sudo bash -c "echo '## end dev.bigbang.mil section' >> /etc/hosts"
+ # run kubectl to add keycloak and vault's hostname/IP to the configmap for coredns, restart coredns
+ kubectl get configmap -n kube-system coredns -o yaml | sed '/^ .* host.k3d.internal$/a\ \ \ \ $PrivateIP2 keycloak.dev.bigbang.mil vault.dev.bigbang.mil' | kubectl apply -f -
+ kubectl delete pod -n kube-system -l k8s-app=kube-dns
+ENDSSH
+ fi
+}
-${action}
+function create_instances {
+ if [[ "${PublicIP}" == "" ]]; then
+ cloud_${CLOUDPROVIDER}_create_instances
+ fi
+ initialize_instance
+ install_docker
+ install_k3d
+ install_kubectl
+ install_metallb
+ fix_etc_hosts
+ if [[ "${QUIET}" == "false" ]]; then
+ print_instructions
+ fi
+}
+
+function main {
+ process_arguments "$@"
+
+ extratools=""
+ if [[ "${CLOUDPROVIDER}" != "" ]]; then
+ extratools=$(cloud_${CLOUDPROVIDER}_toolnames)
+ fi
+ check_missing_tools ${extratools}
+
+ # When -H is NOT provided, we assume we're responsible
+ # for provisioning the cloud infra.
+ if [[ "$PublicIP" == "" ]]; then
+ cloud_${CLOUDPROVIDER}_configure
+ else
+ CLOUDPROVIDER=""
+ fi
+
+ set_kubeconfig
+
+ ${action}
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+ main $@
+ exit $?
+fi
diff --git a/docs/assets/scripts/quickstart.sh b/docs/assets/scripts/quickstart.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5edb34769030303c0006cdfe42904115faa9d937
--- /dev/null
+++ b/docs/assets/scripts/quickstart.sh
@@ -0,0 +1,372 @@
+#!/bin/bash
+
+REGISTRY1_ENDPOINT=registry1.dso.mil
+BIG_BANG_REPO=""
+REGISTRY1_USERNAME="${REGISTRY1_USERNAME:-}"
+REGISTRY1_TOKEN="${REGISTRY1_TOKEN:-}"
+GITLAB_USERNAME=""
+REPO1_LOCATION="${REPO1_LOCATION:-}"
+KUBECONFIG="${KUBECONFIG:-}"
+BB_K3D_PUBLICIP=""
+BB_K3D_PRIVATEIP=""
+
+# Configuration variables sourced from the command line
+declare -g arg_host
+declare -g arg_privateip
+declare -g arg_username
+declare -g arg_keyfile
+declare -g arg_version
+declare -g arg_pipeline_templates_version=master
+declare -g arg_repolocation="${REPO1_LOCATION:-}"
+declare -g arg_registry1_username="${REGISTRY1_USERNAME:-}"
+declare -g arg_registry1_token="${REGISTRY1_TOKEN:-}"
+declare -g arg_cloud_provider=aws
+declare -g arg_metallb=false
+declare -g arg_provision=false
+declare -g arg_deploy=false
+declare -g arg_wait=false
+declare -g arg_destroy=false
+declare -g arg_recreate_k3d=false
+declare -g arg_recreate_cloud=false
+declare -a arg_argv
+
+function checkout_bigbang_repo {
+ if [[ ! -d ${BIG_BANG_REPO} ]]; then
+ mkdir -p ${BIG_BANG_REPO}
+ git clone https://repo1.dso.mil/big-bang/bigbang.git ${BIG_BANG_REPO}
+ cd ${BIG_BANG_REPO}
+ else
+ cd ${BIG_BANG_REPO}
+ fi
+ git fetch -a
+ if [[ "${arg_version}" == "latest" ]]; then
+ arg_version=$(git tag | sort -V | grep -v -- '-rc.' | tail -n 1)
+ fi
+ git checkout ${arg_version}
+}
+
+function checkout_pipeline_templates {
+ PIPELINE_REPO_LOCATION=${REPO1_LOCATION}/big-bang/pipeline-templates/pipeline-templates
+ if [[ ! -d ${PIPELINE_REPO_LOCATION} ]]; then
+ mkdir -p ${PIPELINE_REPO_LOCATION}
+ git clone https://repo1.dso.mil/big-bang/pipeline-templates/pipeline-templates.git ${PIPELINE_REPO_LOCATION}
+ cd ${PIPELINE_REPO_LOCATION}
+ else
+ cd ${PIPELINE_REPO_LOCATION}
+ git reset --hard
+ git clean -df
+ fi
+ git fetch -a
+ git checkout ${arg_pipeline_templates_version}
+}
+
+function destroy_k3d_cluster {
+ if [[ "${arg_cloud_provider}" != "" ]]; then
+ arg_cloud="-c ${arg_cloud_provider}"
+ fi
+
+ ${BIG_BANG_REPO}/docs/assets/scripts/developer/k3d-dev.sh \
+ -t quickstart \
+ ${arg_cloud} \
+ -d
+}
+
+function build_k3d_cluster {
+ args=""
+ if [[ "${arg_privateip}" != "" ]]; then
+ args="${args} -P ${arg_privateip}"
+ fi
+ if [[ "${arg_host}" != "" ]]; then
+ args="${args} -H ${arg_host}"
+ fi
+ if [[ "${arg_username}" != "" ]]; then
+ args="${args} -U ${arg_username}"
+ fi
+ if [[ "${arg_keyfile}" != "" ]]; then
+ args="${args} -k ${arg_keyfile}"
+ fi
+ if [[ "${arg_metallb}" == "true" ]]; then
+ args="${args} -m"
+ fi
+ if [[ "${arg_cloud_provider}" != "" ]]; then
+ args="${args} -c ${arg_cloud_provider}"
+ fi
+ if [[ "${arg_recreate_k3d}" == "true" ]]; then
+ args="${args} -K"
+ fi
+ if [[ "${arg_recreate_cloud}" == "true" ]]; then
+ args="${args} -R"
+ fi
+
+ ${BIG_BANG_REPO}/docs/assets/scripts/developer/k3d-dev.sh \
+ -t quickstart \
+ -T \
+ -q \
+ ${args} \
+ $@
+}
+
+function deploy_flux {
+ KUBECONFIG=${KUBECONFIG} ${REPO1_LOCATION}/big-bang/bigbang/scripts/install_flux.sh \
+ -u ${REGISTRY1_USERNAME} \
+ -p ${REGISTRY1_TOKEN} \
+ -w 900
+}
+
+function deploy_bigbang {
+ cd ${BIG_BANG_REPO} &&
+ helm upgrade -i bigbang \
+ ${BIG_BANG_REPO}/chart \
+ -n bigbang \
+ --create-namespace \
+ --set registryCredentials.username=${REGISTRY1_USERNAME} \
+ --set registryCredentials.password=${REGISTRY1_TOKEN} \
+ $@ \
+ -f ${BIG_BANG_REPO}/chart/ingress-certs.yaml \
+ -f ${BIG_BANG_REPO}/docs/assets/configs/example/dev-sso-values.yaml \
+ -f ${BIG_BANG_REPO}/docs/assets/configs/example/policy-overrides-k3d.yaml
+}
+
+function check_for_tools {
+ missing=0
+ for tool in jq yq kubectl helm git sed awk; do
+ if [[ ! -x $(which ${tool} 2>/dev/null) ]]; then
+ missing=1
+ echo "Required tool ${tool} missing, please fix and run again" >&2
+ fi
+ done
+ if [[ $missing -gt 0 ]]; then
+ exit 1
+ fi
+}
+
+function usage {
+ cat <<EOF
+quickstart.sh (C) 2025 : PlatformOne Big Bang team
+
+PlatformOne Big Bang quickstart : Quickly deploy a development bigbang cluster on a VM
+
+Optional Arguments:
+ -H,--host v : String. IP or Hostname of the VM to operate on
+ -P,--privateip v : String. If your VM has a separate private IP in addition to the public host, provide it here
+ -U,--username v : String. Username to use when SSHing into the target VM
+ -K,--keyfile v : String. SSH Key file to use when SSHing into the target VM
+ -V,--version v : String. Big Bang version to deploy (Default "latest")
+ -v,--pipeline-templates-version v : String. Version of the bigbang pipeline-templates to use (Default "master")
+ -R,--repolocation v : String. Location on your host filesystem where bigbang should be checked out (Default "${arg_repolocation}")
+ -u,--registry1-username v : String. Username for your account on registry1.dso.mil (Default "${arg_registry1_username}")
+ -c,--cloud-provider v : String. If using cloud provisioning, which cloud provider should be used (Default "aws")
+ -t,--registry1-token v : String. Access token for your account on registry1.dso.mil (Default "${arg_registry1_token}")
+ -m,--metallb : Boolean. Deploy a MetalLB on k3d
+ -p,--provision : Boolean. Provision the k3d cluster (implied)
+ -d,--deploy : Boolean. Deploy bigbang (implied)
+ -w,--wait : Boolean. Wait for bigbang (implied by --deploy)
+ -D,--destroy : Boolean. Destroy any previously created quickstart instance(s) created by this tool. (Disables -p, -d, -w)
+ -k,--recreate-k3d : Boolean. Recreate the K3D cluster on the instance from scratch. (implies -p)
+ -C,--recreate-cloud : Boolean. When provisioning with a cloud provider, destroy and rebuild the instance. (implies -p)
+
+EOF
+}
+
+function parse_arguments {
+ while [ -n "$1" ]; do # while loop starts
+
+ case "$1" in
+ "-h") ;&
+ "--help")
+ usage
+ exit 1
+ ;;
+ "-k") ;&
+ "--recreate-k3d")
+ arg_recreate_k3d=true
+ arg_provision=true
+ ;;
+ "-C") ;&
+ "--recreate-cloud")
+ arg_recreate_cloud=true
+ arg_provision=true
+ ;;
+ "-H") ;&
+ "--host")
+ shift
+ arg_host=$1
+ ;;
+ "-P") ;&
+ "--privateip")
+ shift
+ arg_privateip=$1
+ ;;
+ "-U") ;&
+ "--username")
+ shift
+ arg_username=$1
+ ;;
+ "-V") ;&
+ "--version")
+ shift
+ arg_version=$1
+ ;;
+ "-v") ;&
+ "--pipeline-templates-version")
+ shift
+ arg_pipeline_templates_version=$1
+ ;;
+ "-R") ;&
+ "--repolocation")
+ shift
+ arg_repolocation=$1
+ ;;
+ "-u") ;&
+ "--registry1-username")
+ shift
+ arg_registry1_username=$1
+ ;;
+ "-t") ;&
+ "--registry1-token")
+ shift
+ arg_registry1_token=$1
+ ;;
+ "-c") ;&
+ "--cloud-provider")
+ shift
+ arg_cloud_provider=$1
+ ;;
+ "-K") ;&
+ "--keyfile")
+ shift
+ arg_keyfile=${1}
+ ;;
+ "-m") ;&
+ "--metallb")
+ arg_metallb=true
+ ;;
+ "-p") ;&
+ "--provision")
+ arg_provision=true
+ ;;
+ "-d") ;&
+ "--deploy")
+ arg_deploy=true
+ ;;
+ "-w") ;&
+ "--wait")
+ arg_wait=true
+ ;;
+ "-D") ;&
+ "--destroy")
+ arg_destroy=true
+ ;;
+ "--")
+ shift
+ arg_argv=("${@}")
+ return
+ ;;
+ *)
+ echo "Option $1 not recognized"
+ exit 1
+ ;;
+
+ esac
+ shift
+ done
+}
+
+function main {
+ set -e
+ parse_arguments $@
+
+ if [[ ! -z "${arg_keyfile}" ]] && [[ ! -e ${arg_keyfile} ]] ; then
+ echo "SSH key file ${arg_keyfile} does not exist" >&2
+ exit 1
+ fi
+
+ actions="provision deploy wait"
+ user_actions=""
+ if [[ "${arg_provision}" == "true" ]]; then
+ user_actions="provision"
+ fi
+
+ if [[ "${arg_deploy}" == "true" ]]; then
+ user_actions="${user_actions} deploy"
+ # --deploy implies --wait
+ arg_wait="true"
+ fi
+
+ if [[ "${arg_wait}" == "true" ]]; then
+ user_actions="${user_actions} wait"
+ fi
+
+ if [[ "${arg_destroy}" == "true" ]]; then
+ user_actions="destroy"
+ fi
+
+ if [[ "$user_actions" != "" ]]; then
+ actions="$user_actions"
+ fi
+
+ export REPO1_LOCATION=${arg_repolocation}
+ export BIG_BANG_REPO=${REPO1_LOCATION}/big-bang/bigbang
+ export REGISTRY1_TOKEN=${arg_registry1_token}
+ export REGISTRY1_USERNAME=${arg_registry1_username}
+
+ checkout_bigbang_repo
+ checkout_pipeline_templates
+
+ if [[ "${actions}" =~ "destroy" ]]; then
+ destroy_k3d_cluster
+ return
+ elif [[ "${actions}" =~ "provision" ]]; then
+ build_k3d_cluster
+ fi
+
+ if [[ "${arg_host}" != "" ]]; then
+ export KUBECONFIG=~/.kube/${arg_host}-dev-quickstart-config
+ else
+ AWSUSERNAME=$(aws sts get-caller-identity --query Arn --output text | cut -f 2 -d '/')
+ export KUBECONFIG=~/.kube/${AWSUSERNAME}-dev-quickstart-config
+ instanceid=$(aws ec2 describe-instances \
+ --output text \
+ --query "Reservations[].Instances[].InstanceId" \
+ --filters "Name=tag:Name,Values=${AWSUSERNAME}-dev" "Name=tag:Project,Values=quickstart" "Name=instance-state-name,Values=running")
+ arg_host=$(aws ec2 describe-instances --output text --no-cli-pager --instance-id ${instanceid} --query "Reservations[].Instances[].PublicIpAddress")
+ arg_privateip=$(aws ec2 describe-instances --output json --no-cli-pager --instance-ids ${instanceid} | jq -r '.Reservations[0].Instances[0].PrivateIpAddress')
+ arg_keyfile="~/.ssh/${AWSUSERNAME}-dev-quickstart.pem"
+ arg_username="ubuntu"
+ fi
+
+ if [[ "${actions}" =~ "deploy" ]]; then
+ deploy_flux
+
+ deploy_bigbang ${arg_argv}
+ fi
+
+ if [[ "${actions}" =~ "wait" ]]; then
+ export PIPELINE_REPO_DESTINATION=${REPO1_LOCATION}/big-bang/pipeline-templates/pipeline-templates
+ export CI_VALUES_FILE=${BIG_BANG_REPO}/chart/values.yaml
+ export VALUES_FILE=${BIG_BANG_REPO}/chart/values.yaml
+ ${REPO1_LOCATION}/big-bang/pipeline-templates/pipeline-templates/scripts/deploy/03_wait_for_helmreleases.sh
+ fi
+
+ services=$(kubectl get virtualservices -A -o json 2>/dev/null | jq -r .items[].spec.hosts[0] | tr "\n" "\t")
+ echo "=================================================================================="
+ echo " INSTALLATION COMPLETE"
+ echo ""
+ echo "To access your kubernetes cluster via kubectl, export this variable in your shell:"
+ echo
+ echo " export KUBECONFIG=${KUBECONFIG}"
+ echo
+ echo "To access your kubernetes cluster in your browser, add this line to your hosts file:"
+ echo
+ echo " ${arg_host} ${services}"
+ echo
+ echo "To SSH to the instance running your cluster, use this command:"
+ echo
+ echo " ssh -i ${arg_keyfile} -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ${arg_username}@${arg_host}"
+ echo "=================================================================================="
+ set +e
+}
+
+check_for_tools
+
+main $@
diff --git a/docs/guides/deployment-scenarios/quickstart.md b/docs/guides/deployment-scenarios/quickstart.md
index db27250d66480c698cf231e2ee86d9769524cf4d..ecb6b80d9bd9bae29f3f7aef93181c69162b3fc6 100644
--- a/docs/guides/deployment-scenarios/quickstart.md
+++ b/docs/guides/deployment-scenarios/quickstart.md
@@ -2,748 +2,138 @@
[[_TOC_]]
-## Video Walkthrough
+## Big Bang in 1 hour
-A 36-minute speed run video walkthrough of this quickstart can be found on the following two mirrored locations:
-* [Google Drive - Video Mirror](https://drive.google.com/file/d/1m1pR0a-lrWr_Wed4EsI8-vimkYfb06GQ/view)
-* [Repo1 - Video Mirror](https://repo1.dso.mil/platform-one/bullhorn-delivery-static-assets/-/blob/master/big_bang/bigbang_quickstart.mp4)
+An SRE with a reasonable amount of experience operating in a command line environment, equipped with a fast internet connection and a workstation they can install software on, should be able to complete this process and have an operational Big Bang dev environment in 1 hour or less.
-## Overview
+### Satisfy the Prerequisites
-This quick start guide explains in beginner-friendly terminology how to complete the following tasks in under an hour:
+1. Ensure your workstation has a functional GNU environment with `git`. Mac OS and Linux should be good to go out of the box. For Windows, the **only** supported method for this guide is to install WSL and run a WSL bash terminal, following the rest of the guide as a Linux user inside WSL.
+1. Install [jq](https://jqlang.github.io/jq/download/).
+1. Install [yq](https://github.com/mikefarah/yq/#install). yq needs to be available in your system path PATH as `yq`, so we recommend not using a dockerized installation.
+1. Install kubectl. Follow the instructions for [windows](https://kubernetes.io/docs/tasks/tools/install-kubectl-windows/), [macos](https://kubernetes.io/docs/tasks/tools/install-kubectl-macos/) or [linux](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/). (If you are running on WSL in Windows, you should install kubectl using the package manager inside of WSL to install kubectl.)
+1. [Install helm](https://helm.sh/docs/intro/install/).
+1. [Install the Flux CLI](https://fluxcd.io/flux/installation/).
+1. Ensure you have bash version 4 installed. Linux and Windows with WSL users probably don't need to worry about this. For Mac OS users, install bash4 with homebrew or a similar package manager, as the bash that ships with Mac OS is hopelessly old. Mac OS users will use `/opt/homebrew/bin/bash` whenever `bash` is mentioned in this guide.
+1. Ensure you have an account on [PlatformOne RegistryOne](https://registry1.dso.mil). You will need your username and access token ("CLI Secret") for this process. (To retrieve your token, login to registry1, click your name in the top right, and copy the "CLI Secret" field.)
+1. If you do not plan to deploy the bigbang quickstart cluster onto your own VM, and want the quickstart script to provision an AWS VM for you, you need to install and configure [the AWS cli](https://aws.amazon.com/cli/) for your AWS account.
-1. Turn a Virtual Machine (VM) into a k3d single-node Kubernetes cluster.
-1. Deploy Big Bang on the cluster using a demonstration and local development-friendly workflow.
+### Download the Quickstart Script
- > **NOTE:** This guide mainly focuses on the scenario of deploying Big Bang to a remote VM with enough resources to run Big Bang [(refer to step 1 for recommended resources)](#step-1-provision-a-virtual-machine). If your workstation has sufficient resources, or you are willing to disable packages to lower the resource requirements, then local development is possible. This quick start guide is valid for both remote and local deployment scenarios.
+Run the following commands in your terminal to download the quickstart script, which you will use in the next step:
-1. Customize the demonstration deployment of Big Bang.
-
-## Important Security Notice
-
-All Developer and Quick Start Guides in this repo are intended to deploy environments for development, demonstration, and learning purposes. There are practices that are bad for security, but make perfect sense for these use cases: using of default values, minimal configuration, tinkering with new functionality that could introduce a security misconfiguration, and even purposefully using insecure passwords and disabling security measures like Kyverno for convenience. Many applications have default username and passwords combinations stored in the public git repo, these insecure default credentials and configurations are intended to be overridden during production deployments.
-
-When deploying a dev/demo environment there is a high chance of deploying Big Bang in an insecure configuration. Such deployments should be treated as if they could become easily compromised if made publicly accessible.
-
-### Recommended Security Guidelines for Dev/Demo Deployments
-
-* Ideally, these environments should be spun up on VMs with private IP addresses that are not publicly accessible. Local network access or an authenticated remote network access solution like a VPN or [sshuttle](https://github.com/sshuttle/sshuttle#readme) should be used to reach the private network.
-* DO NOT deploy publicly routable dev/demo clusters into shared VPCs (i.e., like a shared dev environment VPCs) or on VMs with IAM Roles attached. If the demo cluster were compromised, an adversary might be able to use it as a stepping stone to move deeper into an environment.
-* If you want to safely demo on Cloud Provider VMs with public IPs you must follow these guidelines:
- * Prevent Compromise:
- * Use firewalls that only allow the two VMs to talk to each other and your whitelisted IP.
- * Limit Blast Radius of Potential Compromise:
- * Only deploy to an isolated VPC, not a shared VPC.
- * Only deploy to VMs with no IAM roles/rights attached.
-
-## Network Requirements Notice
-
-This install guide by default requires network connectivity from your server to external DNS providers, specifically the Google DNS server at `8.8.8.8`, you can test that your node has connectivity to this DNS server by running the command `nslookup google.com 8.8.8.8` (run this from the node).
-
-If this command returns `DNS request timed out`, then you will need to follow the steps in [troubleshooting](#Troubleshooting) to change the upstream DNS server in your kubernetes cluster to your networks DNS server.
-
-Additionally, if your network has a proxy that has custom/internal SSL certificates then this may cause problems with pulling docker images as the image verification process can sometimes fail. Ensure you are aware of your network rules and restrictions before proceeding with the installation in order to understand potential problems when installing.
-
-## Important Background Contextual Information
-
-`BLUF:` This quick start guide optimizes the speed at which a demonstrable and tinker-able deployment of Big Bang can be achieved by minimizing prerequisite dependencies and substituting them with quickly implementable alternatives. Refer to the [Customer Template Repo](https://repo1.dso.mil/big-bang/customers/template) for guidance on production deployments.
-
-`Details of how each prerequisite/dependency is quickly satisfied:`
-
-* **Operating System Prerequisite:** Any Linux distribution that supports Docker should work.
-* **Operating System Pre-configuration:** This quick start includes easy paste-able commands to quickly satisfy this prerequisite.
-* **Kubernetes Cluster Prerequisite:** is implemented using k3d (k3s in Docker)
-* **Default Storage Class Prerequisite:** k3d ships with a local volume storage class.
-* **Support for automated provisioning of Kubernetes Service of type LB Prerequisite:** is implemented by taking advantage of k3d's ability to easily map port 443 of the VM to port 443 of a Dockerized LB that forwards traffic to a single Istio Ingress Gateway. Important limitations of this quick start guide's implementation of k3d to be aware of:
- * Multiple Ingress Gateways aren't supported by this implementation as they would each require their own LB, and this trick of using the host's port 443 only works for automated provisioning of a single service of type LB that leverages port 443.
- * Multiple Ingress Gateways makes a demoable/tinkerable KeyCloak and locally hosted SSO deployment much easier.
- * Multiple Ingress Gateways can be demoed on k3d if configuration tweaks are made, MetalLB is used, and you are developing using a local Linux Desktop. (network connectivity limitations of the implementation would only allow a the web browser on the k3d host server to see the webpages.)
- * If you want to easily demo and tinker with Multiple Ingress Gateways and Keycloak, then MetalLB + k3s (or another non-Dockerized Kubernetes distribution) would be a happy path to look into. (or alternatively create an issue ticket requesting prioritization of a keycloak quick start or better yet a Merge Request.)
-* Access to Container Images Prerequisite is satisfied by using personal image pull credentials and internet connectivity to <https://registry1.dso.mil>
-* Customer Controlled Private Git Repo Prerequisite isn't required due to substituting declarative git ops installation of the Big Bang Helm chart with an imperative helm cli based installation.
-* Encrypting Secrets as code Prerequisite is substituted with clear text secrets on your local machine.
-* Installing and Configuring Flux Prerequisite: Not using GitOps for the quick start eliminates the need to configure flux, and installation is covered within this guide.
-* HTTPS Certificate and hostname configuration Prerequisites: Are satisfied by leveraging default hostname values and the demo HTTPS wildcard certificate that's uploaded to the Big Bang repo, which is valid for *.bigbang.dev, *.admin.bigbang.dev, and a few others. The demo HTTPS wildcard certificate is signed by the Lets Encrypt Free, a Certificate Authority trusted on the public internet, so demo sites like grafana.bigbang.dev will show a trusted HTTPS certificate.
-* DNS Prerequisite: is substituted by making use of your workstation's Hosts file.
-
-## Step 1: Provision a Virtual Machine
-
-The following requirements are recommended for Demonstration Purposes:
-
-* 1 Virtual Machine with 32GB RAM, 8-Core CPU (t3a.2xlarge for AWS users), and 100GB of disk space should be sufficient.
-* Ubuntu Server 20.04 LTS (Ubuntu comes up slightly faster than CentOS, in reality any Linux distribution with Docker installed should work).
-* Most Cloud Service Provider provisioned VMs default to passwordless sudo being preconfigured, but if you're doing local development or a bare metal deployment then it's recommended that you configure passwordless sudo.
- * Steps for configuring passwordless sudo: [(source)](https://unix.stackexchange.com/questions/468416/setting-up-passwordless-sudo-on-linux-distributions)
- 1. `sudo visudo`
- 1. Change:
-
- ```plaintext
- # Allow members of group sudo to execute any command
- %sudo ALL=(ALL:ALL) ALL
- ```
-
- To:
-
- ```plaintext
- # Allow members of group sudo to execute any command, no password
- %sudo ALL=(ALL:ALL) NOPASSWD:ALL
- ```
-
-* Network connectivity to Virtual Machine (provisioning with a public IP and a security group locked down to your IP should work. Otherwise a Bare Metal server or even a Vagrant Box Virtual Machine configured for remote ssh works fine.)
-
-> **NOTE**: If your workstation has Docker, sufficient compute, and has ports 80, 443, and 6443 free, you can use your workstation in place of a remote virtual machine and do local development.
-
-## Step 2: SSH to Remote VM
-
-* ssh and passwordless sudo should be configured on the remote machine.
-* You can skip this step if you are doing local development.
-
-1. Set up SSH.
-
- ```shell
- # [admin@Unix_Laptop:~]
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- touch ~/.ssh/config
- chmod 600 ~/.ssh/config
- temp="""##########################
- Host k3d
- Hostname x.x.x.x #IP Address of k3d node
- IdentityFile ~/.ssh/bb-onboarding-attendees.ssh.privatekey #ssh key authorized to access k3d node
- User ubuntu
- StrictHostKeyChecking no #Useful for vagrant where you'd reuse IP from repeated tear downs
- #########################"""
- echo "$temp" | tee -a ~/.ssh/config #tee -a, appends to preexisting config file
- ```
-
-1. SSH to instance.
-
- ```shell
- # [admin@Laptop:~]
- ssh k3d
-
- # [ubuntu@Ubuntu_VM:~]
- ```
-
-## Step 3: Install Prerequisite Software
-
-**NOTE:** This guide follows the DevOps best practice of left-shifting feedback on mistakes and surfacing errors as early in the process as possible. This is done by leveraging tests and verification commands.
-
-1. Install Git.
-
- ```shell
- sudo apt install git -y
- ```
-
-1. Install Docker and add $USER to Docker group.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- sudo apt update -y && sudo apt install apt-transport-https ca-certificates curl gnupg lsb-release -y && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null && sudo apt update -y && sudo apt install docker-ce docker-ce-cli containerd.io -y && sudo usermod --append --groups docker $USER
-
-
- # Alternative command (less safe due to curl | bash, but more generic):
- # curl -fsSL https://get.docker.com | bash && sudo usermod --append --groups docker $USER
- ```
-
-1. Log out and login to allow the `usermod` change to take effect.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- exit
- ```
-
- ```shell
- # [admin@Laptop:~]
- ssh k3d
- ```
-
-1. Verify Docker Installation.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- docker run hello-world
- ```
-
- ```console
- Hello from Docker!
- ```
-
-1. Install k3d.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- # The following downloads the 64 bit linux version of k3d v5.5.1, checks it
- # against a copy of the sha256 checksum, if they match k3d gets installed
- wget -q -O - https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64 > k3d
-
- echo 4849027dc5e835bcce49070af3f4eeeaada81d96bce49a8b89904832a0c3c2c0 k3d | sha256sum -c | grep OK
- # 4849027dc5e835bcce49070af3f4eeeaada81d96bce49a8b89904832a0c3c2c0 came from running the following against a trusted internet connection.
- # wget -q -O - https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64 | sha256sum | cut -d ' ' -f 1
-
- if [ $? == 0 ]; then chmod +x k3d && sudo mv k3d /usr/local/bin/k3d; fi
-
-
- # Alternative command (less safe due to curl | bash, but more generic):
- # wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=v5.5.1 bash
- ```
-
-1. Verify k3d installation.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- k3d --version
- ```
-
- ```console
- k3d version v5.5.1
- k3s version v1.26.4-k3s1 (default)
- ```
-
-1. Install kubectl.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- # The following downloads the 64 bit linux version of kubectl v1.23.5, checks it
- # against a copy of the sha256 checksum, if they match kubectl gets installed
- wget -q -O - https://dl.k8s.io/release/v1.23.5/bin/linux/amd64/kubectl > kubectl
-
- echo 715da05c56aa4f8df09cb1f9d96a2aa2c33a1232f6fd195e3ffce6e98a50a879 kubectl | sha256sum -c | grep OK
- # 715da05c56aa4f8df09cb1f9d96a2aa2c33a1232f6fd195e3ffce6e98a50a879 came from
- # wget -q -O - https://dl.k8s.io/release/v1.23.5/bin/linux/amd64/kubectl.sha256
-
- if [ $? == 0 ]; then chmod +x kubectl && sudo mv kubectl /usr/local/bin/kubectl; fi
-
- # Create a symbolic link from k to kubectl
- sudo ln -s /usr/local/bin/kubectl /usr/local/bin/k
- ```
-
-1. Verify kubectl installation.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- kubectl version --client
- ```
-
- ```console
- Client Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.5", GitCommit:"c285e781331a3785a7f436042c65c5641ce8a9e9", GitTreeState:"clean", BuildDate:"2022-03-16T15:58:47Z", GoVersion:"go1.17.8", Compiler:"gc", Platform:"linux/amd64"}
- ```
-
-1. Install Kustomize.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- # The following downloads the 64 bit linux version of kustomize v4.5.4, checks it
- # against a copy of the sha256 checksum, if they match kustomize gets installed
- wget -q -O - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv4.5.4/kustomize_v4.5.4_linux_amd64.tar.gz > kustomize.tar.gz
-
- echo 1159c5c17c964257123b10e7d8864e9fe7f9a580d4124a388e746e4003added3 kustomize.tar.gz | sha256sum -c | grep OK
- # 1159c5c17c964257123b10e7d8864e9fe7f9a580d4124a388e746e4003added3
- # came from https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv4.5.4/checksums.txt
-
- if [ $? == 0 ]; then tar -xvf kustomize.tar.gz && chmod +x kustomize && sudo mv kustomize /usr/local/bin/kustomize && rm kustomize.tar.gz ; fi
-
-
- # Alternative commands (less safe due to curl | bash, but more generic):
- # curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
- # chmod +x kustomize
- # sudo mv kustomize /usr/bin/kustomize
- ```
-
-1. Verify Kustomize installation.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- kustomize version
- ```
-
- ```console
- {Version:kustomize/v4.5.4 GitCommit:cf3a452ddd6f83945d39d582243b8592ec627ae3 BuildDate:2022-03-28T23:12:45Z GoOs:linux GoArch:amd64}
- ```
-
-1. Install Helm.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- # The following downloads the 64 bit linux version of helm v3.8.1, checks it
- # against a copy of the sha256 checksum, if they match helm gets installed
- wget -q -O - https://get.helm.sh/helm-v3.13.3-linux-amd64.tar.gz > helm.tar.gz
-
- echo bbb6e7c6201458b235f335280f35493950dcd856825ddcfd1d3b40ae757d5c7d helm.tar.gz | sha256sum -c | grep OK
- # bbb6e7c6201458b235f335280f35493950dcd856825ddcfd1d3b40ae757d5c7d
- # came from https://github.com/helm/helm/releases/tag/v3.13.3
-
- if [ $? == 0 ]; then tar -xvf helm.tar.gz && chmod +x linux-amd64/helm && sudo mv linux-amd64/helm /usr/local/bin/helm && rm -rf linux-amd64 && rm helm.tar.gz ; fi
-
-
- # Alternative command (less safe due to curl | bash, but more generic):
- # curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
- ```
-
-1. Verify Helm installation.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- helm version
- ```
-
- ```console
- version.BuildInfo{Version:"v3.13.3", GitCommit:"c8b948945e52abba22ff885446a1486cb5fd3474", GitTreeState:"clean", GoVersion:"go1.20.11"}
- ```
-
-## Step 4: Configure Host Operating System Prerequisites
-
-* Run Operating System Pre-configuration
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- # Needed for ECK to run correctly without OOM errors
- echo 'vm.max_map_count=524288' | sudo tee -a /etc/sysctl.d/vm-max_map_count.conf
- # Alternatively can use (not persistent after restart):
- # sudo sysctl -w vm.max_map_count=524288
-
-
- # Needed by Sonarqube
- echo 'fs.file-max=131072' | sudo tee -a /etc/sysctl.d/fs-file-max.conf
- # Alternatively can use (not persistent after restart):
- # sudo sysctl -w fs.file-max=131072
-
- # Also Needed by Sonarqube
- ulimit -n 131072
- ulimit -u 8192
-
- # Load updated configuration
- sudo sysctl --load --system
-
- # Preload kernel modules, required by istio-init running on SELinux enforcing instances
- sudo modprobe xt_REDIRECT
- sudo modprobe xt_owner
- sudo modprobe xt_statistic
-
- # Persist kernel modules settings after reboots
- printf "xt_REDIRECT\nxt_owner\nxt_statistic\n" | sudo tee -a /etc/modules
-
- # Kubernetes requires swap disabled
- # Turn off all swap devices and files (won't last reboot)
- sudo swapoff -a
-
- # For swap to stay off, you can remove any references found via
- # cat /proc/swaps
- # cat /etc/fstab
- ```
-
-## Step 5: Create a k3d Cluster
-
-After reading the notes on the purpose of k3d's command flags, you will be able to copy and paste the command to create a k3d cluster.
-
-### Explanation of k3d Command Flags, Relevant to the Quick Start
-
-* `SERVER_IP="10.10.16.11"` and `--k3s-arg "--tls-san=$SERVER_IP@server:0"`:
- These associate an extra IP to the Kubernetes API server's generated HTTPS certificate.
-
- **Explanation of the effect:**
-
- 1. If you are running k3d from a local host or you plan to run 100% of kubectl commands while ssh'd into the k3d server, then you can omit these flags or paste unmodified incorrect values with no ill effect.
-
- 1. If you plan to run k3d on a remote server, but run kubectl, helm, and kustomize commands from a workstation, which would be needed if you wanted to do something like kubectl port-forward then you would need to specify the remote server's public or private IP address here. After pasting the ~/.kube/config file from the k3d server to your workstation, you will need to edit the IP inside of the file from 0.0.0.0 to the value you used for SERVER_IP.
-
- **Tips for looking up the value to plug into SERVER_IP:**
-
- * Method 1: If your k3d server is a remote box, then run the following command from your workstation.
- `cat ~/.ssh/config | grep k3d -A 6`
- * Method 2: If the remote server was provisioned with a Public IP, then run the following command from the server hosting k3d.
- `curl ifconfig.me --ipv4`
- * Method 3: If the server hosting k3d only has a Private IP, then run the following command from the server hosting k3d
- `ip address`
- (You will see more than one address, use the one in the same subnet as your workstation)
-
-* `--volume /etc/machine-id:/etc/machine-id`:
-This is required for fluentbit log shipper to work.
-
-* `IMAGE_CACHE=${HOME}/.k3d-container-image-cache`, `cd ~`, `mkdir -p ${IMAGE_CACHE}`, and `--volume ${IMAGE_CACHE}:/var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content`:
-These make it so that if you fully deploy Big Bang and then want to reset the cluster to a fresh state to retest some deployment logic. Then after running `k3d cluster delete k3s-default` and redeploying, subsequent deployments will be faster because all container images used will have been prefetched.
-
-* `--servers 1 --agents 3`:
-These flags are not used and shouldn't be added. This is because the image caching logic works more reliably on a one node Dockerized cluster, vs a four node Dockerized cluster. If you need to add these flags to simulate multi nodes to test pod and node affinity rules, then you should remove the image cache flags, or you may experience weird image pull errors.
-
-* `--port 80:80@loadbalancer` and `--port 443:443@loadbalancer`:
-These map the virtual machine's port 80 and 443 to port 80 and 443 of a Dockerized LB that will point to the NodePorts of the Dockerized k3s node.
-
-* `--k3s-arg "--disable=traefik@server:0"`:
-This flag prevents the traefik ingress controller from being deployed. Without this flag traefik would provision a service of type LoadBalancer, and claim k3d's only LoadBalancer that works with ports 80 and 443. Disabling this makes it so the Istio Ingress Gateway will be able to claim the service of type LoadBalancer.
+```
+export REGISTRY1_USERNAME=YOUR_REGISTRY1_USERNAME
+export REGISTRY1_TOKEN=YOUR_REGISTRY1_TOKEN
+export REPO1_LOCATION=LOCATION_ON_FILESYSTEM_TO_CHECK_OUT_BIGBANG_CODE
-### k3d Cluster Creation Commands
+curl --output quickstart.sh https://repo1.dso.mil/big-bang/bigbang/-/raw/master/docs/assets/scripts/quickstart.sh?ref_type=heads
+```
-```shell
-# [ubuntu@Ubuntu_VM:~]
-SERVER_IP="10.10.16.11" #(Change this value, if you need remote kubectl access)
+### Build the Cluster
-# Create image cache directory
-IMAGE_CACHE=${HOME}/.k3d-container-image-cache
+#### Using a VM or other hardware you built yourself
-mkdir -p ${IMAGE_CACHE}
+1. Spin up an Ubuntu VM somewhere with 8 CPUs and 32gB of RAM. Make sure you can SSH to it. It doesn't matter what cloud provider you're using, it can even be on your local system if you have enough horsepower for it.
+1. Run the following command in your command terminal:
-k3d cluster create \
- --k3s-arg "--tls-san=$SERVER_IP@server:0" \
- --volume /etc/machine-id:/etc/machine-id \
- --volume ${IMAGE_CACHE}:/var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content \
- --k3s-arg "--disable=traefik@server:0" \
- --port 80:80@loadbalancer \
- --port 443:443@loadbalancer \
- --api-port 6443
+```
+bash quickstart.sh
+ -H YOUR_VM_IP \
+ -U YOUR_VM_SSH_USERNAME \
+ -K YOUR_VM_SSH_KEY_FILE_PATH
```
-### k3d Cluster Verification Command
+#### Using Amazon Web Services
-```shell
-# [ubuntu@Ubuntu_VM:~]
-kubectl config use-context k3d-k3s-default
-kubectl get node
-```
+1. If your system is already configured to use AWS via the `aws-cli` and you don't want to go to the trouble of building your own VM, the quickstart can attempt to do it for you; simply run the quickstart with no arguments. Pay attention to the script output; the IP addresses of the created AWS EC2 instance will be printed after the cluster is built and before big bang is deployed. You may need these later.
+ 1. The quickstart is only so smart, and AWS environments can vary greatly. If the quickstart is not able to build an EC2 instance in AWS for you, please go build an EC2 instance suitable for your use case, then come back and follow the instructions for "Using a VM or other hardware you built yourself".
+1. Run the following commands in your command terminal:
-```console
-Switched to context "k3d-k3s-default".
-NAME STATUS ROLES AGE VERSION
-k3d-k3s-default-server-0 Ready control-plane,master 11m v1.22.7+k3s1
+```
+bash quickstart.sh
```
-## Step 6: Verify Your IronBank Image Pull Credentials
+### It's thinking
-1. Here we continue to follow the DevOps best practice of enabling early left-shifted feedback whenever possible; Before adding credentials to a configuration file and not finding out there is an issue until after we see an ImagePullBackOff error during deployment, we will do a quick left-shifted verification of the credentials.
+The process takes about 45 minutes. Make a sandwich, go for a walk, play with the dog. Check on it every 10 minutes or so. Once the command finishes, your cluster should be ready to use. Proceed to fix your DNS and access your big bang installation.
-1. Look up your IronBank image pull credentials.
+If it seems like it's taking too long, the script will tell you what it's currently waiting on. The most frequent cause of long delays is slow connection between your cluster and registry1. All container images are fetched from registry1, so if your cluster is running on a slow internet uplink, this process can take a long time the first time bigbang is deployed. If your helmreleases say `PodInitializing` or `ContainerInitializing` or `Init:2/3` for a long time, this is usually the cause. There's not much cure for this but patience. Try contemplating the nature of the universe, marvel at the fact that object oriented COBOL exists in `current year`, or peruse the Commander's reading list.
- 1. In a web browser go to [https://registry1.dso.mil](https://registry1.dso.mil).
- 1. Login via OIDC provider.
- 1. In the top right of the page, click your name, and then User Profile.
- 1. Your image pull username is labeled "Username."
- 1. Your image pull password is labeled "CLI secret."
+Eventually the bigbang release process will finish, and you'll see output like this:
- > **NOTE:** The image pull credentials are tied to the life cycle of an OIDC token which expires after ~3 days, so if 3 days have passed since your last login to IronBank, the credentials will stop working until you re-login to the [https://registry1.dso.mil](https://registry1.dso.mil) GUI.
+```
+==================================================================================
+ INSTALLATION COMPLETE
-1. Verify your credentials work.
+To access your kubernetes cluster via kubectl, export this variable in your shell:
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- # Turn off bash history
- set +o history
+ export KUBECONFIG=~/.kube/192.168.1.123-quickstart-dev-config
- export REGISTRY1_USERNAME=<REPLACE_ME>
- export REGISTRY1_PASSWORD=<REPLACE_ME>
- echo $REGISTRY1_PASSWORD | docker login registry1.dso.mil --username $REGISTRY1_USERNAME --password-stdin
+To access your kubernetes cluster in your browser, add this line to your hosts file:
- # Turn on bash history
- set -o history
- ```
+ 192.168.1.123 alertmanager.dev.bigbang.mil prometheus.dev.bigbang.mil grafana.dev.bigbang.mil
-## Step 7: Clone Your Desired Version of the Big Bang Umbrella Helm Chart
+To SSH to the instance running your cluster, use this command:
-```shell
-# [ubuntu@Ubuntu_VM:~]
-cd ~
-git clone https://repo1.dso.mil/big-bang/bigbang.git
-
-# Checkout version latest stable version of Big Bang
-cd ~/bigbang
-git checkout tags/$(grep 'tag:' base/gitrepository.yaml | awk '{print $2}')
-git status
-cd ~
-
-# Note you can do the following to checkout a specific version of bigbang
-# cd ~/bigbang
-# git checkout tags/1.30.1
+ ssh -i ~/.ssh/192.168.1.123-dev-quickstart.pem -o StrictHostKeyChecking=no -o IdentitiesOnly=yes ubuntu@192.168.1.123
+================================================================================
```
-```console
-HEAD detached at (latest version)
-```
+Congratulations, it's ready!
-> **NOTE:** HEAD is git speak for current context within a tree of commits.
+### What Just Happened? In Detail
-## Step 8: Install Flux
+The quickstart.sh script performs several actions:
-The `echo $REGISTRY1_USERNAME` is there to verify that the value of your environmental variable is still populated. If you switch terminals or re-login, you may need to reestablish these variables.
+1. Checks your system to make sure the prerequisites we talked about are present
+1. If you're an AWS Cloud user who didn't provide `-H`, `-K`, and `-U` settings, attempts to build an EC2 instance suitable for use as a Big Bang cluster inside the default VPC in your configured AWS account and region
+1. Connects to your system over SSH to perform several configuration steps, including:
+ 1. Enabling passwordless sudo
+ 1. Ensuring your system packages are up to date
+ 1. Installing k3d/k3s
+ 1. Configuring a single-node Kubernetes cluster on your VM using k3d
+1. Installs the flux kubernetes extensions on your k3d cluster
+1. Checks out the PlatformOne Big Bang repository to the location specified when you ran the command
+1. Installs the Big Bang umbrella chart into your k3d cluster
+1. Waits for Big Bang to completely deploy, which may take a significant amount of time
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- echo $REGISTRY1_USERNAME
- cd ~/bigbang
- $HOME/bigbang/scripts/install_flux.sh -u $REGISTRY1_USERNAME -p $REGISTRY1_PASSWORD
- # NOTE: After running this command the terminal may appear to be stuck on
- # "networkpolicy.networking.k8s.io/allow-webhooks created"
- # It's not stuck, the end of the .sh script has a kubectl wait command, give it 5 min
- # Also if you have slow internet/hardware you might see a false error message
- # error: timed out waiting for the condition on deployments/helm-controller
+### Fix DNS to access the services in your browser
- # As long as the following command shows STATUS Running you're good to move on
- kubectl get pods --namespace=flux-system
- ```
+You can now access your bigbang kubernetes cluster from the command line using `kubectl`, but you will need to perform one extra step to easily access bigbang services in your web browser. You will need to manually override some DNS settings to send specific website requests to your kubernetes cluster. This was included in the final message of the quickstart, but here are the instructions again.
- ```console
- NAME READY STATUS RESTARTS AGE
- helm-controller-746d586c6-ln7dl 1/1 Running 0 3m8s
- notification-controller-f6658d796-fdzjx 1/1 Running 0 3m8s
- kustomize-controller-5887bb8dd7-jzp7m 1/1 Running 0 3m8s
- source-controller-7c4564d74c-7ffrf 1/1 Running 0 3m8s
- ```
+**Remember to un-do this step when you are done experimenting with the bigbang quickstart.**
-## Step 9: Create Helm Values .yaml Files To Act as Input Variables for the Big Bang Helm Chart
+#### Linux/Mac Users
-> Note for those new to linux: The following are multi line copy pasteable commands to quickly generate config files from the CLI, make sure you copy from cat to EOF, if you get stuck in the terminal use ctrl + c
+Run this command in your terminal:
```shell
-# [ubuntu@Ubuntu_VM:~]
-cat << EOF > ~/ib_creds.yaml
-registryCredentials:
- registry: registry1.dso.mil
- username: "$REGISTRY1_USERNAME"
- password: "$REGISTRY1_PASSWORD"
-EOF
-
-
-cat << EOF > ~/demo_values.yaml
-elasticsearchKibana:
- values:
- kibana:
- count: 1
- resources:
- requests:
- cpu: 400m
- memory: 1Gi
- limits:
- cpu: null # nonexistent cpu limit results in faster spin up
- memory: null
- elasticsearch:
- master:
- count: 1
- resources:
- requests:
- cpu: 400m
- memory: 2Gi
- limits:
- cpu: null
- memory: null
- data:
- count: 1
- resources:
- requests:
- cpu: 400m
- memory: 2Gi
- limits:
- cpu: null
- memory: null
-
-clusterAuditor:
- values:
- resources:
- requests:
- cpu: 400m
- memory: 2Gi
- limits:
- cpu: null
- memory: null
-
-gatekeeper:
- enabled: false
- values:
- replicas: 1
- controllerManager:
- resources:
- requests:
- cpu: 100m
- memory: 512Mi
- limits:
- cpu: null
- memory: null
- audit:
- resources:
- requests:
- cpu: 400m
- memory: 768Mi
- limits:
- cpu: null
- memory: null
- violations:
- allowedDockerRegistries:
- enforcementAction: dryrun
-
-istio:
- values:
- values: # possible values found here https://istio.io/v1.5/docs/reference/config/installation-options (ignore 1.5, latest docs point here)
- global: # global istio operator values
- proxy: # mutating webhook injected istio sidecar proxy's values
- resources:
- requests:
- cpu: 0m # null get ignored if used here
- memory: 0Mi
- limits:
- cpu: 0m
- memory: 0Mi
-
-twistlock:
- enabled: false # twistlock requires a license to work, so we're disabling it
-
-# to set all Kyverno policies to audit only
-kyvernoPolicies:
- enabled: true
- values:
- validationFailureAction: "audit"
-
-# under Neuvector section
-neuvector:
- enabled: true
- values:
- k3s:
- enabled: true
-EOF
+echo YOUR_VM_IP $(kubectl get virtualservices -A -o json | jq -r .items[].spec.hosts[0] | tr "\n" "\t") | sudo tee -a /etc/hosts
```
-## Step 10: Install Big Bang Using the Local Development Workflow
-
-```shell
-# [ubuntu@Ubuntu_VM:~]
-helm upgrade --install bigbang $HOME/bigbang/chart \
- --values https://repo1.dso.mil/big-bang/bigbang/-/raw/master/chart/ingress-certs.yaml \
- --values $HOME/ib_creds.yaml \
- --values $HOME/demo_values.yaml \
- --namespace=bigbang --create-namespace
-```
+#### Windows Users
-Explanation of flags used in the imperative helm install command:
-
-`upgrade --install`:
-This makes the command more idempotent by allowing the exact same command to work for both the initial installation and upgrade use cases.
-
-`bigbang $HOME/bigbang/chart`:
-bigbang is the name of the helm release that you'd see if you run `helm list -n=bigbang`. `$HOME/bigbang/chart` is a reference to the helm chart being installed.
-
-`--values https://repo1.dso.mil/big-bang/bigbang/-/raw/master/chart/ingress-certs.yaml`:
-References demonstration HTTPS certificates embedded in the public repository. The *.bigbang.dev wildcard certificate is signed by Let's Encrypt, a free public internet Certificate Authority. Note the URL path to the copy of the cert on master branch is used instead of `$HOME/bigbang/chart/ingress-certs.yaml`, because the Let's Encrypt certs expire after 3 months, and if you deploy a tagged release of BigBang, like 1.15.0, the version of the cert stored in the tagged git commit/release of Big Bang could be expired. Referencing the master branches copy via URL ensures you receive the latest version of the cert, which won't be expired.
-
-`--namespace=bigbang --create-namespace`:
-Means it will install the bigbang helm chart in the bigbang namespace and create the namespace if it doesn't exist.
-
-## Step 11: Verify Big Bang Has Had Enough Time To Finish Installing
-
-* If you try to run the command in Step 11 too soon, you'll see an ignorable temporary error message.
-
- ```shell
- # [ubuntu@Ubuntu_VM:~]
- kubectl get virtualservices --all-namespaces
-
- # Note after running the above command, you may see an ignorable temporary error message
- # The error message may be different based on your timing, but could look like this:
- # error: the server doesn't have a resource type "virtualservices"
- # or
- # No resources found
-
- # The above errors could be seen if you run the command too early
- # Give Big Bang some time to finish installing, then run the following command to check it's status
-
- kubectl get po -A
- ```
-
-* If after running `kubectl get po -A` (which is the shorthand of `kubectl get pods --all-namespaces`) you see something like the following, then you need to wait longer.
-
- ```console
- NAMESPACE NAME READY STATUS RESTARTS AGE
- kube-system metrics-server-86cbb8457f-dqsl5 1/1 Running 0 39m
- kube-system coredns-7448499f4d-ct895 1/1 Running 0 39m
- flux-system notification-controller-65dffcb7-qpgj5 1/1 Running 0 32m
- flux-system kustomize-controller-d689c6688-6dd5n 1/1 Running 0 32m
- flux-system source-controller-5fdb69cc66-s9pvw 1/1 Running 0 32m
- kube-system local-path-provisioner-5ff76fc89d-gnvp4 1/1 Running 1 39m
- flux-system helm-controller-6c67b58f78-6dzqw 1/1 Running 0 32m
- gatekeeper-system gatekeeper-controller-manager-5cf7696bcf-xclc4 0/1 Running 0 4m6s
- gatekeeper-system gatekeeper-audit-79695c56b8-qgfbl 0/1 Running 0 4m6s
- istio-operator istio-operator-5f6cfb6d5b-hx7bs 1/1 Running 0 4m8s
- eck-operator elastic-operator-0 1/1 Running 1 4m10s
- istio-system istiod-65798dff85-9rx4z 1/1 Running 0 87s
- istio-system public-ingressgateway-6cc4dbcd65-fp9hv 0/1 ContainerCreating 0 46s
- logging logging-fluent-bit-dbkxx 0/2 Init:0/1 0 44s
- monitoring monitoring-monitoring-kube-admission-create-q5j2x 0/1 ContainerCreating 0 42s
- logging logging-ek-kb-564d7779d5-qjdxp 0/2 Init:0/2 0 41s
- logging logging-ek-es-data-0 0/2 Init:0/2 0 44s
- istio-system svclb-public-ingressgateway-ggkvx 5/5 Running 0 39s
- logging logging-ek-es-master-0 0/2 Init:0/2 0 37s
- ```
-
-* Wait up to 10 minutes then re-run `kubectl get po -A`, until all pods show STATUS Running.
-
-* `helm list -n=bigbang` should also show STATUS deployed
-
- ```console
- NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
- bigbang bigbang 1 2022-03-31 12:07:49.239343968 +0000 UTC deployed bigbang-1.30.1
- cluster-auditor-cluster-auditor cluster-auditor 1 2022-03-31 12:14:23.004377605 +0000 UTC deployed cluster-auditor-1.4.0-bb.0 0.0.4
- eck-operator-eck-operator eck-operator 1 2022-03-31 12:09:52.921098159 +0000 UTC deployed eck-operator-2.0.0-bb.0 2.0.0
- gatekeeper-system-gatekeeper gatekeeper-system 1 2022-03-31 12:07:53.52890717 +0000 UTC deployed gatekeeper-3.7.1-bb.0 v3.7.1
- istio-operator-istio-operator istio-operator 1 2022-03-31 12:07:55.111321595 +0000 UTC deployed istio-operator-1.13.2-bb.1 1.13.2
- istio-system-istio istio-system 1 2022-03-31 12:08:23.439981427 +0000 UTC deployed istio-1.13.2-bb.0 1.13.2
- jaeger-jaeger jaeger 1 2022-03-31 12:12:58.068313509 +0000 UTC deployed jaeger-operator-2.29.0-bb.0 1.32.0
- kiali-kiali kiali 1 2022-03-31 12:12:57.011215896 +0000 UTC deployed kiali-operator-1.47.0-bb.1 1.47.0
- logging-ek logging 1 2022-03-31 12:10:52.785810021 +0000 UTC deployed logging-0.7.0-bb.0 7.17.1
- logging-fluent-bit logging 1 2022-03-31 12:12:53.27612266 +0000 UTC deployed fluent-bit-0.19.20-bb.1 1.8.13
- monitoring-monitoring monitoring 1 2022-03-31 12:10:02.31254196 +0000 UTC deployed kube-prometheus-stack-33.2.0-bb.0 0.54.1
- ```
-
-## Step 12: Edit Your Workstation’s Hosts File To Access the Web Pages Hosted on the Big Bang Cluster
-
-Run the following command, which is the short hand equivalent of `kubectl get virtualservices --all-namespaces` to see a list of websites you'll need to add to your hosts file.
+Run this command in your bash terminal, and copy the output to your clipboard.
```shell
-kubectl get vs -A
-```
-
-```console
-NAMESPACE NAME GATEWAYS HOSTS AGE
-logging kibana ["istio-system/public"] ["kibana.bigbang.dev"] 10m
-monitoring monitoring-monitoring-kube-alertmanager ["istio-system/public"] ["alertmanager.bigbang.dev"] 10m
-monitoring monitoring-monitoring-kube-grafana ["istio-system/public"] ["grafana.bigbang.dev"] 10m
-monitoring monitoring-monitoring-kube-prometheus ["istio-system/public"] ["prometheus.bigbang.dev"] 10m
-kiali kiali ["istio-system/public"] ["kiali.bigbang.dev"] 8m21s
-jaeger jaeger ["istio-system/public"] ["tracing.bigbang.dev"] 7m46s
+echo YOUR_VM_IP $(kubectl get virtualservices -A -o json | jq -r .items[].spec.hosts[0] | tr "\n" "\t")
```
-### Linux/Mac Users
-
-```shell
-# [admin@Laptop:~]
-sudo vi /etc/hosts
-```
-
-### Windows Users
-
1. Right click Notepad -> Run as Administrator
1. Open C:\Windows\System32\drivers\etc\hosts
+1. Add the line from your clipboard to the bottom of the file
+1. Save and close
-### Linux/Mac/Windows Users
-
-Add the following entries to the Hosts file, where x.x.x.x = k3d virtual machine's IP.
-
-> Hint: find and replace is your friend
+### Access a BigBang Service
-```plaintext
-x.x.x.x kibana.bigbang.dev
-x.x.x.x alertmanager.bigbang.dev
-x.x.x.x grafana.bigbang.dev
-x.x.x.x prometheus.bigbang.dev
-x.x.x.x kiali.bigbang.dev
-x.x.x.x tracing.bigbang.dev
-x.x.x.x argocd.bigbang.dev
-```
-
-## Step 13: Visit a Webpage
-
-In a browser, visit one of the sites listed using the `kubectl get vs -A` command.
+In a browser, visit one of the sites that you just added to your hosts file.
Note, default credentials for Big Bang packages can be found [here](../using-bigbang/default-credentials.md).
-## Step 14: Play
+### Tinker With It
-Here's an example of post deployment customization of Big Bang.
-After looking at <https://repo1.dso.mil/big-bang/bigbang/-/blob/master/chart/values.yaml>
-It should make sense that the following is a valid edit.
+You can use the quickstart script to update your bigbang quickstart deployment with your own modifications. Here's an example of post deployment customization of Big Bang. After looking at <https://repo1.dso.mil/big-bang/bigbang/-/blob/master/chart/values.yaml>, you can see that this will enable the ArgoCD addon, which is not enabled by default.
```shell
# [ubuntu@Ubuntu_VM:~]
@@ -753,26 +143,39 @@ addons:
argocd:
enabled: true
EOF
+```
-helm upgrade --install bigbang $HOME/bigbang/chart \
---values https://repo1.dso.mil/big-bang/bigbang/-/raw/master/chart/ingress-certs.yaml \
---values $HOME/ib_creds.yaml \
---values $HOME/demo_values.yaml \
---values $HOME/tinkering.yaml \
---namespace=bigbang --create-namespace
+Now we redeploy using the same command as before, but adding some additional options to point to the tinkering.yaml file we just created.
-# NOTE: There may be a ~1 minute delay for the change to apply
+If you're deploying your own infrastructure:
-kubectl get vs -A
-# Now ArgoCD should show up, if it doesn't wait a minute and rerun the command
+```
+bash quickstart.sh
+ -H YOUR_VM_IP \
+ -U YOUR_VM_SSH_USERNAME \
+ -K YOUR_VM_SSH_KEY_FILE_PATH \
+ --deploy \
+ -- -f ${HOME}/tinkering.yaml
+```
+
+If you're using the script to provision your own infrastructure on AWS:
+
+```
+bash quickstart.sh --deploy -- -f ${HOME}/tinkering.yaml
+```
+
+You will see the same kind of output as before, with the big bang release being updated, and the script waiting for all releases to be present.
+
+Now you should be able to see the ArgoCD virtual service:
+```
+kubectl get vs -A
kubectl get po -n=argocd
-# Once these are all Running you can visit argocd's webpage
```
-> Remember to un-edit your Hosts file when you are finished tinkering.
+... And you can update your hosts file from the previous step to include the ArgoCD address from the virtualservice, and view it in your browser.
-## Step 15: Implementing Mission Applications within your bigbang environment
+### Implementing Mission Applications within your bigbang environment
Big Bang by itself serves as a jumping off point, but many users will want to implement their own mission specific applications in to the cluster. BigBang has implemented a `packages:` and `wrapper:` section to enable and support this in a way that ensures connectivity between your mission specific requirements and existing BigBang utilities, such as istio, the monitoring stack, and network policy management. [Here](https://repo1.dso.mil/big-bang/bigbang/-/blob/master/docs/guides/deployment-scenarios/extra-package-deployment.md) is the documentation for the `packages` utility.
@@ -865,28 +268,127 @@ packages:
color: "#fcba03" #yellow
EOF
+```
+
+If you're deploying on your own infrastructure:
+
+```
+bash quickstart.sh
+ -H YOUR_VM_IP \
+ -U YOUR_VM_SSH_USERNAME \
+ -K YOUR_VM_SSH_KEY_FILE_PATH \
+ --deploy \
+ -- -f $HOME/podinfo_wrapper.yaml
+```
-helm upgrade --install bigbang $HOME/bigbang/chart \
---values https://repo1.dso.mil/big-bang/bigbang/-/raw/master/chart/ingress-certs.yaml \
---values $HOME/ib_creds.yaml \
---values $HOME/demo_values.yaml \
---values $HOME/podinfo_wrapper.yaml \
---namespace=bigbang --create-namespace
+If you're using the script to provision your own infrastructure on AWS:
-# NOTE: There may be a ~1 minute delay for the change to apply
+```
+bash quickstart.sh --deploy -- -f $HOME/podinfo_wrapper.yaml
+```
+
+Now missionapp should show up, if it doesn't wait a minute and rerun the command
+```
kubectl get vs -A
-# Now missionapp should show up, if it doesn't wait a minute and rerun the command
kubectl get po -n=missionapp
-# Once these are all Running you can visit missionapp's webpage
+# Once these are all Running you can visit missionapp's webpage after editing DNS to include the new host
```
Wrappers also allow you to abstract out Monitoring, Secrets, Network Policies, and ConfigMaps. Additional Configuration information can be found [here](./extra-package-deployment.md)
+
+## Important Security Notice
+
+All Developer and Quick Start Guides in this repo are intended to deploy environments for development, demonstration, and learning purposes. There are practices that are bad for security, but make perfect sense for these use cases: using of default values, minimal configuration, tinkering with new functionality that could introduce a security misconfiguration, and even purposefully using insecure passwords and disabling security measures like Kyverno for convenience. Many applications have default username and passwords combinations stored in the public git repo, these insecure default credentials and configurations are intended to be overridden during production deployments.
+
+When deploying a dev/demo environment there is a high chance of deploying Big Bang in an insecure configuration. Such deployments should be treated as if they could become easily compromised if made publicly accessible.
+
+### Recommended Security Guidelines for Dev/Demo Deployments
+
+* Ideally, these environments should be spun up on VMs with private IP addresses that are not publicly accessible. Local network access or an authenticated remote network access solution like a VPN or [sshuttle](https://github.com/sshuttle/sshuttle#readme) should be used to reach the private network.
+* DO NOT deploy publicly routable dev/demo clusters into shared VPCs (i.e., like a shared dev environment VPCs) or on VMs with IAM Roles attached. If the demo cluster were compromised, an adversary might be able to use it as a stepping stone to move deeper into an environment.
+* If you want to safely demo on Cloud Provider VMs with public IPs you must follow these guidelines:
+ * Prevent Compromise:
+ * Use firewalls that only allow the two VMs to talk to each other and your whitelisted IP.
+ * Limit Blast Radius of Potential Compromise:
+ * Only deploy to an isolated VPC, not a shared VPC.
+ * Only deploy to VMs with no IAM roles/rights attached.
+
+## Network Requirements Notice
+
+This install guide by default requires network connectivity from your server to external DNS providers, specifically the Google DNS server at `8.8.8.8`, you can test that your node has connectivity to this DNS server by running the command `nslookup google.com 8.8.8.8` (run this from the node).
+
+If this command returns `DNS request timed out`, then you will need to follow the steps in [troubleshooting](#Troubleshooting) to change the upstream DNS server in your kubernetes cluster to your networks DNS server.
+
+Additionally, if your network has a proxy that has custom/internal SSL certificates then this may cause problems with pulling docker images as the image verification process can sometimes fail. Ensure you are aware of your network rules and restrictions before proceeding with the installation in order to understand potential problems when installing.
+
+## Important Background Contextual Information
+
+`BLUF:` This quick start guide optimizes the speed at which a demonstrable and tinker-able deployment of Big Bang can be achieved by minimizing prerequisite dependencies and substituting them with quickly implementable alternatives. Refer to the [Customer Template Repo](https://repo1.dso.mil/big-bang/customers/template) for guidance on production deployments.
+
+`Details of how each prerequisite/dependency is quickly satisfied:`
+
+* **Operating System Prerequisite:** Ubuntu is presumed by the guide and all supporting scripts. Any linux distribution that supports Docker can be made to run k3d or kubernetes, but this guide presumes Ubuntu for the sake of efficiency.
+* **Operating System Pre-configuration:** This quick start includes easy paste-able commands to quickly satisfy this prerequisite.
+* **Kubernetes Cluster Prerequisite:** is implemented using k3d (k3s in Docker)
+* **Default Storage Class Prerequisite:** k3d ships with a local volume storage class.
+* **Support for automated provisioning of Kubernetes Service of type LB Prerequisite:** is implemented by taking advantage of k3d's ability to easily map port 443 of the VM to port 443 of a Dockerized LB that forwards traffic to a single Istio Ingress Gateway. Important limitations of this quick start guide's implementation of k3d to be aware of:
+ * Multiple Ingress Gateways aren't supported by this implementation as they would each require their own LB, and this trick of using the host's port 443 only works for automated provisioning of a single service of type LB that leverages port 443.
+ * Multiple Ingress Gateways makes a demoable/tinkerable KeyCloak and locally hosted SSO deployment much easier.
+ * Multiple Ingress Gateways can be demoed on k3d if configuration tweaks are made, MetalLB is used, and you are developing using a local Linux Desktop. (network connectivity limitations of the implementation would only allow a the web browser on the k3d host server to see the webpages.)
+ * If you want to easily demo and tinker with Multiple Ingress Gateways and Keycloak, then MetalLB + k3s (or another non-Dockerized Kubernetes distribution) would be a happy path to look into. (or alternatively create an issue ticket requesting prioritization of a keycloak quick start or better yet a Merge Request.)
+* Access to Container Images Prerequisite is satisfied by using personal image pull credentials and internet connectivity to <https://registry1.dso.mil>
+* Customer Controlled Private Git Repo Prerequisite isn't required due to substituting declarative git ops installation of the Big Bang Helm chart with an imperative helm cli based installation.
+* Encrypting Secrets as code Prerequisite is substituted with clear text secrets on your local machine.
+* Installing and Configuring Flux Prerequisite: Not using GitOps for the quick start eliminates the need to configure flux, and installation is covered within this guide.
+* HTTPS Certificate and hostname configuration Prerequisites: Are satisfied by leveraging default hostname values and the demo HTTPS wildcard certificate that's uploaded to the Big Bang repo, which is valid for *.bigbang.dev, *.admin.bigbang.dev, and a few others. The demo HTTPS wildcard certificate is signed by the Lets Encrypt Free, a Certificate Authority trusted on the public internet, so demo sites like grafana.bigbang.dev will show a trusted HTTPS certificate.
+* DNS Prerequisite: is substituted by making use of your workstation's Hosts file.
+
## Troubleshooting
This section will provide guidance for troubleshooting problems that may occur during your Big Bang installation and instructions for additional configuration changes that may be required in restricted networks.
+### Connection Failures and Read API Failures
+
+Sometimes you will see Read API failures and connection failures immediately after the k3d cluster has been configured, when the flux extensions are being deployed to the cluster. If this happens, wait a few minutes, and re-try the deployment step. Run the same command you just ran, but add the `--deploy` option to it.
+
+If you're deploying your own infrastructure:
+
+```
+bash quickstart.sh
+ -H YOUR_VM_IP \
+ -U YOUR_VM_SSH_USERNAME \
+ -K YOUR_VM_SSH_KEY_FILE_PATH \
+ --deploy
+```
+
+If you're using the script to provision your own infrastructure on AWS:
+
+```
+bash quickstart.sh --deploy
+```
+
+### Timed Out
+
+If you see errors that talk about something timing out while waiting for conditions, re-run your `quickstart.sh` command again, but add the `--wait` flag to it. This will cause the script to resume after the deployment step and continue to wait for the release to become healthy.
+
+If you're deploying your own infrastructure:
+
+```
+bash quickstart.sh
+ -H YOUR_VM_IP \
+ -U YOUR_VM_SSH_USERNAME \
+ -K YOUR_VM_SSH_KEY_FILE_PATH \
+ --wait
+```
+
+If you're using the script to provision your own infrastructure on AWS:
+
+```
+bash quickstart.sh --wait
+```
+
### Changing CoreDNS upstream DNS server:
After completing step 5, if you are unable to connect to external DNS providers using the command `nslookup google.com 8.8.8.8`, to test the connection. Then use the steps below to change the upstream DNS server to your networks DNS server. Please note that this change will not perist after a restart of the host server therefore, if you restart or shutdown your server you will need to re-apply these changes to CoreDNS.
@@ -962,89 +464,17 @@ sudo sysctl fs.inotify.max_user_watches=501208
```
### Failed to provide IP to istio-system/public-ingressgateway
-As one option to provide IP to the istio-system/public-ingressgateway, metallb can be run. The following steps will demonstrate a standard configuration, however, some changes may need to be made for each individual system (e.g., specific /ets/hosts addresses).
-
-#### Step 1: K3d Deploy
+As one option to provide IP to the istio-system/public-ingressgateway, metallb can be run. The following steps will demonstrate a standard configuration.
-To facilitate metallb, servicelb needs to be disabled on the initial install. Replace the above k3d deploy command with the following:
-```shell
-k3d cluster create \
- --k3s-arg "--tls-san=$SERVER_IP@server:0" \
- --volume /etc/machine-id:/etc/machine-id \
- --volume ${IMAGE_CACHE}:/var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content \
- --k3s-arg "--disable=traefik@server:0" \
- --k3s-arg "--disable=servicelb@server:0" \
- --port 80:80@loadbalancer \
- --port 443:443@loadbalancer \
- --api-port 6443
```
-
-#### Step 2: Deploy MetalLB
-
-After following the above instructions to deploy flux, deploy the metallb controller and speaker.
-```shell
-kubectl create -f https://raw.githubusercontent.com/metallb/metallb/v0.13.9/config/manifests/metallb-native.yaml
-```
-Wait for the pods to be running:
-```shell
-kubectl get po -n metallb-system
-```
-
-```console
-NAME READY STATUS RESTARTS AGE
-controller-5684477f66-s99jg 1/1 Running 0 30s
-speaker-jrddv 1/1 Running 0 30s
+bash quickstart.sh
+ -H YOUR_VM_IP \
+ -U YOUR_VM_SSH_USERNAME \
+ -K YOUR_VM_SSH_KEY_FILE_PATH \
+ -R LOCATION_ON_FILESYSTEM_TO_CHECK_OUT_BIGBANG_CODE \
+ -m
```
-#### Step 3: Configure MetalLB
-
-**NOTE:** This step will not work if either the controller or speaker are not in a running condition.
-
-The following configuration addresses will need to be filled with the values that match your configuration. These can typically be found by looking at your docker subnet using the 'docker network ls' command. If there is no subnet currently configured you can use the following as an example to set up your subnet. 'docker network create --opt com.docker.network.bridge.name=$NETWORK_NAME $NETWORK_NAME --driver=bridge -o "com.docker.network.driver.mtu"="1450" --subnet=172.x.x.x/16 --gateway 172.x.x.x'. Be sure to replace the network name, subnet and gateway values as needed.
-
-```shell
-export SUBNET_RANGE=172.x.x.x-172.x.x.x
-cat << EOF > ~/metallb-config.yaml
-apiVersion: metallb.io/v1beta1
-kind: IPAddressPool
-metadata:
- name: default
- namespace: metallb-system
-spec:
- addresses:
- - $SUBNET_RANGE
----
-apiVersion: metallb.io/v1beta1
-kind: L2Advertisement
-metadata:
- name: l2advertisement1
- namespace: metallb-system
-spec:
- ipAddressPools:
- - default
-EOF
-
-kubectl create -f $HOME/metallb-config.yaml
-```
-#### Step 4: Configure /etc/hosts
-
-Lastly, configure /etc/hosts/ with the new IP Addresses (**NOTE:** you can add your own as needed for services). You will need to fill in the values used for the subnet.
-
-```shell
- export PASSTHROUGH_GATEWAY_IP=172.x.x.x
- export PUBLIC_GATEWAY_IP=172.x.x.x
- sudo sed -i '/bigbang.dev/d' /etc/hosts
- sudo bash -c "echo '## begin bigbang.dev section (METAL_LB)' >> /etc/hosts"
- sudo bash -c "echo $PASSTHROUGH_GATEWAY_IP keycloak.bigbang.dev vault.bigbang.dev >> /etc/hosts"
- sudo bash -c "echo $PUBLIC_GATEWAY_IP anchore-api.bigbang.dev anchore.bigbang.dev argocd.bigbang.dev gitlab.bigbang.dev registry.bigbang.dev tracing.bigbang.dev kiali.bigbang.dev kibana.bigbang.dev chat.bigbang.dev minio.bigbang.dev minio-api.bigbang.dev alertmanager.bigbang.dev grafana.bigbang.dev prometheus.bigbang.dev nexus.bigbang.dev sonarqube.bigbang.dev tempo.bigbang.dev twistlock.bigbang.dev >> /etc/hosts"
- sudo bash -c "echo '## end bigbang.dev section' >> /etc/hosts"
- # run kubectl to add keycloak and vault's hostname/IP to the configmap for coredns, restart coredns
- kubectl get configmap -n kube-system coredns -o yaml | sed '/^ $PASSTHROUGH_GATEWAY_IP host.k3d.internal$/a\ \ \ \ $PASSTHROUGH_GATEWAY_IP keycloak.bigbang.dev vault.bigbang.dev' | kubectl apply -f -
- kubectl delete pod -n kube-system -l k8s-app=kube-dns
-```
-
-From this point continue with the helm upgrade command above.
-
### WSL2
This section will provide guidance for troubleshooting problems that may occur during your Big Bang installation specifically involving WSL2.