package with release

.gitlab-ci.yml

@@ -22,6 +22,14 @@ stages:
   - bigbang down
   - cluster down
   - network down
+  - package
+  - release
+
+variables:
+  RELEASE_BUCKET: umbrella-bigbang-releases
+  IMAGE_LIST: images.txt
+  IMAGE_PKG: images.tar.gz
+  REPOS_PKG: repositories.tar.gz
 
 #-----------------------------------------------------------------------------------------------------------------------
 # Pre Stage Jobs
@@ -86,6 +94,11 @@ clean install:
   script:
     - *deploy_bigbang
     - *test_bigbang
+    - docker exec -i k3d-${CLUSTER_NAME}-server-0 crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)' > images.txt
+  artifacts:
+    paths:
+      - images.txt
+    expire_in: 7 days
 
 upgrade:
   stage: smoke tests
@@ -110,6 +123,62 @@ upgrade:
     - *deploy_bigbang
     - *test_bigbang
 
+package:
+  stage: package
+  image: registry.dsop.io/platform-one/big-bang/umbrella/synker:0.0.1
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_TAG'
+    - when: never
+  dependencies:
+    - clean install
+  before_script:
+    # Set up auth
+    - mkdir -p /root/.docker
+    - echo $DOCKER_AUTH_CONFIG > /root/.docker/config.json
+  script:
+    - cp ./scripts/package/synker.yaml ./synker.yaml
+
+    # Populate images list in synker config
+    # NOTE: This is stupid and hacky, use yq or something better to do this
+    - cat images.txt | sed -e 's/^/    - /' >> synker.yaml
+
+    - synker pull -b=1
+
+    # Tar up synker as well?
+    - cp /usr/local/bin/synker /var/lib/registry/synker
+    - tar -C /var/lib/registry -czvf $IMAGE_PKG .
+    - tar -czvf $IMAGE_PKG /var/lib/registry
+
+    # Package dependent repos
+    - ./scripts/package/gits.sh
+    - tar -czf $REPOS_PKG repos/
+
+    # Prep release
+    - mkdir -p release
+    - mv $IMAGE_LIST $IMAGE_PKG $REPOS_PKG release/
+
+    # Publish packages to s3 release
+    - aws s3 sync --quiet --dryrun release/ s3://umbrella-bigbang-releases/umbrella/${CI_COMMIT_TAG}
+  after_script: []
+
+release:
+  stage: release
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_TAG'
+    - when: never
+  variables:
+    RELEASE_ENDPOINT: https://${RELEASE_BUCKET}.s3-${AWS_DEFAULT_REGION}.amazonaws.com/umbrella/${CI_COMMIT_TAG}
+  script:
+    - |
+      release-cli create --name "Big Bang ${CI_COMMIT_TAG}" --tag-name ${CI_COMMIT_TAG} \
+        --description "Automated release notes are a WIP." \
+        --assets-link "{\"name\":\"${IMAGE_LIST}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_LIST}\"}" \
+        --assets-link "{\"name\":\"${IMAGE_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_PKG}\"}" \
+        --assets-link "{\"name\":\"${REPOS_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${REPOS_PKG}\"}"
+
 #-----------------------------------------------------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------------------------------------------------

scripts/package/gits.sh

+#!/bin/bash
+set -ex
+
+mkdir -p repos/
+
+# Clone core
+yq r "chart/values.yaml" "*.git.repo" | while IFS= read -r repo; do
+    git -C repos/ clone --no-checkout $repo
+done
+
+# Clone packages
+yq r "chart/values.yaml" "addons.*.git.repo" | while IFS= read -r repo; do
+    git -C repos/ clone --no-checkout $repo
+done

scripts/package/synker.Dockerfile

+FROM golang:1.13 AS builder
+
+# Download build dependencies
+RUN apt-get update && apt-get install -y \
+    git libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev liblvm2-dev musl-dev \
+    && apt-get clean
+
+# Clone the latest release of p8kr and built the binrary statically
+RUN git clone https://repo1.dsop.io/platform-one/hagrid/sync.git synker && \
+    cd synker && \
+    make binary-local-static DISABLE_CGO=1
+
+#
+FROM registry.access.redhat.com/ubi8/ubi:8.3
+
+COPY --from=registry:2 /bin/registry /usr/local/bin/registry
+COPY --from=builder /go/synker/synker /usr/local/bin/synker
+
+RUN yum install -y unzip git jq
+
+# Install yq
+RUN curl -sfL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 && \
+    chmod +x /usr/local/bin/yq
+
+# Install aws cli
+RUN curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscliv2.zip && \
+    unzip -qq -d /tmp /tmp/awscliv2.zip && \
+    /tmp/aws/install && \
+    rm -rf /tmp/aws*
+
+RUN yum clean all && \
+    rm -r /var/cache/dnf
+

scripts/package/synker.yaml

+# Baseline synker CI configs
+# NOTE: THIS CONFIG IS PARTIAL (it is missing the `destination` component) AND ONLY USED FOR PULLING IN CI
+transport:
+  registry:
+    hostname: localhost
+    port: 25000
+    osChoice: linux
+    disablePolicyChecks: true
+
+options:
+  # TODO: Turn this off for now until parallel syncing is more "production" ready
+  # NOTE: Finding some bugs (even with retry) in pushing images to the airgapped registry that trace back to paralell syncing
+  parallelSync: false
+  quiet: false
+  debug: info
+
+source:
+  authFile: /root/.docker/config.json
+  images:
+    - registry.dsop.io/platform-one/big-bang/apps/security-tools/twistlock/defender:20.04.163

tests/ci/k3d/values.yaml

@@ -39,11 +39,6 @@ clusterAuditor:
 
 monitoring:
   enabled: true
-#  values:
-#    prometheusOperator:
-#      admissionWebhooks:
-#        # Disable admission webhook creation for CI to speed things up
-#        enabled: false
 
 gatekeeper:
   enabled: true