diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c4615091e2d06b4f568b010d2a9f023490f8e7e..4b0b28747cd523652e016287bcc7a95120d8ef7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,6 +22,14 @@ stages: - bigbang down - cluster down - network down + - package + - release + +variables: + RELEASE_BUCKET: umbrella-bigbang-releases + IMAGE_LIST: images.txt + IMAGE_PKG: images.tar.gz + REPOS_PKG: repositories.tar.gz #----------------------------------------------------------------------------------------------------------------------- # Pre Stage Jobs @@ -86,6 +94,11 @@ clean install: script: - *deploy_bigbang - *test_bigbang + - docker exec -i k3d-${CLUSTER_NAME}-server-0 crictl images -o json | jq -r '.images[].repoTags[0] | select(. != null)' > images.txt + artifacts: + paths: + - images.txt + expire_in: 7 days upgrade: stage: smoke tests @@ -110,6 +123,62 @@ upgrade: - *deploy_bigbang - *test_bigbang +package: + stage: package + image: registry.dsop.io/platform-one/big-bang/umbrella/synker:0.0.1 + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_COMMIT_TAG' + - when: never + dependencies: + - clean install + before_script: + # Set up auth + - mkdir -p /root/.docker + - echo $DOCKER_AUTH_CONFIG > /root/.docker/config.json + script: + - cp ./scripts/package/synker.yaml ./synker.yaml + + # Populate images list in synker config + # NOTE: This is stupid and hacky, use yq or something better to do this + - cat images.txt | sed -e 's/^/ - /' >> synker.yaml + + - synker pull -b=1 + + # Tar up synker as well? + - cp /usr/local/bin/synker /var/lib/registry/synker + - tar -C /var/lib/registry -czvf $IMAGE_PKG . + - tar -czvf $IMAGE_PKG /var/lib/registry + + # Package dependent repos + - ./scripts/package/gits.sh + - tar -czf $REPOS_PKG repos/ + + # Prep release + - mkdir -p release + - mv $IMAGE_LIST $IMAGE_PKG $REPOS_PKG release/ + + # Publish packages to s3 release + - aws s3 sync --quiet --dryrun release/ s3://umbrella-bigbang-releases/umbrella/${CI_COMMIT_TAG} + after_script: [] + +release: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_COMMIT_TAG' + - when: never + variables: + RELEASE_ENDPOINT: https://${RELEASE_BUCKET}.s3-${AWS_DEFAULT_REGION}.amazonaws.com/umbrella/${CI_COMMIT_TAG} + script: + - | + release-cli create --name "Big Bang ${CI_COMMIT_TAG}" --tag-name ${CI_COMMIT_TAG} \ + --description "Automated release notes are a WIP." \ + --assets-link "{\"name\":\"${IMAGE_LIST}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_LIST}\"}" \ + --assets-link "{\"name\":\"${IMAGE_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${IMAGE_PKG}\"}" \ + --assets-link "{\"name\":\"${REPOS_PKG}\",\"url\":\"${RELEASE_ENDPOINT}/${REPOS_PKG}\"}" + #----------------------------------------------------------------------------------------------------------------------- #----------------------------------------------------------------------------------------------------------------------- diff --git a/scripts/package/gits.sh b/scripts/package/gits.sh new file mode 100755 index 0000000000000000000000000000000000000000..a0b8e1a40e8099957ba377d8486e680b4c59235b --- /dev/null +++ b/scripts/package/gits.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +mkdir -p repos/ + +# Clone core +yq r "chart/values.yaml" "*.git.repo" | while IFS= read -r repo; do + git -C repos/ clone --no-checkout $repo +done + +# Clone packages +yq r "chart/values.yaml" "addons.*.git.repo" | while IFS= read -r repo; do + git -C repos/ clone --no-checkout $repo +done diff --git a/scripts/package/synker.Dockerfile b/scripts/package/synker.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bb1a151196a3de7a48028bf45c9fc2217a156cf5 --- /dev/null +++ b/scripts/package/synker.Dockerfile @@ -0,0 +1,33 @@ +FROM golang:1.13 AS builder + +# Download build dependencies +RUN apt-get update && apt-get install -y \ + git libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev liblvm2-dev musl-dev \ + && apt-get clean + +# Clone the latest release of p8kr and built the binrary statically +RUN git clone https://repo1.dsop.io/platform-one/hagrid/sync.git synker && \ + cd synker && \ + make binary-local-static DISABLE_CGO=1 + +# +FROM registry.access.redhat.com/ubi8/ubi:8.3 + +COPY --from=registry:2 /bin/registry /usr/local/bin/registry +COPY --from=builder /go/synker/synker /usr/local/bin/synker + +RUN yum install -y unzip git jq + +# Install yq +RUN curl -sfL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64 && \ + chmod +x /usr/local/bin/yq + +# Install aws cli +RUN curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o /tmp/awscliv2.zip && \ + unzip -qq -d /tmp /tmp/awscliv2.zip && \ + /tmp/aws/install && \ + rm -rf /tmp/aws* + +RUN yum clean all && \ + rm -r /var/cache/dnf + diff --git a/scripts/package/synker.yaml b/scripts/package/synker.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d8890288f47338191f476907bfee6b8a9168104 --- /dev/null +++ b/scripts/package/synker.yaml @@ -0,0 +1,20 @@ +# Baseline synker CI configs +# NOTE: THIS CONFIG IS PARTIAL (it is missing the `destination` component) AND ONLY USED FOR PULLING IN CI +transport: + registry: + hostname: localhost + port: 25000 + osChoice: linux + disablePolicyChecks: true + +options: + # TODO: Turn this off for now until parallel syncing is more "production" ready + # NOTE: Finding some bugs (even with retry) in pushing images to the airgapped registry that trace back to paralell syncing + parallelSync: false + quiet: false + debug: info + +source: + authFile: /root/.docker/config.json + images: + - registry.dsop.io/platform-one/big-bang/apps/security-tools/twistlock/defender:20.04.163 diff --git a/tests/ci/k3d/values.yaml b/tests/ci/k3d/values.yaml index a3a000cb7e0ccf27acf82631c98596c422b1da23..53ba5c26f8d498239af4ea175d044f9c30c4f010 100644 --- a/tests/ci/k3d/values.yaml +++ b/tests/ci/k3d/values.yaml @@ -39,11 +39,6 @@ clusterAuditor: monitoring: enabled: true -# values: -# prometheusOperator: -# admissionWebhooks: -# # Disable admission webhook creation for CI to speed things up -# enabled: false gatekeeper: enabled: true