From 7a2f661f6a3eee69d75c0166a70fec2c528b15a6 Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Wed, 2 Jun 2021 09:20:50 -0600 Subject: [PATCH 1/9] meh --- Dockerfile | 112 ++++++++++++++++ LICENSE | 202 ++++++++++++++++++++++++++++ README.md | 11 +- hardening_manifest.yaml | 286 ++++++++++++++++++++++++++++++++++++++++ license.txt | 1 + scripts/build-image.sh | 60 +++++++++ scripts/healthcheck.sh | 10 ++ 7 files changed, 680 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 hardening_manifest.yaml create mode 100644 license.txt create mode 100755 scripts/build-image.sh create mode 100644 scripts/healthcheck.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ca0892a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,112 @@ +ARG BASE_REGISTRY=registry1.dso.mil +ARG BASE_IMAGE=redhat/openjdk/openjdk11 +ARG BASE_TAG=1.11 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +# This affects how strings in Java class files are interpreted. +# We want UTF-8 and this is the only locale in the base image that supports it +ENV LANG="C.UTF-8" + +# IB custom ENV vars +ENV RPM_VERSION=6.1.1-1 +ENV IMG_TAR_VERSION=6.1.1.0 + +USER root + +######################################## +## Install Python +RUN dnf update -y && dnf install -y python38.x86_64 python38-pip-wheel.noarch openssl tar procps iputils hostname findutils nc \ + && ln -s /usr/bin/python3 /usr/bin/python \ + && ln -s /usr/bin/pip3 /usr/bin/pip + +## Python Installed +######################################## + +######################################## +## Add the Confluent Docker Utility Belt which helps with starting the proper applications +## https://github.com/confluentinc/common-docker/tree/master/utility-belt + +COPY confluent_docker_utils-0.0.44-py3-none-any.whl /tmp/confluent_docker_utils-0.0.44-py3-none-any.whl +COPY *.whl /tmp/python-dependencies/ +COPY *.gz /tmp/python-dependencies/ + +RUN pip3 install --no-index --prefix=/usr/local --upgrade --find-links /tmp/python-dependencies/ /tmp/confluent_docker_utils-0.0.44-py3-none-any.whl +## dub installed +######################################## + +RUN mkdir -p /usr/logs /licenses \ + && useradd --no-log-init --create-home --shell /bin/bash appuser + +COPY license.txt /licenses + +RUN chown appuser:appuser -R /usr/logs + +############ ^^^^^ Base Container Details ^^^^^ ############ +############################################################ +############ vvvvv This Container Below vvvvv ############ + +# Copy required RPMs +COPY *.rpm /tmp/ + +######################################## +# confluentinc/kafka-images/server + +ENV COMPONENT=ksqldb-server +ENV CONFLUENT_VERSION=6.1.1 +ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' + +ENV KSQL_CLASSPATH=/usr/share/java/ksqldb-server/* +ENV KSQL_CONFIG_DIR=/mnt/config +ENV KSQL_SECRETS_DIR=/mnt/apikeys +ENV KSQL_LOG4J_DIR=/mnt/log +ENV KSQL_DATA_DIR=/mnt/data +ENV COMPONENT_SCRIPT_DIR=/opt/confluent/etc +ENV DOCKER_SCRIPT_DIR=/opt/caas/bin + +EXPOSE 8088 + +RUN echo "===> Installing ${COMPONENT}" \ + && dnf install --nogpgcheck -y /tmp/confluent-common-${RPM_VERSION}.noarch.rpm \ + /tmp/confluent-security-${RPM_VERSION}.noarch.rpm \ + /tmp/confluent-telemetry-${RPM_VERSION}.noarch.rpm \ + /tmp/confluent-hub-client-${RPM_VERSION}.noarch.rpm \ + && echo "===> Setting up ${COMPONENT} dirs" \ + && mkdir -p /etc/ksqldb /usr/logs /usr/share/confluent-hub-components /var/lib/kafka-streams \ + && chown -R appuser:appuser /etc/ksqldb /usr/share/confluent-hub-components /var/lib/kafka-streams + +## Operator stuff +RUN mkdir -p /etc/${COMPONENT} /etc/${COMPONENT}/secrets /usr/logs /etc/confluent/ironbank +COPY cp-${COMPONENT}-operator-*_OptCaas.tar.gz /opt/ +COPY cp-${COMPONENT}-operator-*_UsrShareJava.tar.gz /usr/share/ +COPY cp-${COMPONENT}-operator-*_EtcConfluent.tar.gz /etc/ +COPY cp-${COMPONENT}-operator-*_EtcKsqldb.tar.gz /etc/ +COPY cp-${COMPONENT}-operator-*_OptConfluentKsqlLibs.tar.gz /opt/confluent/ksql/ +COPY cp-${COMPONENT}-operator-*_UsrBin.tar.gz /usr/bin/ + +## For auditing & debugging +COPY Dockerfile /etc/confluent/ironbank/ +COPY hardening_manifest.yaml /etc/confluent/ironbank/ + +RUN echo "===> Installing operator stuff" \ + && cd /usr/share && tar -xvf *_UsrShareJava.tar.gz && rm *_UsrShareJava.tar.gz \ + && cd /opt && tar -xvf *_OptCaas.tar.gz && rm *_OptCaas.tar.gz \ + && cd /etc && tar -xvf *_EtcConfluent.tar.gz && rm *_EtcConfluent.tar.gz \ + && cd /etc && tar -xvf *_EtcKsqldb.tar.gz && rm *_EtcKsqldb.tar.gz \ + && cd /opt/confluent/ksql && tar -xvf *_OptConfluentKsqlLibs.tar.gz && rm *_OptConfluentKsqlLibs.tar.gz \ + && cd /usr/bin && tar -xvf *_UsrBin.tar.gz && rm *_UsrBin.tar.gz \ + && chown -R appuser:appuser /etc/confluent /etc/${COMPONENT} /usr/logs /usr/bin/ksql /usr/share/java \ + && mkdir -p "${KSQL_LOG4J_DIR}" "${KSQL_SECRETS_DIR}" "${KSQL_CONFIG_DIR}" "${KSQL_DATA_DIR}" "${COMPONENT_SCRIPT_DIR}/${COMPONENT}" \ + && chmod -R ag+w "${KSQL_LOG4J_DIR}" "${KSQL_SECRETS_DIR}" "${KSQL_CONFIG_DIR}" "${KSQL_DATA_DIR}" "/opt" \ + && echo "===> Clean up, Clean up" \ + && dnf clean all \ + && rm -rf /tmp/* \ + && rm -rf /var/cache/dnf + +WORKDIR /opt +CMD ["/opt/caas/bin/run"] + +HEALTHCHECK --start-period=120s --interval=5s --timeout=10s --retries=96 \ + CMD /opt/caas/bin/ensure + +USER 1001 \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e06d208 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/README.md b/README.md index 5dc6fa6..d6c82b2 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ -# +# cp-ksqldb-server-operator + +This repository creates the Confluent cp-ksqldb-server Operator layering as a layer on top of [cp-ksqldb-server:6.0.1 image](https://repo1.dso.mil/dsop/confluent/ksqldb-server/cp-ksqldb-server-6.1.x). + +Architecture: https://www.confluent.io/resources/apache-kafka-confluent-enterprise-reference-architecture/ + +Operator Deployment: https://docs.confluent.io/operator/current/co-deployment.html + +For questions on this container please contact: confluent-fed@confluent.io -Project template for all Iron Bank container repositories. \ No newline at end of file diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml new file mode 100644 index 0000000..b771dd6 --- /dev/null +++ b/hardening_manifest.yaml @@ -0,0 +1,286 @@ +--- +apiVersion: v1 + +# The repository name in registry1, excluding /ironbank/ +name: "confluentinc/cp-ksqldb-server-operator" + +# List of tags to push for the repository in registry1 +# The most specific version should be the first tag and will be shown +# on ironbank.dso.mil +tags: +- "6.1.1.0" +- "6.1.1" +- "6.1.x" +- "6.1" + +# Build args passed to Dockerfile ARGs +args: + BASE_IMAGE: "redhat/openjdk/openjdk11" + BASE_TAG: "1.11" + +# Docker image labels +labels: + org.opencontainers.image.title: "cp-ksqldb-server-operator" + org.opencontainers.image.description: "Confluent Operator image for cp-ksqldb-server." + org.opencontainers.image.licenses: "CONFLUENT ENTERPRISE LICENSE" + org.opencontainers.image.url: "https://docs.confluent.io/6.1.1/installation/operator/index.html" + org.opencontainers.image.vendor: "Confluent" + org.opencontainers.image.version: "6.1.1" + mil.dso.ironbank.image.keywords: "confluent,kafka,zookeeper,operator" + mil.dso.ironbank.image.type: "commercial" + mil.dso.ironbank.product.name: "Confluent Platform" + +# List of resources to make available to the offline build context +resources: +- filename: boto3-1.17.16-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/c2/03/18184037cb21cab227e392962e0ba9a7596d777a08d7c07c2d3640f939bf/boto3-1.17.16-py2.py3-none-any.whl + validation: + type: sha256 + value: 602eadaef665f49090344e0f87aa6a98dbe1ccdd2f20069a372ed35f2706c63c +- filename: paramiko-2.7.2-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/95/19/124e9287b43e6ff3ebb9cdea3e5e8e88475a873c05ccdf8b7e20d2c4201e/paramiko-2.7.2-py2.py3-none-any.whl + validation: + type: sha256 + value: 4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898 +- filename: docker-4.4.4-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/c4/22/410313ad554477e87ec406d38d85f810e61ddb0d2fc44e64994857476de9/docker-4.4.4-py2.py3-none-any.whl + validation: + type: sha256 + value: f3607d5695be025fa405a12aca2e5df702a57db63790c73b927eb6a94aac60af +- filename: docker_compose-1.28.4-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/00/ff/17cdd2bc5f581cd80fc0b45b549d48c6eff7cd70d20a9f805a0c89394e69/docker_compose-1.28.4-py2.py3-none-any.whl + validation: + type: sha256 + value: 92375b30ab7134e8c32470b621e7cf9a3c0771ce2c20de7e1f11cd71f83a088e +- filename: Jinja2-2.11.3-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/7e/c2/1eece8c95ddbc9b1aeb64f5783a9e07a286de42191b7204d67b7496ddf35/Jinja2-2.11.3-py2.py3-none-any.whl + validation: + type: sha256 + value: 03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419 +- filename: mock-4.0.3-py3-none-any.whl + url: https://files.pythonhosted.org/packages/5c/03/b7e605db4a57c0f6fba744b11ef3ddf4ddebcada35022927a2b5fc623fdf/mock-4.0.3-py3-none-any.whl + validation: + type: sha256 + value: 122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62 +- filename: requests-2.25.1-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/29/c1/24814557f1d22c56d50280771a17307e6bf87b70727d975fd6b2ce6b014a/requests-2.25.1-py2.py3-none-any.whl + validation: + type: sha256 + value: c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e +- filename: cryptography-3.4.6-cp36-abi3-manylinux2014_x86_64.whl + url: https://files.pythonhosted.org/packages/f8/1f/acde6ff69864c5e78b56488e3afd93c1ccc8c2651186e2a5f93d93f64859/cryptography-3.4.6-cp36-abi3-manylinux2014_x86_64.whl + validation: + type: sha256 + value: fec7fb46b10da10d9e1d078d1ff8ed9e05ae14f431fdbd11145edd0550b9a964 +- filename: s3transfer-0.3.4-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/ea/43/4b4a1b26eb03a429a4c37ca7fdf369d938bd60018fc194e94b8379b0c77c/s3transfer-0.3.4-py2.py3-none-any.whl + validation: + type: sha256 + value: 1e28620e5b444652ed752cf87c7e0cb15b0e578972568c6609f0f18212f259ed +- filename: jmespath-0.10.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/07/cb/5f001272b6faeb23c1c9e0acc04d48eaaf5c862c17709d20e3469c6e0139/jmespath-0.10.0-py2.py3-none-any.whl + validation: + type: sha256 + value: cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f +- filename: botocore-1.20.16-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/17/da/da506c061c22a4068163dc0b8d1f96e9ae87cf34f8086c7460e94261f90f/botocore-1.20.16-py2.py3-none-any.whl + validation: + type: sha256 + value: 48350c0524fafcc6f1cf792a80080eeaf282c4ceed016e9296f1ebfda7c34fb3 +- filename: python_dateutil-2.8.1-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/d4/70/d60450c3dd48ef87586924207ae8907090de0b306af2bce5d134d78615cb/python_dateutil-2.8.1-py2.py3-none-any.whl + validation: + type: sha256 + value: 75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a +- filename: urllib3-1.26.4-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/09/c6/d3e3abe5b4f4f16cf0dfc9240ab7ce10c2baa0e268989a4e3ec19e90c84e/urllib3-1.26.4-py2.py3-none-any.whl + validation: + type: sha256 + value: 2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df +- filename: cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl + url: https://files.pythonhosted.org/packages/5c/0f/e07df370fac0e99e938edc62c8a15e54b9d75605e11838fa0ef300118e1d/cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl + validation: + type: sha256 + value: 6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e +- filename: pycparser-2.20-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/ae/e7/d9c3a176ca4b02024debf82342dab36efadfc5776f9c8db077e8f6e71821/pycparser-2.20-py2.py3-none-any.whl + validation: + type: sha256 + value: 7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705 +- filename: websocket_client-0.57.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/4c/5f/f61b420143ed1c8dc69f9eaec5ff1ac36109d52c80de49d66e0c36c3dfdf/websocket_client-0.57.0-py2.py3-none-any.whl + validation: + type: sha256 + value: 0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549 +- filename: six-1.15.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl + validation: + type: sha256 + value: 8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced +- filename: docopt-0.6.2.tar.gz + url: https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz + validation: + type: sha256 + value: 49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491 +- filename: jsonschema-3.2.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/c5/8f/51e89ce52a085483359217bc72cdbf6e75ee595d5b1d4b5ade40c7e018b8/jsonschema-3.2.0-py2.py3-none-any.whl + validation: + type: sha256 + value: 4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 +- filename: python_dotenv-0.15.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/32/2e/e4585559237787966aad0f8fd0fc31df1c4c9eb0e62de458c5b6cde954eb/python_dotenv-0.15.0-py2.py3-none-any.whl + validation: + type: sha256 + value: 0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e +- filename: cached_property-1.5.2-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/48/19/f2090f7dad41e225c7f2326e4cfe6fff49e57dedb5b53636c9551f86b069/cached_property-1.5.2-py2.py3-none-any.whl + validation: + type: sha256 + value: df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0 +- filename: dockerpty-0.4.1.tar.gz + url: https://files.pythonhosted.org/packages/8d/ee/e9ecce4c32204a6738e0a5d5883d3413794d7498fe8b06f44becc028d3ba/dockerpty-0.4.1.tar.gz + validation: + type: sha256 + value: 69a9d69d573a0daa31bcd1c0774eeed5c15c295fe719c61aca550ed1393156ce +- filename: PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl + url: https://files.pythonhosted.org/packages/70/96/c7245e551b1cb496bfb95840ace55ca60f20d3d8e33d70faf8c78a976899/PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl + validation: + type: sha256 + value: 8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb +- filename: texttable-1.6.3-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/06/f5/46201c428aebe0eecfa83df66bf3e6caa29659dbac5a56ddfd83cae0d4a4/texttable-1.6.3-py2.py3-none-any.whl + validation: + type: sha256 + value: f802f2ef8459058736264210f716c757cbf85007a30886d8541aa8c3404f1dda +- filename: distro-1.5.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/25/b7/b3c4270a11414cb22c6352ebc7a83aaa3712043be29daa05018fd5a5c956/distro-1.5.0-py2.py3-none-any.whl + validation: + type: sha256 + value: df74eed763e18d10d0da624258524ae80486432cd17392d9c3d96f5e83cd2799 +- filename: MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl + url: https://files.pythonhosted.org/packages/4b/20/f6d7648c81cb84815d0be935d5c74cd1cc0239e43eadb1a61062d34b6543/MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl + validation: + type: sha256 + value: 13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42 +- filename: importlib_metadata-3.7.0-py3-none-any.whl + url: https://files.pythonhosted.org/packages/3a/0d/af9e3dce6524461fa1e8327449f392edac8a3d880b4c91ce3e2d25450d03/importlib_metadata-3.7.0-py3-none-any.whl + validation: + type: sha256 + value: c6af5dbf1126cd959c4a8d8efd61d4d3c83bddb0459a17e554284a077574b614 +- filename: attrs-20.3.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/c3/aa/cb45262569fcc047bf070b5de61813724d6726db83259222cd7b4c79821a/attrs-20.3.0-py2.py3-none-any.whl + validation: + type: sha256 + value: 31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 +- filename: pyrsistent-0.17.3.tar.gz + url: https://files.pythonhosted.org/packages/4d/70/fd441df751ba8b620e03fd2d2d9ca902103119616f0f6cc42e6405035062/pyrsistent-0.17.3.tar.gz + validation: + type: sha256 + value: 2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e +- filename: PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl + url: https://files.pythonhosted.org/packages/9d/57/2f5e6226a674b2bcb6db531e8b383079b678df5b10cdaa610d6cf20d77ba/PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl + validation: + type: sha256 + value: 30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d +- filename: bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl + url: https://files.pythonhosted.org/packages/26/70/6d218afbe4c73538053c1016dd631e8f25fffc10cd01f5c272d7acf3c03d/bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl + validation: + type: sha256 + value: cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1 +- filename: certifi-2020.12.5-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/5e/a0/5f06e1e1d463903cf0c0eebeb751791119ed7a4b3737fdc9a77f1cdfb51f/certifi-2020.12.5-py2.py3-none-any.whl + validation: + type: sha256 + value: 719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 +- filename: chardet-4.0.0-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/19/c7/fa589626997dd07bd87d9269342ccb74b1720384a4d739a1872bd84fbe68/chardet-4.0.0-py2.py3-none-any.whl + validation: + type: sha256 + value: f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5 +- filename: idna-2.10-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/a2/38/928ddce2273eaa564f6f50de919327bf3a00f091b5baba8dfa9460f3a8a8/idna-2.10-py2.py3-none-any.whl + validation: + type: sha256 + value: b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0 +- filename: typing_extensions-3.7.4.3-py3-none-any.whl + url: https://files.pythonhosted.org/packages/60/7a/e881b5abb54db0e6e671ab088d079c57ce54e8a01a3ca443f561ccadb37e/typing_extensions-3.7.4.3-py3-none-any.whl + validation: + type: sha256 + value: 7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918 +- filename: zipp-3.4.0-py3-none-any.whl + url: https://files.pythonhosted.org/packages/41/ad/6a4f1a124b325618a7fb758b885b68ff7b058eec47d9220a12ab38d90b1f/zipp-3.4.0-py3-none-any.whl + validation: + type: sha256 + value: 102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108 +- filename: confluent_docker_utils-0.0.44-py3-none-any.whl + url: https://ironbank-files.s3.amazonaws.com/confluent_docker_utils-0.0.44-py3-none-any.whl + validation: + type: sha256 + value: 7e2622a934f04a2e5b23a355f920473ccf0144e946b433bc619a9802c1e3785c +- filename: confluent-common-6.1.1-1.noarch.rpm + url: https://packages.confluent.io/rpm/6.1/confluent-common-6.1.1-1.noarch.rpm + validation: + type: sha256 + value: +- filename: confluent-security-6.1.1-1.noarch.rpm + url: https://packages.confluent.io/rpm/6.1/confluent-security-6.1.1-1.noarch.rpm + validation: + type: sha256 + value: +- filename: confluent-hub-client-6.1.1-1.noarch.rpm + url: https://packages.confluent.io/rpm/6.1/confluent-hub-client-6.1.1-1.noarch.rpm + validation: + type: sha256 + value: +- filename: confluent-telemetry-6.1.1-1.noarch.rpm + url: https://packages.confluent.io/rpm/6.1/confluent-telemetry-6.1.1-1.noarch.rpm + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_OptCaas.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_OptCaas.tar.gz + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_UsrShareJava.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrShareJava.tar.gz + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_UsrShareDoc.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrShareDoc.tar.gz + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_EtcConfluent.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_EtcConfluent.tar.gz + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_EtcKsqldb.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_EtcKsqldb.tar.gz + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_OptConfluentKsqlLibs.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_OptConfluentKsqlLibs.tar.gz + validation: + type: sha256 + value: +- filename: cp-ksqldb-server-operator-6.1.1.0_UsrBin.tar.gz + url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrBin.tar.gz + validation: + type: sha256 + value: + +# List of project maintainers +maintainers: +- name: "Scott Stroud" + username: "scottstroud" + email: "confluent-fed@confluent.io" + cht_member: false +- name: "Preston McGowan" + username: "preston.mcgowan" + email: "confluent-fed@confluent.io" + cht_member: false + \ No newline at end of file diff --git a/license.txt b/license.txt new file mode 100644 index 0000000..44cfd8b --- /dev/null +++ b/license.txt @@ -0,0 +1 @@ +Copyright 2020 Confluent, Inc. \ No newline at end of file diff --git a/scripts/build-image.sh b/scripts/build-image.sh new file mode 100755 index 0000000..ab22f39 --- /dev/null +++ b/scripts/build-image.sh @@ -0,0 +1,60 @@ +#!/bin/bash +start=`date +%s` + +## +# Docker Dependencies: +# https://docs.confluent.io/current/installation/docker/image-reference.html#image-reference + +## +# Settings for the build process + +if [[ -f .env ]]; then + echo "Loading in '.env' for values ..." + export $(grep -v '^#' .env | xargs) + echo " " +elif [[ -f ../.env ]]; then + echo "Loading in PARENT '.env' for values ..." + export $(grep -v '^#' ../.env | xargs) + echo " " +fi + +# The BASE_ is used by PlatformOne to change the base image, we will make use of RedHat +BASE_REGISTRY=${BASE_REGISTRY:-'registry1.dso.mil'} +MANI='hardening_manifest.yaml' + +## parse manifest with yq (https://github.com/mikefarah/yq) +name=$(yq e '.name' $MANI) +tags=( $(yq e '.tags.[]' $MANI) ) +baseImage=$(yq e '.args.BASE_IMAGE' $MANI) +baseTag=$(yq e '.args.BASE_TAG' $MANI) + +echo "Starting $name build ..." + +echo "$name docker build ..." + +if [[ "$1" == "--squash" ]]; then + echo "Squashing like a cockroach ..." + DOCKER_BUILDKIT=1 docker build \ + --squash \ + --tag $BASE_REGISTRY/$name:latest \ + --build-arg BASE_REGISTRY=$BASE_REGISTRY \ + --build-arg BASE_IMAGE=$baseImage \ + --build-arg BASE_TAG=$baseTag \ + . +else + DOCKER_BUILDKIT=1 docker build \ + --tag $BASE_REGISTRY/$name:latest \ + --build-arg BASE_REGISTRY=$BASE_REGISTRY \ + --build-arg BASE_IMAGE=$baseImage \ + --build-arg BASE_TAG=$baseTag \ + . +fi + +for tag in "${tags[@]}";do + docker tag $BASE_REGISTRY/$name:latest $BASE_REGISTRY/$name:$tag + echo "🏷 Tagged build with $BASE_REGISTRY/$name:$tag" +done + +end=`date +%s` +duration=$((end-start)) +echo "🏁 Finished $CONTAINER build (${duration} seconds)" \ No newline at end of file diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh new file mode 100644 index 0000000..0f106ac --- /dev/null +++ b/scripts/healthcheck.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +curl_status=$(curl -X POST -s -o /dev/null -w %{http_code} $KSQL_LISTENERS/ksql -H 'content-type: application/vnd.ksql.v1+json; charset=utf-8' -d '{"ksql":"SHOW TOPICS;"}') +if [ $curl_status -eq 200 ]; then + echo "Woohoo! KSQL is up!" + exit 0 +else + echo -e $(date) " KSQL server query response code: " $curl_status " (waiting for 200)" + exit 1 +fi \ No newline at end of file -- GitLab From 263c4ad94889b2eb3c817665b3d03f6c684be0e2 Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Wed, 2 Jun 2021 15:13:52 -0600 Subject: [PATCH 2/9] meh --- hardening_manifest.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index b771dd6..4108798 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -221,57 +221,57 @@ resources: url: https://packages.confluent.io/rpm/6.1/confluent-common-6.1.1-1.noarch.rpm validation: type: sha256 - value: + value: 3ad1efeceed8b2febedc2534191e92980766ebc88d6a44e665b8febd7978a105 - filename: confluent-security-6.1.1-1.noarch.rpm url: https://packages.confluent.io/rpm/6.1/confluent-security-6.1.1-1.noarch.rpm validation: type: sha256 - value: + value: efb748578c9c14b9ac8eec16da3bc1bdfb47bcd9eb8f741a0be393df8b3a9867 - filename: confluent-hub-client-6.1.1-1.noarch.rpm url: https://packages.confluent.io/rpm/6.1/confluent-hub-client-6.1.1-1.noarch.rpm validation: type: sha256 - value: + value: 43da2b6c9362043ff12bc436bc325a407ca44e1c90a1794e00e856da4ec17e01 - filename: confluent-telemetry-6.1.1-1.noarch.rpm url: https://packages.confluent.io/rpm/6.1/confluent-telemetry-6.1.1-1.noarch.rpm validation: type: sha256 - value: + value: 339ae6258cb2e1152d91633defdabb5d76ba13cc12a5eedc06f3cd12a3fffd4c - filename: cp-ksqldb-server-operator-6.1.1.0_OptCaas.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_OptCaas.tar.gz validation: type: sha256 - value: + value: fc357ef94cd836b914d568d21cbd2b0e5782f1d4ecfa3b82e581a22d21c071ad - filename: cp-ksqldb-server-operator-6.1.1.0_UsrShareJava.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrShareJava.tar.gz validation: type: sha256 - value: + value: 0ddb9aad6bf78ef57fdd2dfac250bbbde848234a4ce389520652994bd1e9e6d2 - filename: cp-ksqldb-server-operator-6.1.1.0_UsrShareDoc.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrShareDoc.tar.gz validation: type: sha256 - value: + value: 7110c4f688ddc23339481694a3bafde74dfc0faa8ca1860cccd7fcf8c7fcdb31 - filename: cp-ksqldb-server-operator-6.1.1.0_EtcConfluent.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_EtcConfluent.tar.gz validation: type: sha256 - value: + value: f28ee141284c731f552eb66f93a4e26b85fb881751377ab7af3807ea7a99f901 - filename: cp-ksqldb-server-operator-6.1.1.0_EtcKsqldb.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_EtcKsqldb.tar.gz validation: type: sha256 - value: + value: b7b2ddea51736e4219be80282575d128863b040c2dfb71542cf6fac269a4a921 - filename: cp-ksqldb-server-operator-6.1.1.0_OptConfluentKsqlLibs.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_OptConfluentKsqlLibs.tar.gz validation: type: sha256 - value: + value: 0ca5f0420f1e64d15e6f066574b4123cb7ca7287ecf50963caa9289ab373df5c - filename: cp-ksqldb-server-operator-6.1.1.0_UsrBin.tar.gz url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrBin.tar.gz validation: type: sha256 - value: + value: d76da0002ea5b36cf3ea788aaeacc55eafc5f8ab4e5807e9d6e3ecc4794066b4 # List of project maintainers maintainers: -- GitLab From cc8cf4def95405c7a95a0768b92d554d69ca36b7 Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Thu, 3 Jun 2021 09:32:34 -0600 Subject: [PATCH 3/9] meh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6c82b2..e0daa36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cp-ksqldb-server-operator -This repository creates the Confluent cp-ksqldb-server Operator layering as a layer on top of [cp-ksqldb-server:6.0.1 image](https://repo1.dso.mil/dsop/confluent/ksqldb-server/cp-ksqldb-server-6.1.x). +This repository creates the Confluent cp-ksqldb-server Operator. Fundamentally the Dockerfile is the summation of Confluent's cp-base-new, cp-server, and cp-server-operator images. Architecture: https://www.confluent.io/resources/apache-kafka-confluent-enterprise-reference-architecture/ -- GitLab From 02302fa97ead815fbf49af1bc05391c78eee1be9 Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Fri, 4 Jun 2021 18:13:18 +0000 Subject: [PATCH 4/9] cve fix --- hardening_manifest.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 4108798..074fc2a 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -92,11 +92,11 @@ resources: validation: type: sha256 value: 75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a -- filename: urllib3-1.26.4-py2.py3-none-any.whl - url: https://files.pythonhosted.org/packages/09/c6/d3e3abe5b4f4f16cf0dfc9240ab7ce10c2baa0e268989a4e3ec19e90c84e/urllib3-1.26.4-py2.py3-none-any.whl +- filename: urllib3-1.26.5-py2.py3-none-any.whl + url: https://files.pythonhosted.org/packages/0c/cd/1e2ec680ec7b09846dc6e605f5a7709dfb9d7128e51a026e7154e18a234e/urllib3-1.26.5-py2.py3-none-any.whl validation: type: sha256 - value: 2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df + value: 753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c - filename: cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl url: https://files.pythonhosted.org/packages/5c/0f/e07df370fac0e99e938edc62c8a15e54b9d75605e11838fa0ef300118e1d/cffi-1.14.5-cp38-cp38-manylinux1_x86_64.whl validation: @@ -283,4 +283,4 @@ maintainers: username: "preston.mcgowan" email: "confluent-fed@confluent.io" cht_member: false - \ No newline at end of file + -- GitLab From 7ff945c667c67fd8e5ea1243fb6d8d95d9930abd Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Mon, 7 Jun 2021 13:24:47 -0600 Subject: [PATCH 5/9] feedback --- .dockerignore | 3 + .gitignore | 4 + Dockerfile | 2 +- LICENSE | 203 +-------------------------------- build-image.sh | 64 +++++++++++ license.txt | 1 - scripts/build-image.sh | 4 + scripts/genManifestTemplate.sh | 80 +++++++++++++ scripts/importArtifacts.sh | 70 ++++++++++++ 9 files changed, 227 insertions(+), 204 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100755 build-image.sh delete mode 100644 license.txt create mode 100755 scripts/genManifestTemplate.sh create mode 100755 scripts/importArtifacts.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2967ddf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +scripts/build-image.sh +scripts/genManifestTemplate.sh +scripts/importArtifacts.sh \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..00dcc5c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.whl +*.tar.gz +*.rpm +jsonnet \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ca0892a..3e9c610 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ RUN pip3 install --no-index --prefix=/usr/local --upgrade --find-links /tmp/pyth RUN mkdir -p /usr/logs /licenses \ && useradd --no-log-init --create-home --shell /bin/bash appuser -COPY license.txt /licenses +COPY LICENSE /licenses RUN chown appuser:appuser -R /usr/logs diff --git a/LICENSE b/LICENSE index e06d208..44cfd8b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,202 +1 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - +Copyright 2020 Confluent, Inc. \ No newline at end of file diff --git a/build-image.sh b/build-image.sh new file mode 100755 index 0000000..4826cb6 --- /dev/null +++ b/build-image.sh @@ -0,0 +1,64 @@ +#!/bin/bash +start=`date +%s` + +## +# Docker Dependencies: +# https://docs.confluent.io/current/installation/docker/image-reference.html#image-reference + +## +# Settings for the build process + +if [[ -f .env ]]; then + echo "Loading in '.env' for values ..." + export $(grep -v '^#' .env | xargs) + echo " " +elif [[ -f ../.env ]]; then + echo "Loading in PARENT '.env' for values ..." + export $(grep -v '^#' ../.env | xargs) + echo " " +fi + +# The BASE_ is used by PlatformOne to change the base image, we will make use of RedHat +BASE_REGISTRY=${BASE_REGISTRY:-'registry1.dso.mil'} +MANI='hardening_manifest.yaml' + +cd ../ + +## parse manifest with yq (https://github.com/mikefarah/yq) +name=$(yq e '.name' $MANI) +tags=( $(yq e '.tags.[]' $MANI) ) +baseImage=$(yq e '.args.BASE_IMAGE' $MANI) +baseTag=$(yq e '.args.BASE_TAG' $MANI) + +echo "Starting $name build ..." + +echo "$name docker build ..." + +if [[ "$1" == "--squash" ]]; then + echo "Squashing like a cockroach ..." + DOCKER_BUILDKIT=1 docker build \ + --squash \ + --tag $BASE_REGISTRY/$name:latest \ + --build-arg BASE_REGISTRY=$BASE_REGISTRY \ + --build-arg BASE_IMAGE=$baseImage \ + --build-arg BASE_TAG=$baseTag \ + . +else + DOCKER_BUILDKIT=1 docker build \ + --tag $BASE_REGISTRY/$name:latest \ + --build-arg BASE_REGISTRY=$BASE_REGISTRY \ + --build-arg BASE_IMAGE=$baseImage \ + --build-arg BASE_TAG=$baseTag \ + . +fi + +for tag in "${tags[@]}";do + docker tag $BASE_REGISTRY/$name:latest $BASE_REGISTRY/$name:$tag + echo "🏷 Tagged build with $BASE_REGISTRY/$name:$tag" +done + +cd scripts + +end=`date +%s` +duration=$((end-start)) +echo "🏁 Finished $CONTAINER build (${duration} seconds)" \ No newline at end of file diff --git a/license.txt b/license.txt deleted file mode 100644 index 44cfd8b..0000000 --- a/license.txt +++ /dev/null @@ -1 +0,0 @@ -Copyright 2020 Confluent, Inc. \ No newline at end of file diff --git a/scripts/build-image.sh b/scripts/build-image.sh index ab22f39..4826cb6 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -22,6 +22,8 @@ fi BASE_REGISTRY=${BASE_REGISTRY:-'registry1.dso.mil'} MANI='hardening_manifest.yaml' +cd ../ + ## parse manifest with yq (https://github.com/mikefarah/yq) name=$(yq e '.name' $MANI) tags=( $(yq e '.tags.[]' $MANI) ) @@ -55,6 +57,8 @@ for tag in "${tags[@]}";do echo "🏷 Tagged build with $BASE_REGISTRY/$name:$tag" done +cd scripts + end=`date +%s` duration=$((end-start)) echo "🏁 Finished $CONTAINER build (${duration} seconds)" \ No newline at end of file diff --git a/scripts/genManifestTemplate.sh b/scripts/genManifestTemplate.sh new file mode 100755 index 0000000..943df45 --- /dev/null +++ b/scripts/genManifestTemplate.sh @@ -0,0 +1,80 @@ +if [[ -f .env ]]; then + echo "Loading in '.env' for values ..." + export $(grep -v '^#' .env | xargs) + echo " " +fi + +VERSION=${VERSION:-'6.1.1.0'} + +SCRIPTS_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +PROJECT_HOME=${PROJECT_HOME:-'../'} +PROJECT_HOME=$(realpath $PROJECT_HOME) + +myShaRona () { + myfile=$(basename $1) + mydir=$(dirname $1) + + mkdir -p $mydir + if [ ! -e "$mydir/filelist.txt" ] ; then + touch "$mydir/filelist.txt" + fi + + if [[ "$OSTYPE" == "darwin"* ]] ; then + mySha=$(shasum -a 256 $1 | cut -d " " -f 1) + else + mySha=$(sha256sum $1 | cut -d " " -f 1) + fi + + echo -e "$myfile | $mySha" >> $mydir/filelist.txt +} + +filelist () { + rm -f $1/filelist.txt + + for filename in $1/*; do + myShaRona $filename + done +} + +if [ $# -eq 0 ]; then + echo "filelist.txt or dir path argument required " + exit 1 +fi + +# if there is not a filelist.txt, generate one +if [ -f "$1" ]; then + filefile=$(realpath $1) +elif [ -f "$1/filelist.txt" ]; then + filefile=$(realpath $1)/filelist.txt +elif [ -d "$1" ]; then + filefile=$(realpath $1)/filelist.txt +fi + +filelist $1 + +basedir=$(dirname $filefile) +cd $basedir + +if [[ $filefile == *"ironbank-files"* ]]; then + urlPrefix="https://ironbank-files.s3.amazonaws.com" +elif [[ $filefile == *"confluent"* ]]; then + urlPrefix="https://packages.confluent.io/rpm/${VERSION:0:3}" +elif [[ $filefile == *"pythonhosted"* ]]; then + echo "unsupported cause of the weird guid in the path" + exit 0 +fi + +echo "resources:" > $basedir/hardened_manifest_template.yaml + +while IFS= read -r line +do +IFS=" | " read -a parts <<< $line +cat >> $basedir/hardened_manifest_template.yaml << EOL +- filename: ${parts[0]} + url: ${urlPrefix}/${parts[0]} + validation: + type: sha256 + value: ${parts[1]} +EOL + +done < "$filefile" \ No newline at end of file diff --git a/scripts/importArtifacts.sh b/scripts/importArtifacts.sh new file mode 100755 index 0000000..3213270 --- /dev/null +++ b/scripts/importArtifacts.sh @@ -0,0 +1,70 @@ + +#!/bin/bash + +start=`date +%s` + +if [[ -f .env ]]; then + echo "Loading in '.env' for values ..." + export $(grep -v '^#' .env | xargs) + echo " " +fi + +SCRIPTS_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +PROJECT_HOME=${PROJECT_HOME:-'../'} +PROJECT_HOME=$(realpath $PROJECT_HOME) +RESOURCE_HOME=${RESOURCE_HOME:-'../../resources'} +RESOURCE_HOME=$(realpath $RESOURCE_HOME) + +importArtifacts() { + + manifestPath=$1 + httpdomain=$2 + resourceDir=$3 + + echo -e "\nImporting ${httpdomain} artifacts ..." + + maniDir=$(dirname "$manifestPath") + if [[ ! -f "$maniDir/build-image.sh" ]]; then + cp build-image.sh $maniDir/ + fi + + # (re)set the filelist.txt + mkdir -p $resourceDir + prefix="url: " + pushd $resourceDir + grep $httpdomain $manifestPath | while read -r line ; do + url=${line##*$prefix} + filename=$(basename "$url") + + if [[ ! -f "$filename" ]]; then + echo "downloading ${url} ..." + wget $url + else + echo "Using existing $filename. Run ./clean.sh if thats not desired." + fi + + if [[ ! -f "$maniDir/$filename" ]]; then + echo "copying ${filename} into ${maniDir} ..." + cp $filename $maniDir/ + fi + done + popd +} + +mani=$(realpath "../hardening_manifest.yaml") + +importArtifacts $mani 'ironbank-files.s3.amazonaws.com' "$RESOURCE_HOME/ironbank-files" +importArtifacts $mani 'packages.confluent.io' "$RESOURCE_HOME/confluent" +importArtifacts $mani 'files.pythonhosted.org' "$RESOURCE_HOME/pythonhosted" + +echo "Generating hardening_manifest_template.yaml ..." +$SCRIPTS_HOME/genManifestTemplate.sh "$RESOURCE_HOME/ironbank-files" +$SCRIPTS_HOME/genManifestTemplate.sh "$RESOURCE_HOME/confluent" +$SCRIPTS_HOME/genManifestTemplate.sh "$RESOURCE_HOME/pythonhosted" + +end=`date +%s` +duration=$((end-start)) +echo "#########################" +echo "All done (${duration} seconds)" +echo "#########################" + -- GitLab From 332ac49505ab24644ea0b93f0a66e05e11fc92bb Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Mon, 7 Jun 2021 15:43:06 -0600 Subject: [PATCH 6/9] rpm key stuff --- Dockerfile | 4 ++- build-image.sh | 64 -------------------------------------- hardening_manifest.yaml | 5 +++ scripts/importArtifacts.sh | 3 -- 4 files changed, 8 insertions(+), 68 deletions(-) delete mode 100755 build-image.sh diff --git a/Dockerfile b/Dockerfile index 3e9c610..b6e22ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,7 @@ RUN chown appuser:appuser -R /usr/logs # Copy required RPMs COPY *.rpm /tmp/ +COPY archive.key /tmp/ ######################################## # confluentinc/kafka-images/server @@ -67,7 +68,8 @@ ENV DOCKER_SCRIPT_DIR=/opt/caas/bin EXPOSE 8088 RUN echo "===> Installing ${COMPONENT}" \ - && dnf install --nogpgcheck -y /tmp/confluent-common-${RPM_VERSION}.noarch.rpm \ + && rpm --import /tmp/archive.key \ + && dnf install -y /tmp/confluent-common-${RPM_VERSION}.noarch.rpm \ /tmp/confluent-security-${RPM_VERSION}.noarch.rpm \ /tmp/confluent-telemetry-${RPM_VERSION}.noarch.rpm \ /tmp/confluent-hub-client-${RPM_VERSION}.noarch.rpm \ diff --git a/build-image.sh b/build-image.sh deleted file mode 100755 index 4826cb6..0000000 --- a/build-image.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -start=`date +%s` - -## -# Docker Dependencies: -# https://docs.confluent.io/current/installation/docker/image-reference.html#image-reference - -## -# Settings for the build process - -if [[ -f .env ]]; then - echo "Loading in '.env' for values ..." - export $(grep -v '^#' .env | xargs) - echo " " -elif [[ -f ../.env ]]; then - echo "Loading in PARENT '.env' for values ..." - export $(grep -v '^#' ../.env | xargs) - echo " " -fi - -# The BASE_ is used by PlatformOne to change the base image, we will make use of RedHat -BASE_REGISTRY=${BASE_REGISTRY:-'registry1.dso.mil'} -MANI='hardening_manifest.yaml' - -cd ../ - -## parse manifest with yq (https://github.com/mikefarah/yq) -name=$(yq e '.name' $MANI) -tags=( $(yq e '.tags.[]' $MANI) ) -baseImage=$(yq e '.args.BASE_IMAGE' $MANI) -baseTag=$(yq e '.args.BASE_TAG' $MANI) - -echo "Starting $name build ..." - -echo "$name docker build ..." - -if [[ "$1" == "--squash" ]]; then - echo "Squashing like a cockroach ..." - DOCKER_BUILDKIT=1 docker build \ - --squash \ - --tag $BASE_REGISTRY/$name:latest \ - --build-arg BASE_REGISTRY=$BASE_REGISTRY \ - --build-arg BASE_IMAGE=$baseImage \ - --build-arg BASE_TAG=$baseTag \ - . -else - DOCKER_BUILDKIT=1 docker build \ - --tag $BASE_REGISTRY/$name:latest \ - --build-arg BASE_REGISTRY=$BASE_REGISTRY \ - --build-arg BASE_IMAGE=$baseImage \ - --build-arg BASE_TAG=$baseTag \ - . -fi - -for tag in "${tags[@]}";do - docker tag $BASE_REGISTRY/$name:latest $BASE_REGISTRY/$name:$tag - echo "🏷 Tagged build with $BASE_REGISTRY/$name:$tag" -done - -cd scripts - -end=`date +%s` -duration=$((end-start)) -echo "🏁 Finished $CONTAINER build (${duration} seconds)" \ No newline at end of file diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 074fc2a..9dd7b99 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -217,6 +217,11 @@ resources: validation: type: sha256 value: 7e2622a934f04a2e5b23a355f920473ccf0144e946b433bc619a9802c1e3785c +- filename: archive.key + url: https://packages.confluent.io/rpm/6.1/archive.key + validation: + type: sha256 + value: b569e9d80bc08b65d342491f8b94e47eb7032701ce17b091f212e1072672a4d5 - filename: confluent-common-6.1.1-1.noarch.rpm url: https://packages.confluent.io/rpm/6.1/confluent-common-6.1.1-1.noarch.rpm validation: diff --git a/scripts/importArtifacts.sh b/scripts/importArtifacts.sh index 3213270..43dfcaf 100755 --- a/scripts/importArtifacts.sh +++ b/scripts/importArtifacts.sh @@ -24,9 +24,6 @@ importArtifacts() { echo -e "\nImporting ${httpdomain} artifacts ..." maniDir=$(dirname "$manifestPath") - if [[ ! -f "$maniDir/build-image.sh" ]]; then - cp build-image.sh $maniDir/ - fi # (re)set the filelist.txt mkdir -p $resourceDir -- GitLab From 39c09798707072699d548080d458e41168e8e7f3 Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Thu, 10 Jun 2021 11:18:24 -0600 Subject: [PATCH 7/9] meh --- .gitignore | 5 +++ Dockerfile | 7 +++- hardening_manifest.yaml | 12 ++++++- scripts/build-image.sh | 4 +++ scripts/importArtifacts.sh | 68 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100755 scripts/importArtifacts.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58ce6e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +archive.key +*.whl +*.jar +*.tar.gz +*.rpm \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index ca0892a..a20da8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,7 @@ RUN chown appuser:appuser -R /usr/logs # Copy required RPMs COPY *.rpm /tmp/ +COPY archive.key /tmp/ ######################################## # confluentinc/kafka-images/server @@ -67,7 +68,8 @@ ENV DOCKER_SCRIPT_DIR=/opt/caas/bin EXPOSE 8088 RUN echo "===> Installing ${COMPONENT}" \ - && dnf install --nogpgcheck -y /tmp/confluent-common-${RPM_VERSION}.noarch.rpm \ + && rpm --import /tmp/archive.key \ + && dnf install -y /tmp/confluent-common-${RPM_VERSION}.noarch.rpm \ /tmp/confluent-security-${RPM_VERSION}.noarch.rpm \ /tmp/confluent-telemetry-${RPM_VERSION}.noarch.rpm \ /tmp/confluent-hub-client-${RPM_VERSION}.noarch.rpm \ @@ -87,6 +89,7 @@ COPY cp-${COMPONENT}-operator-*_UsrBin.tar.gz /usr/bin/ ## For auditing & debugging COPY Dockerfile /etc/confluent/ironbank/ COPY hardening_manifest.yaml /etc/confluent/ironbank/ +COPY maven-artifact-3.8.1.jar /tmp/ RUN echo "===> Installing operator stuff" \ && cd /usr/share && tar -xvf *_UsrShareJava.tar.gz && rm *_UsrShareJava.tar.gz \ @@ -95,6 +98,8 @@ RUN echo "===> Installing operator stuff" \ && cd /etc && tar -xvf *_EtcKsqldb.tar.gz && rm *_EtcKsqldb.tar.gz \ && cd /opt/confluent/ksql && tar -xvf *_OptConfluentKsqlLibs.tar.gz && rm *_OptConfluentKsqlLibs.tar.gz \ && cd /usr/bin && tar -xvf *_UsrBin.tar.gz && rm *_UsrBin.tar.gz \ + && find /usr/share/java /opt/confluent/ksql -name "maven-artifact-3.6.3.jar" -execdir cp /tmp/maven-artifact-3.8.1.jar . \; -exec rm {} \; \ + && chown appuser:appuser /opt/confluent/ksql/libs/maven-artifact-3.8.1.jar && chmod 755 /opt/confluent/ksql/libs/maven-artifact-3.8.1.jar \ && chown -R appuser:appuser /etc/confluent /etc/${COMPONENT} /usr/logs /usr/bin/ksql /usr/share/java \ && mkdir -p "${KSQL_LOG4J_DIR}" "${KSQL_SECRETS_DIR}" "${KSQL_CONFIG_DIR}" "${KSQL_DATA_DIR}" "${COMPONENT_SCRIPT_DIR}/${COMPONENT}" \ && chmod -R ag+w "${KSQL_LOG4J_DIR}" "${KSQL_SECRETS_DIR}" "${KSQL_CONFIG_DIR}" "${KSQL_DATA_DIR}" "/opt" \ diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml index 074fc2a..7408dc9 100644 --- a/hardening_manifest.yaml +++ b/hardening_manifest.yaml @@ -217,6 +217,11 @@ resources: validation: type: sha256 value: 7e2622a934f04a2e5b23a355f920473ccf0144e946b433bc619a9802c1e3785c +- filename: archive.key + url: https://packages.confluent.io/rpm/6.1/archive.key + validation: + type: sha256 + value: b569e9d80bc08b65d342491f8b94e47eb7032701ce17b091f212e1072672a4d5 - filename: confluent-common-6.1.1-1.noarch.rpm url: https://packages.confluent.io/rpm/6.1/confluent-common-6.1.1-1.noarch.rpm validation: @@ -271,7 +276,12 @@ resources: url: https://ironbank-files.s3.amazonaws.com/cp-ksqldb-server-operator-6.1.1.0_UsrBin.tar.gz validation: type: sha256 - value: d76da0002ea5b36cf3ea788aaeacc55eafc5f8ab4e5807e9d6e3ecc4794066b4 + value: d76da0002ea5b36cf3ea788aaeacc55eafc5f8ab4e5807e9d6e3ecc4794066b4 +- filename: maven-artifact-3.8.1.jar + url: https://repo1.maven.org/maven2/org/apache/maven/maven-artifact/3.8.1/maven-artifact-3.8.1.jar + validation: + type: sha256 + value: 9dbd3db15ac4816471e72981cb06ef90f3ffa8be6628dddf7135f7bd69bee0c0 # List of project maintainers maintainers: diff --git a/scripts/build-image.sh b/scripts/build-image.sh index ab22f39..4826cb6 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -22,6 +22,8 @@ fi BASE_REGISTRY=${BASE_REGISTRY:-'registry1.dso.mil'} MANI='hardening_manifest.yaml' +cd ../ + ## parse manifest with yq (https://github.com/mikefarah/yq) name=$(yq e '.name' $MANI) tags=( $(yq e '.tags.[]' $MANI) ) @@ -55,6 +57,8 @@ for tag in "${tags[@]}";do echo "🏷 Tagged build with $BASE_REGISTRY/$name:$tag" done +cd scripts + end=`date +%s` duration=$((end-start)) echo "🏁 Finished $CONTAINER build (${duration} seconds)" \ No newline at end of file diff --git a/scripts/importArtifacts.sh b/scripts/importArtifacts.sh new file mode 100755 index 0000000..3f2b73d --- /dev/null +++ b/scripts/importArtifacts.sh @@ -0,0 +1,68 @@ + +#!/bin/bash + +start=`date +%s` + +if [[ -f .env ]]; then + echo "Loading in '.env' for values ..." + export $(grep -v '^#' .env | xargs) + echo " " +fi + +SCRIPTS_HOME="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +PROJECT_HOME=${PROJECT_HOME:-'../'} +PROJECT_HOME=$(realpath $PROJECT_HOME) +RESOURCE_HOME=${RESOURCE_HOME:-'../../resources'} +RESOURCE_HOME=$(realpath $RESOURCE_HOME) + +importArtifacts() { + + manifestPath=$1 + httpdomain=$2 + resourceDir=$3 + + echo -e "\nImporting ${httpdomain} artifacts ..." + + maniDir=$(dirname "$manifestPath") + + # (re)set the filelist.txt + mkdir -p $resourceDir + prefix="url: " + pushd $resourceDir + grep $httpdomain $manifestPath | while read -r line ; do + url=${line##*$prefix} + filename=$(basename "$url") + + if [[ ! -f "$filename" ]]; then + echo "downloading ${url} ..." + wget $url + else + echo "Using existing $filename. Run ./clean.sh if thats not desired." + fi + + if [[ ! -f "$maniDir/$filename" ]]; then + echo "copying ${filename} into ${maniDir} ..." + cp $filename $maniDir/ + fi + done + popd +} + +mani=$(realpath "../hardening_manifest.yaml") + +importArtifacts $mani 'ironbank-files.s3.amazonaws.com' "$RESOURCE_HOME/ironbank-files" +importArtifacts $mani 'packages.confluent.io' "$RESOURCE_HOME/confluent" +importArtifacts $mani 'files.pythonhosted.org' "$RESOURCE_HOME/pythonhosted" +importArtifacts $mani 'repo1.maven.org' "$RESOURCE_HOME/maven" + +echo "Generating hardening_manifest_template.yaml ..." +$SCRIPTS_HOME/genManifestTemplate.sh "$RESOURCE_HOME/ironbank-files" +$SCRIPTS_HOME/genManifestTemplate.sh "$RESOURCE_HOME/confluent" +$SCRIPTS_HOME/genManifestTemplate.sh "$RESOURCE_HOME/pythonhosted" + +end=`date +%s` +duration=$((end-start)) +echo "#########################" +echo "All done (${duration} seconds)" +echo "#########################" + -- GitLab From c2e83154e4fc5effe8bf8c928d04d7072276f20d Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Fri, 11 Jun 2021 10:22:26 -0600 Subject: [PATCH 8/9] update --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a20da8c..2c3aae0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ RUN pip3 install --no-index --prefix=/usr/local --upgrade --find-links /tmp/pyth RUN mkdir -p /usr/logs /licenses \ && useradd --no-log-init --create-home --shell /bin/bash appuser -COPY license.txt /licenses +COPY LICENSE /licenses RUN chown appuser:appuser -R /usr/logs -- GitLab From 106e263679639d658d2e6b03f9b90f529a2651c1 Mon Sep 17 00:00:00 2001 From: Scott Stroud Date: Mon, 14 Jun 2021 19:41:15 +0000 Subject: [PATCH 9/9] Update LICENSE --- LICENSE | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 44cfd8b..0174374 100644 --- a/LICENSE +++ b/LICENSE @@ -1 +1,70 @@ -Copyright 2020 Confluent, Inc. \ No newline at end of file +CONFLUENT LICENSE AGREEMENT +This Confluent License Agreement (“Agreement”) is a legal agreement between you (either an individual or an entity) (“Customer”) and Confluent, Inc. a Delaware corporation with offices at 101 University Avenue, Suite 111, Palo Alto, CA 94301 (“Confluent”), regarding proprietary software made available by Confluent for download in object code format (“Confluent Software”). BY CLICKING ON THE CHECKBOX THAT DEMONSTRATES ACCEPTANCE OF THIS AGREEMENT, OR BY USING THE CONFLUENT SOFTWARE, CUSTOMER EXPRESSLY ACCEPTS AND AGREES TO THE TERMS OF THIS AGREEMENT. IF YOU ARE AN INDIVIDUAL AGREEING TO THE TERMS OF THIS AGREEMENT ON BEHALF OF AN ENTITY, SUCH AS YOUR EMPLOYER, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THAT ENTITY AND “CUSTOMER” SHALL REFER HEREIN TO SUCH ENTITY. IF YOU DO NOT HAVE SUCH AUTHORITY, OR IF YOU DO NOT AGREE WITH THE TERMS OF THIS AGREEMENT, YOU MUST NOT ACCEPT THIS AGREEMENT AND MAY NOT USE THE SOFTWARE. + +1. LICENSE + +1.1 Evaluation License. Subject to the terms of this Agreement, Confluent grants to Customer a limited, non-exclusive, non-transferable license solely during the Evaluation Term (as defined below) to use the Confluent Software solely for evaluation use in development and testing environments, and not for production use. As used herein, “Evaluation Term” means thirty (30) days from the date of download by Customer or such other period specified in writing by Confluent. + +1.2 Developer License. In addition to the license granted in Section 1.1, and subject to the terms of this Agreement, Confluent grants to Customer a limited, non-exclusive, non-transferable license to use the Confluent Software on one or more clusters with a single broker per cluster, solely for evaluation use in development and testing environments, and not for production use. + +1.3 Confluent Connectors. If Customer has an active subscription to the Confluent Platform, then, except for any Confluent Connectors that are designated as “preview” releases, any Confluent Connectors that Customer downloads from Confluent’s website or otherwise receives from Confluent will be subject to the terms of Customer’s Confluent subscription agreement applicable to Confluent’s proprietary software, not the terms of this Agreement. + +1.4 Additional Restrictions and Limitations. The licenses granted herein do not include a right to sublicense. Customer shall not, and shall not permit or encourage any third party to: (a) use the Confluent Software other than as expressly permitted by this Agreement, or use the Confluent Software for third-party training, software-as-a-service, time-sharing or service bureau use or (b) disassemble, decompile or reverse engineer any portions of the Confluent Software, or otherwise attempt to gain access to the source code to such Confluent Software (or the underlying ideas, algorithms, structure or organization of the object code in the Confluent Software). The foregoing restriction is inapplicable to the extent prohibited by applicable law; provided that, in the event that Customer intends to disassemble, decompile or reverse engineer such Confluent Software, Customer shall first provide Confluent with written notice thereof. + +1.5 Reservation of Rights. Confluent reserves all rights not expressly granted in this section. No rights are granted by implication. + +1.6 Delivery of Materials. The Confluent Software, and any versions, updates or maintenance releases of any component thereof, will be delivered only through an electronic transfer. + +1.7 Support and Maintenance. Confluent is not obligated to provide maintenance and support services for the Confluent Software licensed under this Agreement. In the event that Confluent, in its sole discretion, elects to make available to Customer any updates or maintenance releases of the Confluent Software, such updates or maintenance releases shall be deemed Confluent Software under this Agreement. + +1.8 Preview Releases. Confluent may make available for download a preview release or beta version of Confluent Software, and Customer may elect to use such preview release at its sole discretion. Preview releases are intended for evaluation use in development and testing environments only, and not for production use. Confluent may discontinue providing preview releases of the Confluent Software at any time in Confluent’s sole discretion. Confluent is undertaking no obligation to release a generally available version of a preview release or any specific feature thereof. If Customer provides Confluent any suggestions, enhancement requests, recommendations, or other feedback regarding preview releases (“Feedback”), Confluent may freely use and incorporate into Confluent’s products and services any such Feedback. Feedback shall be considered Confidential Information, regardless of whether marked as such. + +2. USER AND PERFORMANCE DATA. Confluent may from time to time use and process data about Customer’s use of the Confluent Software and Third Party Software for the purposes of creating statistics and analytics data. Confluent may use such data for its own business purposes, including to maintain and improve the Confluent Software, Third Party Software and other services and to monitor and analyse its activities in connection with the performance of such services. Customer acknowledges that certain features of the Confluent Software and Third Party Software are configured to collect and report telemetry data to Confluent as more particularly described in More Information Regarding Confluent Data Collection at https://www.confluent.io/moreinformation/. Customer may choose to disable transmission of this data to Confluent at any time. Confluent will use user data in accordance with Confluent’s Privacy Policy located at https://www.confluent.io/privacy/ subject, as applicable, to the Standard Contractual Clauses for Controllers as approved by the European Commission and available at http://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A32004D0915 (as amended, superseded or updated from time to time), which are incorporated by reference in, and form an integral part of, this Agreement. + +3. OWNERSHIP. Customer acknowledges that Confluent or its licensors retain all proprietary rights, title and interest, including all intellectual property rights, in and to the Confluent Software and any changes, corrections, bug fixes, enhancements, updates and other modifications thereto (collectively, “Modifications”), and as between the parties all such rights shall vest in Confluent. Customer acknowledges that the licenses granted in Section 1 do not include the right to prepare any Modifications of the Confluent Software. + +4. CONFIDENTIALITY + +4.1 Nondisclosure and Limited Use. Each party shall retain in confidence the non-public information and know-how disclosed or made available by the other party pursuant to this Agreement which is either designated in writing as proprietary and/or confidential, if disclosed in writing, or if disclosed orally, is designated in writing (which may be via email) as confidential within thirty (30) days of the oral disclosure or should reasonably be understood to be confidential by the recipient (the “Confidential Information”). The Confluent Software, Modifications, and the terms and conditions of this Agreement shall be Confluent’s Confidential Information. Each party agrees to: (a) maintain the confidentiality of the other party’s Confidential Information; (b) refrain from using the other party’s Confidential Information except for the purpose of performing its obligations under this Agreement; and (c) not disclose such Confidential Information to any third party except to employees and subcontractors as is reasonably required in connection with the exercise of its rights and obligations under this Agreement (and only subject to binding written use and disclosure restrictions at least as protective as those set forth herein). Each party shall immediately notify the other party of any unauthorized disclosure or use of any Confidential Information and assist the other party in remedying such unauthorized use or disclosure by taking such steps as are reasonably requested by such other party. The foregoing obligations will not apply to Confidential Information of the other party which is: (i) already publicly known without breach of this Agreement; (ii) discovered or created by the receiving party without use of, or reference to, the Confidential Information of the disclosing party, as shown in records of the receiving party; (iii) otherwise known to the receiving party through no wrongful conduct of the receiving party, or (iv) required to be disclosed by law or court order; provided that the receiving party shall provide prompt notice thereof and reasonable assistance to the disclosing party to enable the disclosing party to seek a protective order or otherwise prevent or restrict such disclosure. Moreover, either party hereto may disclose any Confidential Information hereunder to such party’s agents, attorneys and other representatives (and only subject to confidentiality obligations at least as protective as those set forth herein) or any court of competent jurisdiction as reasonably required to resolve any dispute between the parties hereto. + +4.2 Remedies. Any breach or threatened breach of this Section may cause irreparable injury to the disclosing party and, in addition to any other remedies that may be available, in law, in equity or otherwise, the disclosing party shall be entitled to seek injunctive relief against the threatened breach of this Agreement or the continuation of any such breach by the receiving party, without the necessity of proving actual damages or posting any bond, in addition to any other rights or remedies provided by law. + +5. DISCLAIMER OF WARRANTIES AND LIMITATION OF LIABILITY + +5.1 Disclaimer of Any Warranties. CONFLUENT MAKES NO WARRANTIES, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE CONFLUENT SOFTWARE OR ANY OTHER MATERIALS PROVIDED HEREUNDER. CONFLUENT SPECIFICALLY DISCLAIMS ALL OTHER WARRANTIES, EXPRESS AND IMPLIED, INCLUDING WITHOUT LIMITATION THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT AND THOSE ARISING FROM A COURSE OF DEALING OR USAGE OR TRADE, AND ALL SUCH WARRANTIES ARE HEREBY EXCLUDED TO THE FULLEST EXTENT PERMITTED BY LAW. THE CONFLUENT SOFTWARE IS PROVIDED ON AN “AS IS” AND “AS AVAILABLE” BASIS. + +5.2 Limitation of Liability. IN NO EVENT SHALL CONFLUENT’S LIABILITY ARISING UNDER THIS AGREEMENT EXCEED $500. CONFLUENT WILL NOT BE LIABLE TO CUSTOMER FOR ANY CONSEQUENTIAL, INCIDENTAL, SPECIAL, INDIRECT, PUNITIVE OR EXEMPLARY DAMAGES, INCLUDING WITHOUT LIMITATION LOST PROFITS, BUSINESS, CONTRACTS, REVENUE, GOODWILL, PRODUCTION, ANTICIPATED SAVINGS, LOSS OF DATA, OR COSTS OF PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY CLAIM OR DEMAND BY ANY OTHER PARTY, HOWEVER CAUSED AND (TO THE FULLEST EXTENT PERMITTED BY LAW) UNDER ANY THEORY OF LIABILITY (INCLUDING NEGLIGENCE) EVEN IF CONFLUENT HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. CUSTOMER ACKNOWLEDGES THAT THESE LIMITATIONS SHALL APPLY NOTWITHSTANDING ANY FAILURE OF ESSENTIAL PURPOSE OF ANY LIMITED REMEDY. + +6. TERM AND TERMINATION + +6.1 Term. Unless earlier terminated as provided in this Section, this Agreement and the license granted hereunder will be effective as of the Effective Date and will immediately terminate if Customer breaches Section 1 of this Agreement, regardless of whether Confluent notifies Customer of such termination. + +6.2 Termination. Either party shall have the right to terminate this Agreement and the license granted herein upon written notice in the event the other party fails to perform or observe any material term or condition of this Agreement and such default has not been cured no later than ten (10) days after written notice of such default to the other party. Confluent may also terminate this Agreement immediately if the Customer: (a) terminates or suspends its business; (b) becomes subject to any bankruptcy or insolvency proceeding under Federal or state statute; (c) becomes insolvent or subject to direct control by a trustee, receiver or similar authority; or (d) has wound up or liquidated, voluntarily or otherwise. + +6.3 Effect of Termination. The provisions of this Agreement that by their nature extend beyond the termination of this Agreement will survive termination. All of Customer’s rights in the Confluent Software will terminate immediately upon termination of this Agreement. No later than five (5) days after termination of this Agreement, Customer shall return to Confluent or, upon Confluent’s request, destroy or render inaccessible, at Customer’s sole expense, all Confidential Information of Confluent and materials containing any Confidential Information of Confluent, and discontinue use of and uninstall the Confluent Software, including all copies thereof. Nothing contained herein shall limit any other remedies that Confluent may have for the default of Customer under this Agreement nor relieve Customer of any of its obligations incurred prior to such termination. + +7. MISCELLANEOUS + +7.1 Assignment. Customer shall not assign or otherwise transfer this Agreement or any rights or obligations hereunder, in whole or in part, whether by operation of law or otherwise, to any third party without Confluent’s prior written consent. Any purported transfer, assignment or delegation without such prior written consent will be null and void and of no force or effect. Confluent shall have the right to assign this Agreement to any successor to its business or assets to which this Agreement relates, whether by merger, sale of assets, sale of stock, reorganization or otherwise. Subject to this Section, this Agreement shall be binding upon and inure to the benefit of the parties hereto, and their respective successors and permitted assigns. + +7.2 Entire Agreement; Modification; Waiver. This Agreement represents the entire agreement between the parties, and supersedes all prior agreements and understandings, written or oral, with respect to the matters covered by this Agreement, and is not intended to confer upon any third party any rights or remedies hereunder. Customer acknowledges that it has not entered in this Agreement based on any representations other than those contained herein. No modification of or amendment to this Agreement, nor any waiver of any rights under this Agreement, shall be effective unless in writing and signed by both parties. The waiver of one breach or default or any delay in exercising any rights shall not constitute a waiver of any subsequent breach or default. + +7.3 Third Party Software. Confluent also makes available certain third party open source software as identified at http://www.confluent.io/third_party_software (“Third Party Software”). The Third Party Software shall be subject to the applicable open source license(s) and not this Agreement, and is provided by Confluent at no charge. To the extent the terms of open source licenses applicable to Third Party Software prohibit any of the restrictions in this Agreement, such restrictions will not apply to such Third Party Software. To the extent the terms of open source licenses applicable to Third Party Software require Confluent to make an offer to provide source code or related information in connection with the Third Party Software, such offer is made. + +7.4 Governing Law. This Agreement shall in all respects be governed by the laws of the State of California without reference to its principles of conflicts of laws. The parties hereby agree that all disputes arising out of this Agreement shall be subject to the exclusive jurisdiction of and venue in the federal and state courts within Santa Clara County, California. Customer hereby consents to the personal and exclusive jurisdiction and venue of these courts. The parties hereby disclaim and exclude the application hereto of the United Nations Convention on Contracts for the International Sale of Goods. + +7.5 Severability. If any provision of this Agreement is held invalid or unenforceable under applicable law by a court of competent jurisdiction, it shall be replaced with the valid provision that most closely reflects the intent of the parties and the remaining provisions of the Agreement will remain in full force and effect. + +7.6 Relationship of the Parties. Nothing in this Agreement is to be construed as creating an agency, partnership, or joint venture relationship between the parties hereto. Neither party shall have any right or authority to assume or create any obligations or to make any representations or warranties on behalf of any other party, whether express or implied, or to bind the other party in any respect whatsoever. + +7.7 Notices. All notices permitted or required under this Agreement shall be in writing and shall be deemed to have been given when delivered in person (including by overnight courier), or three (3) business days after being mailed by first class, registered or certified mail, postage prepaid, to the address of the party specified in this Agreement or such other address as either party may specify in writing. + +7.8 U.S. Government Restricted Rights. If Confluent Software is being licensed by the U.S. Government, the Confluent Software is deemed to be “commercial computer software” and “commercial computer documentation” developed exclusively at private expense, and (a) if acquired by or on behalf of a civilian agency, shall be subject solely to the terms of this computer software license as specified in 48 C.F.R. 12.212 of the Federal Acquisition Regulations and its successors; and (b) if acquired by or on behalf of units of the Department of Defense (“DOD”) shall be subject to the terms of this commercial computer software license as specified in 48 C.F.R. 227.7202-2, DOD FAR Supplement and its successors. + +7.9 Export Law Assurances. Customer understands that the Confluent Software is subject to export control laws and regulations. Customer may not download or otherwise export or re-export the Confluent Software or any underlying information or technology except in full compliance with all applicable laws and regulations, in particular, but without limitation, United States export control laws. None of the Confluent Software or any underlying information or technology may be downloaded or otherwise exported or re- exported: (a) into (or to a national or resident of) any country to which the United States has embargoed goods; or (b) to anyone on the U.S. Treasury Department’s list of specially designated nationals or the U.S. Commerce Department’s list of prohibited countries or debarred or denied persons or entities. Customer hereby agrees to the foregoing and represents and warrants that customer is not located in, under control of, or a national or resident of any such country or on any such list. + +7.10 Construction. The titles and section headings used in this Agreement are for ease of reference only and shall not be used in the interpretation or construction of this Agreement. No rule of construction resolving any ambiguity in favor of the non-drafting party shall be applied hereto. The word “including”, when used herein, is illustrative rather than exclusive and means “including, without limitation.” + +(v. March 2019) + +Link: https://www.confluent.io/confluent-software-evaluation-license/ -- GitLab