diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000000000000000000000000000000000000..098c399ffe3dc1d2e33ce9bf7781e7dce0e6de67 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +/bin/ +.classpath +.project +/src/ + + diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000000000000000000000000000000000000..98a77b89f9a9ce8ffe9124c7674f38e2f9e1261f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,106 @@ +################################################################################ +# +################################################################################ +ARG BASE_REGISTRY=registry1.dsop.io +ARG BASE_IMAGE=redhat/ubi/ubi7 +ARG BASE_TAG=7.9 + +FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} + +# Set user +USER 0 + +# Set the Baseline Image +RUN yum -y update && yum -y install hostname fontconfig java-11-openjdk-devel + +# Set up working directory for Kinetica +RUN mkdir -p /mnt/data/gpudb +RUN mkdir /mnt/data/gpudb/etc +RUN mkdir /mnt/data/gpudb/logs +RUN mkdir /mnt/data/gpudb/persist + +# Staging GPG key for rpm +COPY RPM-GPG-KEY-PK /etc/pki/rpm-gpg/ +RUN gpg --import /etc/pki/rpm-gpg/* + +# Stage RPMS to install +COPY msttcorefont-1-1-signed.x86_64.rpm /tmp +COPY libmspack-0.5-0.8.alpha.el7.x86_64.rpm /tmp +COPY openssh-server-7.4p1-21.el7.x86_64.rpm /tmp +COPY numactl-libs-2.0.12-5.el7.x86_64.rpm /tmp +COPY fuse-2.9.2-11.el7.x86_64.rpm /tmp +COPY gpudb-intel-license-7.0.20.5.20210428141653-0-signed.el7.x86_64.rpm /tmp +COPY gpudbsql-7.0.0.0.jar /tmp + +# Installing Required RPMS +RUN yum -y install /tmp/fuse-2.9.2-11.el7.x86_64.rpm +RUN yum -y install /tmp/numactl-libs-2.0.12-5.el7.x86_64.rpm +RUN yum -y install /tmp/openssh-server-7.4p1-21.el7.x86_64.rpm +RUN yum -y install /tmp/libmspack-0.5-0.8.alpha.el7.x86_64.rpm +RUN yum -y install --nogpgcheck /tmp/msttcorefont-1-1-signed.x86_64.rpm + +# Update font cache after msttcore fonts are installed +RUN fc-cache -v + +# Install Kinetica DB +RUN yum -y install --nogpgcheck /tmp/gpudb-intel-license-7.0.20.5.20210428141653-0-signed.el7.x86_64.rpm + +# Applying REVEAL Code +COPY caravel_all_2.tar.gz /tmp +RUN tar -zxvf /tmp/caravel_all_2.tar.gz -C /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/ +COPY caravel.db /opt/gpudb/connectors/reveal/var/ + +# change working directory +WORKDIR /opt + +# moving config and scripts +RUN mv /opt/gpudb/core/etc/gpudb.conf /opt/gpudb/core/etc/gpudb.orig +COPY config/gpudb.conf /opt/gpudb/core/etc/gpudb.conf +RUN chown gpudb:gpudb /opt/gpudb/core/etc/gpudb.conf + +# move start scripts +COPY scripts/start.sh . +RUN chmod 755 /opt/start.sh + +# Fixing gpudbsql jar CVE finding +RUN mv -f /tmp/gpudbsql-7.0.0.0.jar /opt/gpudb/sql/ #remove origina JAR with outdated commons-io dependency + +# cleanup /tmp folder +RUN rm -rf /tmp/* + +# fix permission issue for now +RUN chown -R gpudb:gpudb /opt/gpudb/ +RUN chown -R gpudb:gpudb /mnt/data/gpudb/ + +# Fixing CVE finding +RUN chmod g-s /usr/bin/{fusermount,ksu,ssh-agent} +RUN chmod u-s /usr/bin/{fusermount,ksu,ssh-agent} +RUN chmod g-s /usr/libexec/openssh/ssh-keysign +RUN chmod u-s /usr/libexec/openssh/ssh-keysign +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/anadarko_calculator/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/anadarko_histogram/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/anadarko_linecharts/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/attribute_selection/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/kinetica_3d/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/kinetica_bar/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/kinetica_map/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/kinetica_mapbox/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/kinetica_timeframe/node_modules/* +RUN rm -rf /opt/gpudb/connectors/reveal/lib/python3.7/site-packages/caravel/slices/pivot_table/node_modules/* +RUN rm -rf /opt/gpudb/stats/lib/python3.7/site-packages/Django-2.2.20-py3.7.egg-info +RUN rm -rf /opt/gpudb/httpd +RUN rm -rf /opt/gpudb/connectors/reveal/lib/node_modules/npm/node_modules/path-parse +RUN yum remove -y libX11 libX11-common +RUN rpm -e --nodeps cups-libs +RUN rpm --setugids gpudb-intel-license + +# Expose ports +EXPOSE 8080 8088 + +# Start the Service +CMD ["/opt/start.sh"] + +# Changing to Kinetica User +USER gpudb + +HEALTHCHECK --interval=5m --timeout=30s --start-period=1m --retries=3 CMD [ "curl -f http://localhost:8080" || exit 1 ] diff --git a/Kinetica License Notice - April 2021.pdf b/Kinetica License Notice - April 2021.pdf new file mode 100755 index 0000000000000000000000000000000000000000..a43667ace2d4a366d636f09b92d80c4109320a57 Binary files /dev/null and b/Kinetica License Notice - April 2021.pdf differ diff --git a/LICENSE b/LICENSE new file mode 100755 index 0000000000000000000000000000000000000000..115ea5a9e5845952b7aece729b6fc27a5e84b121 --- /dev/null +++ b/LICENSE @@ -0,0 +1,2 @@ +License File in PDF "Kinetica License Notice" + diff --git a/README.md b/README.md index 5dc6fa6db4361c22da2f35edf0544d83ba6001e2..2415a05baa83ba93c1307b3e1705772625985c23 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ -# +## REVEAL with Kinetica DB -Project template for all Iron Bank container repositories. \ No newline at end of file +## Description + +Reveal is a visual data exploration and insight discovery tool. It allows users who may or may not have extensive experience with big data or data analysis to quickly build charts and graphs in a meaningful way to explore their dataset. Reveal was designed to be interactive and easy to use. + +Reveal is a Python Flask web application with a mostly React frontend. By default, it uses SQLite as its application data storage engine. To begin visualizing data, Reveal must first be connected to your RDBMS datastore. While Reveal can and does work with other flavors of databases, it is most powerful when connected to Kinetica. Kinetica allows this through a custom ODBC driver/connector via Reveal's SQLAlchemy interface + + +https://docs.kinetica.com/7.1/analytics/reveal/ diff --git a/config/gpudb.conf b/config/gpudb.conf new file mode 100644 index 0000000000000000000000000000000000000000..661175c9afa68abe1c5d65506f6456f920ea7d6c --- /dev/null +++ b/config/gpudb.conf @@ -0,0 +1,174 @@ +# ============================================================================== +# Kinetica configuration file. +# ============================================================================== + +[gaia] + +# ============================================================================== +# Network + +# Head HTTP server IP address. +# Set to the publicly accessible IP address of the first process, **rank0**. + +head_ip_address = 127.0.0.1 + +# Head HTTP server port to use for 'head_ip_address'. + +head_port = 9191 + +# Set to "true" to use HTTPS; if "true" then 'https_key_file' and +# 'https_cert_file' must be provided +use_https = false + +# Files containing the SSL private Key and the SSL certificate for. +# If required, a self signed certificate (expires after 10 years) can be +# generated via the command: +# +## openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout key.pem -out cert.pem +# +https_key_file = +https_cert_file = + +# Value to return via Access-Control-Allow-Origin HTTP header +# (for Cross-Origin Resource Sharing). +# Set to empty to not return the header and disallow CORS. +http_allow_origin = * + +# Start an HTTP server as a proxy to handle LDAP and/or Kerberos authentication. +# Each host will run an HTTP server and access to each rank is available through +# http://host:8082/gpudb-1, where port "8082" is defined by 'httpd_proxy_port'. +# +# NOTE: HTTP external endpoints are not affected by the 'use_https' parameter +# above. If you wish to enable HTTPS, you must edit the +# "/opt/gpudb/httpd/conf/httpd.conf" and setup HTTPS as per the Apache +# httpd documentation at https://httpd.apache.org/docs/2.2/ +enable_httpd_proxy = false + + +# TCP port that the httpd auth proxy server will listen on if +# 'enable_httpd_proxy' is "true". +httpd_proxy_port = 8082 + +# Set to "true" if the httpd auth proxy server is configured to use HTTPS. +httpd_proxy_use_https = false + +# Internal use IP address of the head HTTP server, **rank0**. +# Set to either a second internal network accessible by all ranks or to +# '${gaia.head_ip_address}'. +rank0_ip_address = ${gaia.rank0.host} + +# Trigger ZMQ publisher server port ("-1" to disable), uses the +# 'head_ip_address' interface. +trigger_port = 9001 + +# Set monitor ZMQ publisher server port (-1 to disable), uses the 'head_ip_address' interface. +set_monitor_port = 9002 + +# Set monitor ZMQ publisher internal proxy server port ("-1" to disable), uses +# the 'head_ip_address' interface. +# +# IMPORTANT: Disabling this port effectively prevents worker nodes from +# publishing set monitor notifications when multi-head ingest is enabled (see +# 'enable_worker_http_servers'). +set_monitor_proxy_port = 9003 + +# Enable Reveal runtime +enable_reveal = true + +# Internal communication ports +global_manager_port_one = 5552 + +# Host manager synchronization port +global_manager_pub_port = 5553 + +# HTTP port for web portal of the host manager +host_manager_http_port = 9300 + +# Enable worker HTTP servers; each process runs its own server for multi-head +# ingest. +enable_worker_http_servers = false + +# Optionally, specify the worker HTTP server ports. +# The default is to use ('head_port' + *rank #*) for each worker process where +# rank number is from "1" to number of ranks in 'rank<#>.host' below. + +#rank1.worker_http_server_port = 9192 +#rank2.worker_http_server_port = 9193 + + +# Optionally, specify a public URL for each worker HTTP server that clients +# should use to connect for multi-head operations. +# +# NOTE: If specified for any ranks, a public URL must be specified for all +# ranks. + +#rank0.public_url = +#rank1.public_url = +#rank2.public_url = + +# Specify the hosts to run each rank worker process in the cluster. +# For a single machine system, use "127.0.0.1", but if using two or more +# machines, a hostname or IP address must be specified for each rank that is +# accessible from the other ranks. See also 'head_ip_address' and +# 'rank0_ip_address'. + +rank0.host = 127.0.0.1 +rank1.host = 127.0.0.1 +rank2.host = 127.0.0.1 + +# Specify the TCP ports each rank will use to communicate with the others. +# If the port for any 'rank<#>' is not specified the port will be assigned to +# 'rank0.communicator_port' + *rank #*. + +rank0.communicator_port = 6555 +#rank1.communicator_port = 6556 +#rank2.communicator_port = 6557 + + +# Enables compression of inter-node network data transfers. +compress_network_data = false + + +# ============================================================================== +# Security + +# Require authentication. +require_authentication = false + +# Enable authorization checks. +enable_authorization = false + +# Minimum password length. +min_password_length = 0 + +# Enable external (LDAP, Kerberos, etc.) authentication. User IDs of +# externally-authenticated users must be passed in via the "REMOTE_USER" HTTP +# header from the authentication proxy. May be used in conjuntion with the +# 'enable_httpd_proxy' setting above for an integrated external authentication +# solution. +# +# IMPORTANT: DO NOT ENABLE unless external access to GPUdb ports +# has been blocked via firewall AND the authentication proxy is +# configured to block "REMOTE_USER" HTTP headers passed in from clients. +enable_external_authentication = false + +# Key that, if specified, must be passed in via the "KINETICA_HANDSHAKE_KEY" +# HTTP header from the authentication proxy if a "REMOTE_USER" HTTP header is +# also passed in. A missing or incorrect handshake key will result in rejection +# of the request. +external_authentication_handshake_key = + +# Automatically create accounts for externally-authenticated users. +# If 'enable_external_authentication' is "false", this setting has no effect. +# Note that accounts are not automatically deleted if users are removed +# from the external authentication provider and will be orphaned. +auto_create_external_users = false + +# Automatically add roles passed in via the "KINETICA_ROLES" HTTP header to +# externally-authenticated users. Specified roles that do not exist are +# ignored. If 'enable_external_authentication' is "false", this setting has no +# effect. +# +# IMPORTANT: DO NOT ENABLE unless the authentication proxy is +# configured to block "KINETICA_ROLES" HTTP headers passed in from clients. +auto_grant_external_roles = false diff --git a/hardening_manifest.yaml b/hardening_manifest.yaml new file mode 100755 index 0000000000000000000000000000000000000000..772a722a72d12e185cd659d538987e5bb53b32f7 --- /dev/null +++ b/hardening_manifest.yaml @@ -0,0 +1,139 @@ +--- +apiVersion: v1 + +# The repository name in registry1, excluding /ironbank/ +name: "kinetica/pathfinder/reveal" + +# List of tags to push for the repository in registry1 +# The most specific version should be the first tag and will be shown +# on ironbank.dso.mil +tags: +- "7.0.20.5.20210428141653" +- "latest" + +# Build args passed to Dockerfile ARGs +args: + BASE_IMAGE: "redhat/ubi/ubi7" + BASE_TAG: "7.9" + +# Docker image labels +labels: + # Name of the image + org.opencontainers.image.title: "reveal" + # Human-readable description of the software packaged in the image + org.opencontainers.image.description: "kinetica container with reveal included" + # License(s) under which contained software is distributed + org.opencontainers.image.licenses: "Government Purpose Rights" + # URL to find more information on the image + org.opencontainers.image.url: "https://www.kinetica.com/" + # Name of the distributing entity, organization or individual + org.opencontainers.image.vendor: "Kinetica" + # Authoritative version of the software + org.opencontainers.image.version: "7.0.20.5.20210428141653" + # Keywords to help with search (ex. "cicd,gitops,golang") + mil.dso.ironbank.image.keywords: "reveal" + # This value can be "opensource" or "commercial" + mil.dso.ironbank.image.type: "commercial" + # Product the image belongs to for grouping multiple images + mil.dso.ironbank.product.name: "kinetica/reveal" + +resources: + - url: s3://eightynine-bucket/gpudb-intel-license-7.0.20.5.20210428141653-0-signed.el7.x86_64.rpm + filename: gpudb-intel-license-7.0.20.5.20210428141653-0-signed.el7.x86_64.rpm + validation: + type: sha256 + value: 4bc19f495baa221821cffe9dd7cf503759d8b2b06acb8bf6882959842836c9bb + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/msttcorefont-1-1-signed.x86_64.rpm + filename: msttcorefont-1-1-signed.x86_64.rpm + validation: + type: sha256 + value: af8d805b8b733cc3603c68fc43036521492c8687232d49c0561943bd35fb6126 + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/caravel.db + filename: caravel.db + validation: + type: sha256 + value: daac6ae9cbc8e1dcf274581be8b3463c8e96f20612d001ddd988f35cc35025f9 + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/fuse-2.9.2-11.el7.x86_64.rpm + filename: fuse-2.9.2-11.el7.x86_64.rpm + validation: + type: sha256 + value: 078494302e9d4c4ce914f5681c583b9ab571f3e13537c61e4a6db711528d7fcb + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/numactl-libs-2.0.12-5.el7.x86_64.rpm + filename: numactl-libs-2.0.12-5.el7.x86_64.rpm + validation: + type: sha256 + value: bd4df28dbd6928faf3bc3bb48de9c511cf623c78d95c0af7cc4cdb5b1818ea7a + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/openssh-server-7.4p1-21.el7.x86_64.rpm + filename: openssh-server-7.4p1-21.el7.x86_64.rpm + validation: + type: sha256 + value: d4b2de8f877b5c86b4c6751fbfba32d805bff4ecbb6183831888c833fb1af967 + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/libmspack-0.5-0.8.alpha.el7.x86_64.rpm + filename: libmspack-0.5-0.8.alpha.el7.x86_64.rpm + validation: + type: sha256 + value: 1fcc64016b2bf4ae5ab2657b3acd6eb3be1cfa9e19b2972df8aaf1a552db5b3a + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/caravel_all_2.tar.gz + filename: caravel_all_2.tar.gz + validation: + type: sha256 + value: 797e1f20eb9fa1f4b5f79e78905725f5403bbf112a34322e1b921a863a922138 + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/RPM-GPG-KEY-PK + filename: RPM-GPG-KEY-PK + validation: + type: sha256 + value: 895c34aedec8b1b8728ba5f5a05d02e5ae4e2d41eaefc6380507117bfcc79d96 + auth: + id: pathfinder-credential + region: us-east-1 + + - url: s3://eightynine-bucket/gpudbsql-7.0.0.0.jar + filename: gpudbsql-7.0.0.0.jar + validation: + type: sha256 + value: 05b7be15018205e632e41870146579203a495f00f5ad640af36df883c793e620 + auth: + id: pathfinder-credential + region: us-east-1 + +# List of project maintainers +maintainers: +- email: "kla.ctr@kinetica.com" + # The name of the current container owner + # name: "Jane Dow" + name: "Kevin La" + # The gitlab username of the current container owner + # username: "jdow" + username: "kla" diff --git a/scripts/remediation.sh b/scripts/remediation.sh new file mode 100644 index 0000000000000000000000000000000000000000..17ae43f77a4beb020b567c26ee403d7b7f8daa7d --- /dev/null +++ b/scripts/remediation.sh @@ -0,0 +1,20 @@ +# Declare array to hold set of RPM packages we need to correct permissions for +declare -A SETPERMS_RPM_DICT + +# Create a list of files on the system having permissions different from what +# is expected by the RPM database +readarray -t FILES_WITH_INCORRECT_PERMS < <(rpm -Va --nofiledigest | awk '{ if (substr($0,6,1)=="U" || substr($0,7,1)=="G") print $NF }') + +for FILE_PATH in "${FILES_WITH_INCORRECT_PERMS[@]}" +do + RPM_PACKAGE=$(rpm -qf "$FILE_PATH") + # Use an associative array to store packages as it's keys, not having to care about duplicates. + SETPERMS_RPM_DICT["$RPM_PACKAGE"]=1 +done + +# For each of the RPM packages left in the list -- reset its permissions to the +# correct values +for RPM_PACKAGE in "${!SETPERMS_RPM_DICT[@]}" +do + rpm --setugids "${RPM_PACKAGE}" +done diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..3fd3dd7456c65198b3697a6a2e43bb3cbc6c9f4a --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# start gpudb +/etc/init.d/gpudb start + +# keep container running +tail -f /dev/null