From cb93bff467d8c6b61ebd41a6a91313798d91298a Mon Sep 17 00:00:00 2001
From: Warren Kim <warren.kim@falkonry.com>
Date: Mon, 25 Oct 2021 15:56:22 -0700
Subject: [PATCH] initial app porting commit
---
Dockerfile | 67 +++++++++++++++++
build.sh | 205 +++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 272 insertions(+)
create mode 100644 Dockerfile
create mode 100644 build.sh
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..ddce0de
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,67 @@
+FROM registry.access.redhat.com/ubi8/ubi
+
+LABEL com.redhat.component="ubi8-container" \
+ name="ubi8" \
+ version="8.4"
+
+#label for EULA
+LABEL com.redhat.license_terms="https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI"
+
+#labels for container catalog
+LABEL summary="Provides the latest release of Red Hat Universal Base Image 8."
+LABEL description="The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly."
+LABEL io.k8s.display-name="Red Hat Universal Base Image 8"
+LABEL io.openshift.expose-services=""
+LABEL io.openshift.tags="base rhel"
+ENV container oci
+
+#install curl and jq
+RUN yum install -y \
+ wget \
+ jq \
+ curl \
+ gnupg2 \
+ java-1.8.0-openjdk \
+ yum-utils \
+ crun
+
+RUN dnf -y update; rpm --restore shadow-utils 2>/dev/null; \
+yum -y install podman fuse-overlayfs --exclude container-selinux; \
+rm -rf /var/cache /var/log/dnf* /var/log/yum.*
+
+RUN useradd podman; \
+echo podman:10000:5000 > /etc/subuid; \
+echo podman:10000:5000 > /etc/subgid;
+
+VOLUME /var/lib/containers
+VOLUME /home/podman/.local/share/containers
+RUN mkdir -p /home/podman/.local/share/containers
+
+ADD https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/containers.conf /etc/containers/containers.conf
+ADD https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/podman-containers.conf /home/podman/.config/containers/containers.conf
+
+RUN chown podman:podman -R /home/podman
+
+# chmod containers.conf and adjust storage.conf to enable Fuse storage.
+RUN chmod 644 /etc/containers/containers.conf; sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' /etc/containers/storage.conf
+RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers /var/lib/shared/vfs-images /var/lib/shared/vfs-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock; touch /var/lib/shared/vfs-images/images.lock; touch /var/lib/shared/vfs-layers/layers.lock
+
+ENV _CONTAINERS_USERNS_CONFIGURED=""
+
+# Adding podman as docker alias
+# Fedora/CentOS/RedHat
+RUN echo "alias docker=podman" >> /etc/environment
+RUN echo "export JAVA_HOME=$(readlink -f /usr/bin/java | sed 's:bin/java::')" >> /etc/environment
+
+ENV HADOOP_VERSION 3.2.2
+ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
+RUN wget $HADOOP_URL -O hadoop-$HADOOP_VERSION.tar.gz
+RUN tar -xzf hadoop-$HADOOP_VERSION.tar.gz -C /usr/local
+ENV PATH /usr/local/hadoop-$HADOOP_VERSION/bin/:$PATH
+RUN cp -R /usr/local/hadoop-$HADOOP_VERSION/lib/native/* /usr/lib/
+RUN cp -R /usr/local/hadoop-$HADOOP_VERSION/share/hadoop/tools/lib/* /usr/local/hadoop-$HADOOP_VERSION/share/hadoop/common/lib/
+
+COPY EdgeStandaloneDockerfile /edgebuilder/EdgeStandaloneDockerfile
+COPY build.sh /edgebuilder/build.sh
+RUN cd /edgebuilder
+CMD ["/bin/bash","/edgebuilder/build.sh"]
\ No newline at end of file
diff --git a/build.sh b/build.sh
new file mode 100644
index 0000000..7008027
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,205 @@
+#!/bin/bash
+shopt -s expand_aliases
+source /etc/environment
+set -x
+
+# Docker quay.io registry login
+mkdir -p /run/containers/0/
+echo $QUAY_AUTH_TOKEN > /run/containers/0/auth.json
+
+cd /edgebuilder
+mkdir -p /edge/standalone
+
+if [ $# -gt 1 ]; then
+ echo "command line contains $# arguments and arguments are: $@ "
+else
+ echo "command line should have 2 arguments, first one for jobUrl and second one is image type [amd64, arm32v7]"
+ exit 1
+fi
+
+falkonry_edge_job_url=$1
+falkonry_edge_image_type=$2
+falkonry_edge_image_type_prefix=${falkonry_edge_image_type}
+
+
+if [ -n "$falkonry_kube_configmountpath" ]; then
+ cp -a $falkonry_kube_configmountpath/. /usr/local/hadoop-$HADOOP_VERSION/etc/hadoop
+fi
+
+#default image is amd64 which does not have any prefix in image name, other images will have image type - as prefix
+if [ ${falkonry_edge_image_type} == "amd64" ]; then
+ falkonry_edge_image_type_prefix=""
+else
+ falkonry_edge_image_type_prefix="${falkonry_edge_image_type}-"
+fi
+
+echo "riactor export dir: ${falkonry_tercel_riactor_export_directory} job url: ${falkonry_edge_job_url} edge version: ${falkonry_tercel_edge_kubernetes_edge_version} edge image type: ${falkonry_edge_image_type}"
+
+#write job to a file and extract modelId
+echo "Writing ${falkonry_edge_job_url} content to job.json file"
+curl ${falkonry_edge_job_url} > /job.json
+
+job_id=$(cat /job.json | jq -r '.id')
+tenant_id=$(cat /job.json | jq -r '.tenant')
+model_id=$(cat /job.json | jq -r '.model')
+datastream_id=$(cat /job.json | jq -r '.datastream')
+assessment_id=$(cat /job.json | jq -r '.assessment')
+
+echo "Extracted job id: ${job_id}, tenant id: ${tenant_id}, model id: $model_id, datastream id: $datastream_id, assessment id: $assessment_id from job.json"
+
+#write model application object to a file to extract modelType
+falkonry_edge_model_url="${falkonry_tercel_riactor_process_dataservice_url}/api/1.1/accounts/$tenant_id/datastreams/$datastream_id/assessments/$assessment_id/models/$model_id"
+echo "Fetching model: ${falkonry_edge_model_url}"
+curl ${falkonry_edge_model_url} > /app-model.json
+model_type=$(cat /app-model.json | jq -r '.modelType')
+echo "Extracted model type: ${model_type}"
+
+#for eventhorizon analyzers, there are two models
+if [ "$model_type" = "EVENTHORIZON" ];
+then
+ condition_model_id=$(cat /app-model.json | jq -r '.details.classificationModel')
+ falkonry_edge_eventhorizon_model_path=$(cat /app-model.json | jq -r '.modelPath')
+ mkdir -p /edgebuilder/edge/eventhorizon
+ hadoop distcp -update ${falkonry_edge_eventhorizon_model_path} file:///edgebuilder/edge/eventhorizon/model
+else
+ condition_model_id=$model_id
+fi
+
+if [ -n "$falkonry_kube_configmountpath" ];
+then
+ falkonry_edge_process_model_path="${falkonry_edge_model_directory}/${tenant_id}/${condition_model_id}.json"
+ falkonry_edge_process_feature_path="${falkonry_edge_model_directory}/${tenant_id}/${condition_model_id}"
+else
+ falkonry_edge_process_model_path="hdfs://falkonry-hadoop:9000/tercel/model/${tenant_id}/${condition_model_id}.json"
+ falkonry_edge_process_feature_path="hdfs://falkonry-hadoop:9000/tercel/model/${tenant_id}/${condition_model_id}"
+fi
+
+# set analyzerType in spec
+jq ".spec.analyzerType = \"${model_type}\"" /job.json > /job2.json
+mv /job2.json /job.json #jq can't modify file in place
+
+#from environment variables first find all falkonry_edge* or process_* environment variables
+# and prefix it with ENV space enviroment variables
+#envs=`printenv | grep -v -i -E "falkonry_*|process_*|KUBERNETES*|HADOOP*" | paste -d' ' -s`
+envs=`printenv | grep -i -E "^^pwd*|^path*" | paste -d' ' -s`
+
+echo "printing content of final docker file"
+cat EdgeStandaloneDockerfile
+
+mkdir -p /edgebuilder/edge
+
+#copy spec and model file to standalone dir
+hadoop distcp -update file:///job.json file:///edgebuilder/edge/standalone/spec.json
+hadoop distcp -update ${falkonry_edge_process_model_path} file:///edgebuilder/edge/standalone/model.json
+hadoop distcp -update ${falkonry_edge_process_feature_path} file:///edgebuilder/edge/standalone/${condition_model_id}
+
+#replace image registry if set in barbary
+if [[ ! -z ${falkonry_tercel_riactor_process_kubernetes_live_image} ]]; then
+ sed -i "s#quay.io/falkonry/edge#${falkonry_tercel_riactor_process_kubernetes_live_image}#g" EdgeStandaloneDockerfile
+fi
+
+#replace edge base image version in docker file
+sed -i.bak "s/master.latest/${falkonry_edge_image_type_prefix}${falkonry_tercel_edge_kubernetes_edge_version}/" EdgeStandaloneDockerfile
+
+echo "model_type=$model_type"
+model_type_short="condition"
+#event horizon analyzer uses a different base image and then copies in both edge and ml-ops distribution code
+if [ "$model_type" = "EVENTHORIZON" ]; then
+ # Change base image
+
+ sed -i.bak "s/FROM.*/FROM openjdk:8/g" EdgeStandaloneDockerfile
+ model_type_short="ehe"
+ # Copy edge code
+ falkonry_edge_image="quay.io/falkonry/edge:${falkonry_tercel_edge_kubernetes_edge_version}"
+ edge_container_id=$(docker create ${falkonry_edge_image})
+ docker cp "${edge_container_id}:/edge" /edgebuilder/edge-tmp
+ mv /edgebuilder/edge-tmp/* /edgebuilder/edge/
+
+ # Copy ml-ops code
+ falkonry_mlops_version="${falkonry_tercel_riactor_process_kubernetes_mlops_version:-master.latest}"
+ falkonry_mlops_image="quay.io/falkonry/ml-ops:${falkonry_mlops_version}"
+ mlops_container_id=$(docker create ${falkonry_mlops_image})
+ docker cp "${mlops_container_id}:/src" /edgebuilder/edge/eventhorizon/
+ docker cp "${mlops_container_id}:/requirements.txt" /edgebuilder/edge/eventhorizon/
+
+ # Additional docker commands for installing python, python libraries, and above code
+ echo "RUN apt-get update && apt-get install -y -q python3 python3-pip && apt-get clean && pip3 install --upgrade pip" >> EdgeStandaloneDockerfile
+ echo "COPY edge /edge" >> EdgeStandaloneDockerfile
+ echo "RUN pip3 install -r /edge/eventhorizon/requirements.txt" >> EdgeStandaloneDockerfile
+ echo "CMD /edge/Run-EventHorizonAnalyzer.sh" >> EdgeStandaloneDockerfile
+fi
+
+#build docker edge standalone image
+edge_standalone_image=falkonry-${model_type_short}-analyzer-${falkonry_edge_image_type}-${model_id}:${falkonry_tercel_edge_kubernetes_edge_version}
+#lowercase docker image name as uppercase is not accepted in docker repository name and sometimes tenantid has upper case letters
+edge_standalone_image=`echo "$edge_standalone_image" | awk '{print tolower($0)}'`
+
+echo "printing content of final docker file"
+cat EdgeStandaloneDockerfile
+
+echo "Building edge standalone docker image ${edge_standalone_image}"
+if [ ${falkonry_edge_image_type} == "amd64" ]; then
+ docker build --arch linux/amd64 -t ${edge_standalone_image} -f EdgeStandaloneDockerfile .
+else
+ docker build --arch linux/armhf -t ${edge_standalone_image} -f EdgeStandaloneDockerfile .
+fi
+
+hadoop fs -test -d ${falkonry_tercel_riactor_export_directory}
+if [ $? -eq 0 ]
+then
+ echo "Tercel edge directory exists....checking if tenant id exists..."
+ hadoop fs -test -d ${falkonry_tercel_riactor_export_directory}/${tenant_id}
+ tenant=`echo $?`
+ if [ $? -eq 0 ]
+ then
+ echo "Tenant id also exists....continuing the script..."
+ else
+ echo "Tenant id does not exist......Creating one....."
+ hadoop fs -mkdir ${falkonry_tercel_riactor_export_directory}/${tenant_id}
+ fi
+else
+ echo "Tercel edge directory does not exist....Creating one......"
+ hadoop fs -mkdir ${falkonry_tercel_riactor_export_directory}
+ hadoop fs -mkdir ${falkonry_tercel_riactor_export_directory}/${tenant_id}
+fi
+
+local_destination=/edge-standalone.tar
+image_file_name="falkonry-${model_type_short}-analyzer-${falkonry_edge_image_type}-${job_id}.tar"
+hdfs_destination=${falkonry_tercel_riactor_export_directory}/${tenant_id}/${datastream_id}/${assessment_id}/${model_id}/${image_file_name}
+
+echo "Saving docker image to local file: ${local_destination} and going to copy to hdfs: ${hdfs_destination}"
+docker save ${edge_standalone_image} > ${local_destination}
+
+hadoop distcp file://${local_destination} ${hdfs_destination}
+
+echo "Successfully copied docker image to ${hdfs_destination}"
+
+download_url=`grep -Po '\w\K/\w+[^?]+' <<<$falkonry_edge_job_url`/download
+
+#if tar file exists and size is > 0 then job is completed successfully
+hadoop fs -test -e ${hdfs_destination}
+if [ $? -eq 0 ]
+then
+ echo "Found tar ${hdfs_destination} going to check if size is greater than zero"
+ hadoop fs -test -z ${hdfs_destination}
+ if [ $? -eq 1 ]
+ then
+ echo "Found tar ${hdfs_destination} with size > 0 going to update job status as COMPLETED"
+ jobStatus='{ "status" : "COMPLETED", "links" : [{"url" : "'"$download_url"'", "name":"image", "file": "'"$image_file_name"'"}]}'
+ echo $jobStatus
+ curl -X PUT ${falkonry_edge_job_url} -d "$jobStatus" -H"Content-Type: application/json" -v
+ else
+ echo "Did not find valid tar ${hdfs_destination}, its size is zero, going to update job status as FAILED"
+ jobStatus="{ \"status\" : \"FAILED\"}"
+ curl -X PUT ${falkonry_edge_job_url} -d"$jobStatus" -H"Content-Type: application/json" -v
+ fi
+else
+
+ echo "Did not find tar ${hdfs_destination} going to update job status as FAILED"
+ jobStatus="{ \"status\" : \"FAILED\"}"
+ curl -X PUT ${falkonry_edge_job_url} -d"$jobStatus" -H"Content-Type: application/json" -v
+fi
+
+echo "Done"
+
+exit 0
\ No newline at end of file
--
GitLab