Merge branch 'repo1-mirror' into 'development'

Changes to use s3 for dependencies.

See merge request !3

.gitignore

@@ -7,4 +7,5 @@ main.py
 test_methods.py
 results.txt
 mike-run-local-loop.bash
-*-local-loop.sh
\ No newline at end of file
+*-local-loop.sh
+build_dependencies
\ No newline at end of file

Dockerfile

@@ -7,15 +7,11 @@ RUN dnf update -y && dnf clean all
 # Copy in the project files.
 WORKDIR /app
 COPY ./scripts .
+COPY redis-cli.tar.gz .
 
 # Install redis.
-WORKDIR /app/bundles
-RUN tar xzf redis-cli.tar.gz
+RUN tar xzf redis-cli.tar.gz && rm redis-cli.tar.gz
 RUN mv redis-cli /usr/local/bin/redis-cli
-WORKDIR /app
-
-# Remove bundles.
-RUN rm -rf bundles
 
 # Set permissions and ownership.
 RUN chown -R 1001 /app

README.md

@@ -9,6 +9,23 @@ code snippets that were submitted from students through the LEARN application.
 - Redis CLI
 - Python 3
 
+## Container Usage:
+
+### Container info:
+
+- workdir: /usr/src/app
+- user id: 1001
+- Does not require container port mapping.
+
+### Recommended Resources:
+
+- Min/Max CPU: 0.1/0.2
+- Min/Max Memory: 64Mi/128Mi
+
+### Required Environment Variables:
+
+- ```REDIS_HOST``` - The redis server host address.
+
 ## Local Development Setup
 
 1. Install redis. Skip this if you already have it installed and running.

hardening_manifest.yaml

@@ -28,6 +28,18 @@ labels:
   mil.dso.ironbank.image.type: "commercial"
   mil.dso.ironbank.product.name: "Learn"
 
+# List of resources to make available to the offline build context
+resources:
+- auth:
+    type: s3
+    id: galvanize
+    region: us-gov-west-1
+  url: s3://learn-dependencies/python-evaluator/redis-cli.tar.gz
+  filename: redis-cli.tar.gz
+  validation:
+    type: sha256
+    value: 1a4a61818d33d8f9c9aea0f3ce1ed3093500f25ac750365ae6c06a6d2f0ee123
+
 # List of project maintainers
 maintainers:
 - email: "dchong@revacomm.com"

scripts/Dockerfile.packages

+ARG BASE_REGISTRY
+ARG BASE_IMAGE
+ARG BASE_TAG
+
+FROM ${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}
+USER 0
+
+ARG NODE_VERSION
+ARG YARN_VERSION
+ARG JQUERY_RAILS_VERSION
+
+# Install required libs.
+RUN dnf update -y &&  dnf install -y \
+        curl \
+        make \
+        gcc \
+        gcc-c++
+
+# Setup our environment
+WORKDIR /app
+
+# Redis
+RUN curl -L http://download.redis.io/redis-stable.tar.gz -o redis-stable.tar.gz
+RUN tar xzf redis-stable.tar.gz
+WORKDIR /app/redis-stable
+RUN make redis-cli
+WORKDIR /app/redis-stable/src
+RUN tar czf redis-cli.tar.gz redis-cli
+RUN mv redis-cli.tar.gz /app/
+
+# Switch back to app dir.
+WORKDIR /app
+
+# Add write permissions.
+RUN chown -R 1001 .
+
+# Set the entry point.
+CMD tail -f /dev/null

scripts/config.py

 import os
 
-DATA_FILE_PATH   = "json-object.json"
+DATA_FILE_PATH = "json-object.json"
 OUTPUT_FILE_NAME = "test_methods.py"
 OUTPUT_FILE_PATH = os.path.join('./', OUTPUT_FILE_NAME)
-CODE_FILE_NAME   = "main.py"
-CODE_FILE_PATH   = os.path.join('./', CODE_FILE_NAME)
-RESULTS_FILE     = "results.txt"
-ERROR_INDICATOR  = "FAILED"
+CODE_FILE_NAME = "main.py"
+CODE_FILE_PATH = os.path.join('./', CODE_FILE_NAME)
+RESULTS_FILE = "results.txt"
+ERROR_INDICATOR = "FAILED"

scripts/get-job.sh

@@ -3,19 +3,11 @@
 loop=1
 
 while true; do
-    echo "Processing job number: $loop"
-    let "loop=loop+1"
-
     # Start fresh, recopy the app into the working directory
     rm -rf /usr/src/app
     cp -r /app /usr/src
     cd /usr/src/app
 
-    # If it exists, delete the json-object.json file and test folder
-    [ -e test_methods.py ] && rm test_methods.py
-    [ -e results.txt ] && rm results.txt
-    [ -e json-object.json ] && rm json-object.json
-
     # Connect to the redis service,
     # apply the redis commands to block-pop
     # grep out the actual job payload, and
@@ -27,6 +19,14 @@ while true; do
     # continue if the json file is empty.
     [ -s json-object.json ] || continue
 
+    # Log the current job number.
+    echo
+    echo "============================"
+    echo "Processing job number: $loop"
+    echo "============================"
+    echo
+    let "loop=loop+1"
+
     # Do some pre-processing
     python process-job.py
 

scripts/process-job.py

-import sys
-from config import *
+from config import DATA_FILE_PATH, CODE_FILE_PATH, OUTPUT_FILE_PATH
 from utilities import file_utils
 
 job_args = file_utils.get_job_args(DATA_FILE_PATH)

scripts/repackage.sh

+#!/bin/bash
+
+# Image Params.
+BASE_REGISTRY=registry.il2.dso.mil
+BASE_IMAGE=platform-one/devops/pipeline-templates/ironbank/python38
+BASE_TAG=3.8.31
+
+# This is the profile name in your aws credentials file.
+DEPENDENCY_FOLDER=build_dependencies
+AWS_PROFILE_NAME=revacomm
+AWS_BUCKET_NAME=learn-dependencies/python-evaluator
+AWS_REGION=us-gov-west-1
+
+echo "Clearing dependency folder."
+rm -rf $DEPENDENCY_FOLDER
+mkdir $DEPENDENCY_FOLDER
+
+echo "Building docker image."
+IMAGE_ID=$(docker build --file Dockerfile.packages . -q \
+  --build-arg BASE_REGISTRY=$BASE_REGISTRY \
+  --build-arg BASE_IMAGE=$BASE_IMAGE \
+  --build-arg BASE_TAG=$BASE_TAG)
+echo "Image ID: ${IMAGE_ID}"
+
+echo "Starting docker container."
+CONTAINER_ID=$(docker run -d "$IMAGE_ID")
+echo "Container ID: ${CONTAINER_ID}"
+
+echo "Copying node modules and gems to the bundles directory."
+docker cp "$CONTAINER_ID":/app/redis-cli.tar.gz $DEPENDENCY_FOLDER
+
+echo "Stopping the docker container."
+docker stop "$CONTAINER_ID"
+
+echo "Uploading all the build dependencies to AWS."
+aws s3 sync $DEPENDENCY_FOLDER s3://$AWS_BUCKET_NAME --delete --profile $AWS_PROFILE_NAME --region $AWS_REGION
+
+echo "Checksums:"
+cd $DEPENDENCY_FOLDER
+sha256sum redis-cli.tar.gz
+cd ..
+
+echo "Removing dependency folder."
+rm -rf $DEPENDENCY_FOLDER
+
+echo "Done!"
\ No newline at end of file

scripts/send-results.py

 import sys
-from config import *
+from config import DATA_FILE_PATH, RESULTS_FILE
 from utilities import file_utils
 from services import results_service
 
@@ -7,13 +7,13 @@ from services import results_service
 job_args = file_utils.get_job_args(DATA_FILE_PATH)
 
 # Get the callback url
-if (not job_args['callback_url'] or len(job_args['callback_url'].strip()) == 0):
-  sys.exit('Callback URL is missing.')
+if not job_args['callback_url'] or len(job_args['callback_url'].strip()) == 0:
+    sys.exit('Callback URL is missing.')
 
 # get the results data.
-results_data = results_service.build_results_data(RESULTS_FILE);
-if (not results_data):
-  sys.exit('Failed to generate result data.')
+results_data = results_service.build_results_data(RESULTS_FILE)
+if not results_data:
+    sys.exit('Failed to generate result data.')
 
 # Fire the request
 results_service.send_results(results_data, job_args['callback_url'])

scripts/services/results_service.py

 import json
 import urllib.request
-from config import *
+from config import ERROR_INDICATOR
 from utilities import file_utils
 from utilities import time_utils
 
+
 def build_results_data(results_file_path):
-  # Get the results data
-  results_data = file_utils.read_file(results_file_path)
+    # Get the results data
+    results_data = file_utils.read_file(results_file_path)
+
+    # Get the status
+    status = 'correct'
+    if results_data.find(ERROR_INDICATOR) != -1:
+        status = 'incorrect'
 
-  # Get the status
-  status = 'correct'
-  if (results_data.find(ERROR_INDICATOR) != -1):
-    status = 'incorrect'
+    return {
+        "results": results_data,
+        "status": status,
+        "assessments_ran_at": time_utils.get_current_time_str()
+    }
 
-  return {
-    "results": results_data,
-    "status": status,
-    "assessments_ran_at": time_utils.get_current_time_str()
-  }
 
 def send_results(results, url):
-  params = json.dumps(results).encode('utf8')
-  req = urllib.request.Request(url, data=params,
-                               headers={'content-type': 'application/json'})
-  req.get_method = lambda: 'PATCH'
-  response = urllib.request.urlopen(req)
+    params = json.dumps(results).encode('utf8')
+    req = urllib.request.Request(
+        url,
+        data=params,
+        headers={'content-type': 'application/json'}
+    )
+    req.get_method = lambda: 'PATCH'
+    urllib.request.urlopen(req)

scripts/utilities/file_utils.py

@@ -2,34 +2,37 @@ import os
 import sys
 import json
 
+
 def get_job_args(file_path):
-  # make sure the JSON data file exists.
-  if not os.path.exists(file_path):
-    sys.exit("JSON data file is missing.")
+    # make sure the JSON data file exists.
+    if not os.path.exists(file_path):
+        sys.exit("JSON data file is missing.")
+
+    # read the json file.
+    try:
+        with open(file_path) as raw_data:
+            job = json.load(raw_data)
 
-  # read the json file.
-  try:
-    with open(file_path) as raw_data:
-      job = json.load(raw_data)
+        # get the job arguments.
+        return job['args'][0]['arguments'][0]
+    except IOError as e:
+        sys.exit("Failed to get job arguments: " + str(e))
 
-    # get the job arguments.
-    return job['args'][0]['arguments'][0];
-  except Exception as e:
-    sys.exit("Failed to get job arguments: " + str(e))
 
 def write_file(file_path, data):
-  try:
-    filehandle = open(file_path, 'w')
-    filehandle.write(data)
-    filehandle.close()
-  except Exception as e:
-    sys.exit("Failed to write file: " + str(e))
+    try:
+        filehandle = open(file_path, 'w')
+        filehandle.write(data)
+        filehandle.close()
+    except IOError as e:
+        sys.exit("Failed to write file: " + str(e))
+
 
 def read_file(file_path):
-  try:
-    file = open(file_path, "r")
-    data = file.read()
-    file.close()
-    return data
-  except Exception as e:
-    sys.exit("Failed to read file: " + str(e))
+    try:
+        f = open(file_path, "r")
+        data = f.read()
+        f.close()
+        return data
+    except IOError as e:
+        sys.exit("Failed to read file: " + str(e))

scripts/utilities/time_utils.py

 from datetime import datetime
 
 def get_current_time_str():
-  # Get current date
-  now = datetime.utcnow()
-  return now.isoformat(' ', 'seconds') + ' UTC'
+    # Get current date
+    now = datetime.utcnow()
+    return now.isoformat(' ', 'seconds') + ' UTC'