Merge branch 'BULL-3230_find_unauthorized_job' into 'main'

BULL-3230: find unauthorized job

See merge request !7

docker/pipeline-jobs/docker-compose-dockerfile-lint.yml

@@ -8,6 +8,7 @@ services:
     environment:
       - REPORTS_DIR=${BASE_REPORTS_DIR}/dockerfile-lint<<subProject>>
       - SCRIPTS_DIR=${BASE_SCRIPTS_DIR}/dockerfile-lint
+      - UTILITY_DIR=${BASE_SCRIPTS_DIR}/utility
       - SCAN_DIR=/app
       - CI_JOB_NAME=dockerfile-lint<<subProject>>
       - DOCKERFILE_LOC=Dockerfile

docker/pipeline-jobs/docker-compose-find-unauthorized.yml

+services:
+
+  find-unauthorized<<subProject>>:
+    image: registry1.dso.mil/ironbank/opensource/python:v3.12.5
+    container_name: find-unauthorized<<subProject>>
+    entrypoint: ["${BASE_SCRIPTS_DIR}/find-unauthorized/entrypoint.sh"]
+    working_dir: /root
+    environment:
+      - REPORTS_DIR=${BASE_REPORTS_DIR}/find-unauthorized<<subProject>>
+      - SCRIPTS_DIR=${BASE_SCRIPTS_DIR}/find-unauthorized
+      - UTILITY_DIR=${BASE_SCRIPTS_DIR}/utility
+      - SCAN_DIR=/app
+      - CI_JOB_NAME=find-unauthorized<<subProject>>
+      - DOCKERFILE_LOC=Dockerfile
+    volumes:
+      - ./:/root
+      - ./<<projectName>><<subProject>>:/app

formulas/baselines/universal.yml

@@ -7,16 +7,22 @@ pipeline:
       composeFile: docker/pipeline-jobs/docker-compose-trufflehog.yml
     dockerfile-lint:
       composeFile: docker/pipeline-jobs/docker-compose-dockerfile-lint.yml
+    find-unauthorized:
+      composeFile: docker/pipeline-jobs/docker-compose-find-unauthorized.yml
 
 getLocalDevDirs:
   - from: root
     to: /
   - from: config
     to: config
+  - from: scripts/utility
+    to: scripts/utility
   - from: scripts/trufflehog
     to: scripts/trufflehog
   - from: scripts/dockerfile-lint
     to: scripts/dockerfile-lint
+  - from: scripts/find-unauthorized
+    to: scripts/find-unauthorized
 
 getLocalDevFiles:
   - from: scripts/setup.sh

scripts/dockerfile-lint/entrypoint.sh

@@ -2,15 +2,16 @@
 
 mkdir -p ${REPORTS_DIR}
 rm -f ${REPORTS_DIR}/*
-${SCRIPTS_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -s fail -r config -l "Job run in local dev"
+
+${UTILITY_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -s fail -r config -l "Job run in local dev"
 set -o pipefail
 hadolint $APPROVED_REGISTRY $SCAN_DIR/$DOCKERFILE_LOC --failure-threshold warning | tee ${REPORTS_DIR}/${CI_JOB_NAME}.out
 
 if [ "$?" == "0" ]; then
-  ${SCRIPTS_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -s pass -r pass
+  ${UTILITY_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -s pass -r pass
 else
   if [ -s "${REPORTS_DIR}/${CI_JOB_NAME}.out" ]; then
-    ${SCRIPTS_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -r findings -l "${CI_JOB_NAME} process found findings, check job for details"
+    ${UTILITY_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -r findings -l "${CI_JOB_NAME} process found findings, check job for details"
   fi
   exit 1
 fi

scripts/find-unauthorized/cmdscan.py

+from sys import argv, exit, stderr
+from os import path
+import json
+import re
+
+def __parseline(line):
+    line = line.strip('& \\ \n')
+    line = line.replace('RUN', '')
+    line = line.replace('||','&&').replace(';','&&')
+    line = line.split('&&')
+    commands = [command.strip().split(' ') for command in line]
+    return commands
+
+
+def __formatmessage(file, linenum, command):
+    return f'{file}:{linenum} error: Contains an unauthorized command: {command}'
+
+
+def __findcommands(commands, lookup):
+    if not commands: return ''
+
+    result = []; found = False
+    for command in commands:
+        if __ismatch(command.copy(), lookup):
+            result.append(' '.join(command))
+        
+    return ', '.join(result) if result else ''
+
+
+def __ismatch(command, lookup):
+    while command:
+        symbol = command.pop(0)
+        for term in lookup:
+            if isinstance(term, dict):
+                if symbol in term:
+                    return __ismatch(command, term[symbol])
+            elif isinstance(term, str):
+                if re.match(term, symbol):
+                    return True
+            else:
+                print(f'Bad entry found in lookup')
+                exit(1)
+    return False
+
+def __loadlookup(lookup_json):
+    try:
+        file = open(lookup_json)
+        commands = json.load(file)
+    except:
+        print(f'Could not find lookup json: {lookup_json}', file=stderr)
+        exit(1)
+    return commands
+
+
+def __loadlines(script):
+    try:
+        lines = open(script).readlines()
+    except:
+        print(f'Could not find file to scan: {script}', file=stderr)
+        exit(1)
+    return lines
+
+
+def scanfile(script, lookup_json):
+    lines = __loadlines(script)
+    lookup = __loadlookup(lookup_json)
+
+    issues = []
+    for line_num, line in enumerate(lines):
+        line_num = line_num + 1
+        commands = __parseline(line)
+        commands = __findcommands(commands, lookup)
+        if commands: issues.append(__formatmessage(script, line_num, commands))
+    return issues
+
+
+if __name__ == "__main__":
+    if len(argv) != 3:
+        print(f'Unexpected number of arguments. Expected 2, got {len(argv)-1}', file=stderr)
+        print('Usage: python cmdscan.py {script} {lookup.json}', file=stderr)
+        exit(1)
+
+    script = argv[1]
+    lookup_json = argv[2]
+
+    print(f'Scanning file: {script}')
+    print(f'Using lookup: {lookup_json}\n')
+
+    issues = scanfile(script, lookup_json)
+
+    if issues:
+        for issue in issues:
+            error_msg = f'{issue}'
+            print(error_msg)
+        print(f'\nFound {len(issues)} issues with {script}')
+        exit(1)
+
+    print('No issues found.')
\ No newline at end of file

scripts/find-unauthorized/entrypoint.sh

+#!/bin/bash
+
+mkdir -p ${REPORTS_DIR}
+rm -f ${REPORTS_DIR}/*
+
+${UTILITY_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -s fail -r config -l "Job run in local dev"
+set -o pipefail
+python $SCRIPTS_DIR/cmdscan.py $SCAN_DIR/$DOCKERFILE_LOC $SCRIPTS_DIR/unauthorized.json | tee ${REPORTS_DIR}/${CI_JOB_NAME}.out
+
+if [ "$?" == "0" ]; then
+  ${UTILITY_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -s pass -r pass
+else
+  if [ -s "${REPORTS_DIR}/${CI_JOB_NAME}.out" ]; then
+    ${UTILITY_DIR}/monitorstatus.sh -j ${CI_JOB_NAME} -r findings -l "${CI_JOB_NAME} process found findings, check job for details"
+  fi
+  exit 1
+fi

scripts/find-unauthorized/unauthorized.json

+[
+  { "#": [{ "hadolint": ["ignore"] }] },
+  { "npm": ["i","ci","install"] },
+  { "gradlew": ["assemble","build"] },
+  { "bundle": ["add","install","update"] },
+  { "mvn": ["install", "package"] },
+  { "yarn": ["install", "add"] },
+  { "dotnet": ["publish"] },
+  { "go": ["build", "get"] },
+  { "gem": ["install"] },
+  { "pip": ["install"] },
+  { "WORKDIR": ["../"]},
+  "yum", 
+  "apt", 
+  "apt-get", 
+  "apk", 
+  "dnf", 
+  "pipenv", 
+  "rpm", 
+  "curl", 
+  "wget", 
+  "adduser", 
+  "useradd", 
+  "make", 
+  "gcc", 
+  "g\\+\\+",
+  "microdnf",
+  "/proc/",
+  "--mount",
+  "syntax=",
+  "\\.\\./"
+]