trufflehog emulation prototype

scripts/trufflehog/docker-compose.yml

+version: "0.1"
+services:
+  trufflehog:
+    image: registry1.dso.mil/ironbank/opensource/trufflehog/trufflehog3:3.0.10
+    container_name: trufflehog3
+    entrypoint: ["/root/scripts/trufflehog/entrypoint.sh"]
+    working_dir: /root
+    # env_file:
+      # - .env # TODO: Setting env vars in here allows use of env vars inside this file itself, i.e. the entrypoint directory. Probably good to define a BASE_SCRIPTS_DIR for all jobs.
+    environment:
+      - REPORT_DIR=/root/reports/trufflehog
+      - SCRIPTS_DIR=/root/scripts/trufflehog
+      - SCAN_DIR=/app
+      # - TRUFFLEHOG_EXCLUDE_PATHS= # TODO: Refine
+    volumes:
+      - ./:/root
+      - ./<<projectName>>:/app

scripts/trufflehog/entrypoint.sh

+#!/bin/bash
+
+ls -al # TODO: Delete
+mkdir -p ${REPORT_DIR}
+REPORT_FILE=${REPORT_DIR}/trufflehog_report.json 
+# enable shell globbing for recursive exclude matching. allows use of '**/*' format
+shopt -s globstar
+configfiletoremove=".trufflehog3.yml"
+if [ -f "$configfiletoremove" ]; then rm -f "$configfiletoremove" && echo "I deleted trufflehog3.yml"; fi
+# set -x so we can see the real command being run
+set -x
+trufflehog3 -vvv --ignore-nosecret --exclude ${TRUFFLEHOG_EXCLUDE_PATHS} \
+  --format json --zero --no-history \
+  -r ${SCRIPTS_DIR}/rules.yml \
+  -o ${REPORT_FILE} ${SCAN_DIR} 2>&1 | tee ${REPORT_DIR}/trufflehog_log.txt # TODO: Writing to trufflehog_log.txt not currently working
+set +x
+shopt -u globstar
+trufflehog3 --version > ${REPORT_DIR}/trufflehog_version.txt
+python ${SCRIPTS_DIR}/trufflehog-gate-check.py "${REPORT_FILE}"

scripts/trufflehog/rules.yml

+# Rules originally from https://github.com/feeltheajf/trufflehog3/blob/master/trufflehog3/static/rules.yml
+# Any severity "MEDIUM" or "HIGH" rules will fail the P1 gate check; "LOW" rules will just be warnings.
+
+#
+# entropy-based rules
+#
+- id: high-entropy
+  message: High Entropy
+  minlen: 20
+  alphabet: "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+/="
+  threshold: 4.5
+  severity: MEDIUM
+- id: high-entropy
+  message: High Entropy
+  minlen: 20
+  alphabet: "0123456789abcdefABCDEF"
+  threshold: 3.0
+  severity: MEDIUM
+#
+# regexes-based rules
+#
+- id: private.pgp-key
+  message: PGP Private Key Block
+  pattern: "-----BEGIN PGP PRIVATE KEY BLOCK-----"
+  severity: HIGH
+- id: private.key
+  message: Private Key
+  pattern: "-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----"
+  severity: HIGH
+
+- id: amazon.aws-api-key
+  message: AWS API Key
+  pattern: "AKIA[0-9A-Z]{16}"
+  severity: MEDIUM
+- id: amazon.mws-auth-token
+  message: Amazon MWS Auth Token
+  pattern: "amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"
+  severity: MEDIUM
+
+- id: facebook.access-token
+  message: Facebook Access Token
+  pattern: "EAACEdEose0cBA[0-9A-Za-z]+"
+  severity: MEDIUM
+- id: facebook.oauth-token
+  message: Facebook OAuth Token
+  pattern: '[f|F][a|A][c|C][e|E][b|B][o|O][o|O][k|K].*[''|"][0-9a-f]{32}[''|"]'
+  severity: MEDIUM
+
+# TODO add new github token formats
+- id: github.token
+  message: GitHub Token
+  pattern: '[g|G][i|I][t|T][h|H][u|U][b|B].*[''|"][0-9a-zA-Z]{35,40}[''|"]'
+  severity: MEDIUM
+
+- id: google.access-token
+  message: Google Access Token
+  pattern: "ya29\\.[0-9A-Za-z\\-_]+"
+  severity: MEDIUM
+- id: google.api-key
+  message: Google API Key
+  pattern: "AIza[0-9A-Za-z\\-_]{35}"
+  severity: MEDIUM
+- id: google.oauth-token
+  message: Google OAuth
+  pattern: "[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com"
+  severity: MEDIUM
+- id: google.gcp-service-account
+  message: Google Cloud Platform Service Account
+  pattern: '"type": "service_account"'
+  severity: MEDIUM
+
+- id: heroku.api-key
+  message: Heroku API Key
+  pattern: "[h|H][e|E][r|R][o|O][k|K][u|U].*[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"
+  severity: MEDIUM
+
+- id: mailchimp.api-key
+  message: MailChimp API Key
+  pattern: "[0-9a-f]{32}-us[0-9]{1,2}"
+  severity: MEDIUM
+
+- id: mailgun.api-key
+  message: Mailgun API Key
+  pattern: "key-[0-9a-zA-Z]{32}"
+  severity: MEDIUM
+
+- id: paypal.braintree-access-token
+  message: PayPal Braintree Access Token
+  pattern: "access_token\\$production\\$[0-9a-z]{16}\\$[0-9a-f]{32}"
+  severity: MEDIUM
+
+- id: picatic.api-key
+  message: Picatic API Key
+  pattern: "sk_live_[0-9a-z]{32}"
+  severity: MEDIUM
+
+- id: slack.token
+  message: Slack Token
+  pattern: "xox[p|b|o|a]-[0-9]{12}-[0-9]{12}-[0-9]{12}-[a-z0-9]{32}"
+  severity: MEDIUM
+- id: slack.bot-token
+  message: Slack Bot Token
+  pattern: "xox[p|b|o|a]-[0-9]{12}-[0-9]{12}-[a-z0-9]{24}"
+  severity: MEDIUM
+- id: slack.webhook
+  message: Slack Webhook
+  pattern: "https://hooks.slack.com/services/T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}"
+  severity: MEDIUM
+
+- id: stripe.api-key
+  message: Stripe API Key
+  pattern: "sk_live_[0-9a-zA-Z]{24}"
+  severity: MEDIUM
+- id: stripe.restricted-api-key
+  message: Stripe Restricted API Key
+  pattern: "rk_live_[0-9a-zA-Z]{24}"
+  severity: MEDIUM
+
+- id: square.access-token
+  message: Square Access Token
+  pattern: "sq0atp-[0-9A-Za-z\\-_]{22}"
+  severity: MEDIUM
+- id: square.oauth-secret
+  message: Square OAuth Secret
+  pattern: "sq0csp-[0-9A-Za-z\\-_]{43}"
+  severity: MEDIUM
+
+- id: twilio.api-key
+  message: Twilio API Key
+  pattern: "SK[0-9a-fA-F]{32}"
+  severity: MEDIUM
+
+- id: twitter.access-token
+  message: Twitter Access Token
+  pattern: "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[1-9][0-9]+-[0-9a-zA-Z]{40}"
+  severity: MEDIUM
+- id: twitter.oauth-token
+  message: Twitter OAuth Token
+  pattern: '[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[''|"][0-9a-zA-Z]{35,44}[''|"]'
+  severity: MEDIUM
+
+  #
+  # These 3 upstream rules were originally LOW severity, but P1 upgraded them to MEDIUM:
+  #
+- id: generic.api-key
+  message: Generic API Key
+  pattern: '[a|A][p|P][i|I][_]?[k|K][e|E][y|Y].*[''|"][0-9a-zA-Z]{32,45}[''|"]'
+  severity: MEDIUM
+- id: generic.secret
+  message: Generic Secret
+  pattern: '[s|S][e|E][c|C][r|R][e|E][t|T].*[''|"][0-9a-zA-Z]{32,45}[''|"]'
+  severity: MEDIUM
+- id: generic.password-in-url
+  message: Password in URL
+  pattern: "[a-zA-Z]{3,10}://[^/\\s:@]{3,20}:[^/\\s:@]{3,20}@.{1,100}[\"'\\s]"
+  severity: MEDIUM
+
+  #
+  # P1 additions to upstream rules:
+  #
+- id: generic.password
+  message: Password 
+  pattern: "[p|P]assword\\s*[:|=]"
+  severity: HIGH
+- id: generic.dso-url
+  message: Hardcoded dso.mil URL
+  pattern: "(?i)https?://[0-9a-z.-]+\\.dso\\.mil"
+  severity: LOW
+- id: polyfill-urls
+  message: PolyFill URLs
+  pattern: "polyfill[.]io|bootcdn[.]net|bootcss[.]com|kuurza[.]com|newcrbpc[.]com|staticfile[.]net|staticfile[.]org|macoms[.]la|unionadjs[.]com|googie-anaiytics[.]com|xhsbpza[.]com"
+  severity: HIGH

scripts/trufflehog/trufflehog-gate-check.py

+import sys
+import json
+
+def parseTrufflehogReport(path):
+  """ Parse the JSON report format and
+      separate the findings (severity "MEDIUM" or "HIGH")
+      from the warnings (severity "LOW")
+  """
+  with open(path, 'r') as f:
+    data = json.load(f)
+  findings = [x for x in data if x["rule"]["severity"]=="HIGH" or x["rule"]["severity"]=="MEDIUM"]
+  warnings = [x for x in data if x["rule"]["severity"]=="LOW"]
+  return (findings, warnings)
+
+if __name__ == "__main__":
+  if len(sys.argv) < 2:
+    filename = "trufflehog_report.json"
+    print("No filename supplied; using default.")
+  else:
+    filename = sys.argv[1]
+  print(f'Parsing {filename}')
+  (findings, warnings) = parseTrufflehogReport(filename)
+  if len(findings) > 0:
+    print("Trufflehog gate check findings - ")
+    print(json.dumps(findings,indent=2))
+    print(f'Trufflehog gate check Failed with finding count -  {len(findings)}')
+    print('For help, see https://confluence.il2.dso.mil/display/P1MDOHD/TS+-+Trufflehog+-+Stage+Failure')
+    sys.exit(1)
+  if len(warnings) > 0:
+    print(json.dumps(warnings,indent=2))
+  print(f'Trufflehog gate check Passed with {len(warnings)} warnings')