From f40f2825a57ded071cec8c0942661313afe8935f Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 08:48:13 -0700 Subject: [PATCH 1/9] add debug statement --- stages/publish/create_repo_map_default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stages/publish/create_repo_map_default.py b/stages/publish/create_repo_map_default.py index 43516450..15b2db1a 100644 --- a/stages/publish/create_repo_map_default.py +++ b/stages/publish/create_repo_map_default.py @@ -26,6 +26,7 @@ def get_repomap(object_name, bucket="ironbank-pipeline-artifacts"): print(object_name) try: response = s3_client.download_file(bucket, object_name, "repo_map.json") + logging.debug(f"S3 download response: {response}") except ClientError as e: logging.error(e) print("Existing repo_map.json not found, creating new repo_map.json") -- GitLab From c3cf7f134445d6ad1d5017e90e0c954cd4f76205 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 08:48:38 -0700 Subject: [PATCH 2/9] add debug message --- stages/publish/create_repo_map_other.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stages/publish/create_repo_map_other.py b/stages/publish/create_repo_map_other.py index ca482e1b..3f105be3 100644 --- a/stages/publish/create_repo_map_other.py +++ b/stages/publish/create_repo_map_other.py @@ -26,6 +26,7 @@ def get_repomap(object_name, bucket="ironbank-pipeline-artifacts"): print(object_name) try: response = s3_client.download_file(bucket, object_name, "repo_map.json") + logging.debug(f"S3 download response: {response}") except ClientError as e: logging.error(e) print("Existing repo_map.json not found, creating new repo_map.json") -- GitLab From c6b513642d3e6850bbbcd699f44f0efbbc1a5d00 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 08:49:17 -0700 Subject: [PATCH 3/9] add dynamic content-type --- stages/publish/s3_upload.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/stages/publish/s3_upload.py b/stages/publish/s3_upload.py index a7abccf9..eb1d7ae0 100644 --- a/stages/publish/s3_upload.py +++ b/stages/publish/s3_upload.py @@ -5,6 +5,7 @@ import argparse import datetime from botocore.exceptions import ClientError import logging +import mimetypes def upload_file(file_name, bucket, object_name=None): @@ -19,8 +20,32 @@ def upload_file(file_name, bucket, object_name=None): access_key = os.environ["S3_ACCESS_KEY"] secret_key = os.environ["S3_SECRET_KEY"] + filetype = mimetypes.guess_type(file_name) + + if not filetype[0]: + # If mimetype is NoneType use default value + mimetype = "application/octet-stream" + elif filetype[1] == "gzip": + # mimetypes returns 'application/x-tar' + # but for S3 to properly serve gzip we need to set to the following + mimetype = "application/x-compressed-tar" + else: + mimetype = filetype[0] + # TODO: Add signature - extra_args = {"ContentType": "application/octet-stream", "ACL": "private"} + # If there is not an encoding-type value we don't add it to the extra args + if not filetype[1]: + extra_args = { + "ContentType": mimetype, + "ACL": "private", + } + else: + encoding = filetype[1] + extra_args = { + "ContentType": mimetype, + "ACL": "private", + "ContentEncoding": encoding, + } # If S3 object_name was not specified, use file_name if object_name is None: @@ -35,6 +60,7 @@ def upload_file(file_name, bucket, object_name=None): ) try: response = s3_client.upload_file(file_name, bucket, object_name, extra_args) + logging.debug(f"S3 upload response: {response}") except ClientError as e: logging.error(e) return False -- GitLab From d0fc1fa9e714dd9abcd33f1d427cdd77425f7289 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 09:11:11 -0700 Subject: [PATCH 4/9] update tar name to match how this worked in Jenkins pipeline --- stages/publish/repo_map_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/publish/repo_map_vars.sh b/stages/publish/repo_map_vars.sh index 772438c9..c538f9b3 100644 --- a/stages/publish/repo_map_vars.sh +++ b/stages/publish/repo_map_vars.sh @@ -20,7 +20,7 @@ export pgp_signature="${S3_HTML_LINK}/${REMOTE_REPORT_DIRECTORY}/${SIG_FILE}.sig export signature_name="${SIG_FILE}.sig" export version_documentation="${S3_HTML_LINK}/${REMOTE_REPORT_DIRECTORY}/${DOCUMENTATION_FILENAME}.json" export tar_location="${S3_HTML_LINK}/${REMOTE_REPORT_DIRECTORY}/${REPORT_TAR_NAME}" -export tar_name="${IMAGE_FILE}.tar" +export tar_name="${REPORT_TAR_NAME}" export openscap_compliance_results="${S3_HTML_LINK}/${REMOTE_REPORT_DIRECTORY}/csvs/oscap.csv" export openscap_oval_results="${S3_HTML_LINK}/${REMOTE_REPORT_DIRECTORY}/csvs/oval.csv" export twistlock_results="${S3_HTML_LINK}/${REMOTE_REPORT_DIRECTORY}/csvs/tl.csv" -- GitLab From 89a10a690c342890d68caab49dff664f9c4cbb12 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 09:11:51 -0700 Subject: [PATCH 5/9] update approval status for IBFE --- stages/check-cves/pipeline_wl_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 22982fc6..048ef547 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -366,7 +366,7 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma with open("variables.env", "w") as f: # all cves for container have container approval at ind 2 if check_container_approval[2].lower() == "approve": - f.write(f"IMAGE_APPROVAL_STATUS='approved'\n") + f.write(f"IMAGE_APPROVAL_STATUS='Approved'\n") else: f.write(f"IMAGE_APPROVAL_STATUS='notapproved'\n") f.write(f"BASE_IMAGE={hardening_manifest['args']['BASE_IMAGE']}\n") -- GitLab From f792ede5e017f960f658bff6663f3b70081156bd Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 09:16:50 -0700 Subject: [PATCH 6/9] Revert "update approval status for IBFE" This reverts commit 89a10a690c342890d68caab49dff664f9c4cbb12. --- stages/check-cves/pipeline_wl_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 048ef547..22982fc6 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -366,7 +366,7 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma with open("variables.env", "w") as f: # all cves for container have container approval at ind 2 if check_container_approval[2].lower() == "approve": - f.write(f"IMAGE_APPROVAL_STATUS='Approved'\n") + f.write(f"IMAGE_APPROVAL_STATUS='approved'\n") else: f.write(f"IMAGE_APPROVAL_STATUS='notapproved'\n") f.write(f"BASE_IMAGE={hardening_manifest['args']['BASE_IMAGE']}\n") -- GitLab From 0e157d761a94de17207c9add2ae9ca48c5ba1959 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 09:29:10 -0700 Subject: [PATCH 7/9] fix approval status strings --- stages/check-cves/pipeline_wl_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 22982fc6..3ba5c023 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -366,9 +366,9 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma with open("variables.env", "w") as f: # all cves for container have container approval at ind 2 if check_container_approval[2].lower() == "approve": - f.write(f"IMAGE_APPROVAL_STATUS='approved'\n") + f.write(f"IMAGE_APPROVAL_STATUS=approved\n") else: - f.write(f"IMAGE_APPROVAL_STATUS='notapproved'\n") + f.write(f"IMAGE_APPROVAL_STATUS=notapproved\n") f.write(f"BASE_IMAGE={hardening_manifest['args']['BASE_IMAGE']}\n") f.write(f"BASE_TAG={hardening_manifest['args']['BASE_TAG']}") # -- GitLab From dac8e59b2077416e89ea0d804baac6394ae0e9d1 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 10:05:12 -0700 Subject: [PATCH 8/9] debug logging for S3 upload --- stages/publish/s3_upload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stages/publish/s3_upload.py b/stages/publish/s3_upload.py index eb1d7ae0..02959b03 100644 --- a/stages/publish/s3_upload.py +++ b/stages/publish/s3_upload.py @@ -47,6 +47,7 @@ def upload_file(file_name, bucket, object_name=None): "ContentEncoding": encoding, } + logging.debug(f"extra_args for {file_name}: {extra_args}") # If S3 object_name was not specified, use file_name if object_name is None: object_name = file_name -- GitLab From c34e5f9dec1fe5d5d8ef1b1d1438488df55a7569 Mon Sep 17 00:00:00 2001 From: David Freeman Date: Fri, 18 Dec 2020 10:16:57 -0700 Subject: [PATCH 9/9] add debug printing of lint env vars --- stages/check-cves/pipeline_wl_compare.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 3ba5c023..210edb43 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -367,10 +367,15 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma # all cves for container have container approval at ind 2 if check_container_approval[2].lower() == "approve": f.write(f"IMAGE_APPROVAL_STATUS=approved\n") + logging.debug(f"IMAGE_APPROVAL_STATUS=approved") else: f.write(f"IMAGE_APPROVAL_STATUS=notapproved\n") + logging.debug(f"IMAGE_APPROVAL_STATUS=notapproved") f.write(f"BASE_IMAGE={hardening_manifest['args']['BASE_IMAGE']}\n") f.write(f"BASE_TAG={hardening_manifest['args']['BASE_TAG']}") + logging.debug( + f"BASE_IMAGE={hardening_manifest['args']['BASE_IMAGE']}\nBASE_TAG={hardening_manifest['args']['BASE_TAG']}" + ) # # Use the local hardening manifest to get the first parent. From here *only* the # the master branch should be used for the ancestry. -- GitLab