From 314bc858d611effb2f1d915a99a9f0b123c82686 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 10:47:46 -0700 Subject: [PATCH 01/25] fix url formatting in requests.get --- stages/check-cves/pipeline_wl_compare.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index bfa2e868..ea16265d 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -310,11 +310,15 @@ def _get_greylist_file_contents(image_path, branch): def _vat_findings_query(im_name, im_version): logging.info("Running query to vat api") - url = f"{os.environ['VAT_BACKEND_SERVER_ADDRESS']}/internal/container?name={im_name}&tag={im_version}" - logging.info(f"GET {url}") try: - r = requests.get(url) + r = requests.get( + f"{os.environ['VAT_BACKEND_SERVER_ADDRESS']}/internal/container", + params={ + "name": im_name, + "tag": im_version, + }, + ) except requests.exceptions.RequestException as e: logging.warning(f"Could not access VAT API: {url}") logging.warning(e) -- GitLab From 9553d90c5c78016529204dc3d35c6232c28a90f9 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 10:52:58 -0700 Subject: [PATCH 02/25] add assert for slashes in branch --- scripts/hardening_manifest_yaml/generate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/hardening_manifest_yaml/generate.py b/scripts/hardening_manifest_yaml/generate.py index 8baa429e..43256fc8 100644 --- a/scripts/hardening_manifest_yaml/generate.py +++ b/scripts/hardening_manifest_yaml/generate.py @@ -22,6 +22,7 @@ def _fetch_file(url, file, branch="development"): Grabs a raw file from gitlab. """ + assert "/" not in branch url = f"{url}/-/raw/{branch}/{file}" logger.debug(url) -- GitLab From 451cc1646344cbbaacf1087a2175546b809f0807 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 11:56:20 -0700 Subject: [PATCH 03/25] add allow failure on api specific exit code --- stages/check-cves/base.yaml | 2 ++ stages/check-cves/pipeline_wl_compare.py | 6 +++++- stages/csv-output/pipeline_csv_gen.py | 1 + stages/lint/base.yaml | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index 007c63e8..2ca160d4 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -20,3 +20,5 @@ check cves: - export PROJ_PATH - pip3 install jsonschema - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" + allow_failure: + exit_codes: 500 diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index ea16265d..744a8a8f 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -30,6 +30,7 @@ from scanners import anchore from scanners import twistlock import swagger_to_jsonschema +global api_exit_code = 0 def _connect_to_db(): """ @@ -357,7 +358,7 @@ def _vat_findings_query(im_name, im_version): logging.warning(f"Unknown response from VAT {r.status_code}") logging.warning(r.text) logging.error("Failing the pipeline, please contact the administrators") - sys.exit(1) + api_exit_code = 500 def _vat_approval_query(im_name, im_version): @@ -394,6 +395,7 @@ def _vat_approval_query(im_name, im_version): result = cursor.fetchall() except Error as error: logging.info(error) + sys.exit(1) finally: if conn is not None and conn.is_connected(): conn.close() @@ -441,6 +443,7 @@ def _vat_vuln_query(im_name, im_version): result = cursor.fetchall() except Error as error: logging.info(error) + sys.exit(1) finally: if conn is not None and conn.is_connected(): conn.close() @@ -663,6 +666,7 @@ def main(): hardening_manifest=hardening_manifest, lint=args.lint, ) + sys.exit(api_exit_code) if __name__ == "__main__": diff --git a/stages/csv-output/pipeline_csv_gen.py b/stages/csv-output/pipeline_csv_gen.py index a61a38e2..836d6e3f 100644 --- a/stages/csv-output/pipeline_csv_gen.py +++ b/stages/csv-output/pipeline_csv_gen.py @@ -346,6 +346,7 @@ def _vat_vuln_query(im_name, im_version): result = cursor.fetchall() except Error as error: logging.info(error) + sys.exit(1) finally: if conn is not None and conn.is_connected(): conn.close() diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 01993011..61bb0e3b 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -63,6 +63,8 @@ wl compare lint: - pip3 install jsonschema - 'mkdir -p "${ARTIFACT_DIR}"' - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' + allow_failure: + exit_codes: 500 artifacts: paths: - "${ARTIFACT_DIR}/" -- GitLab From 700dca25badd13b562528238f48195b65ed2e676 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 12:01:42 -0700 Subject: [PATCH 04/25] fix syntax for global var --- stages/check-cves/pipeline_wl_compare.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 744a8a8f..370f38b2 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -30,7 +30,7 @@ from scanners import anchore from scanners import twistlock import swagger_to_jsonschema -global api_exit_code = 0 + def _connect_to_db(): """ @@ -626,6 +626,9 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma def main(): + # add global var for api failures. + # TODO: Remove api_exit_code when converting to using the api instead of the query + global api_exit_code = 0 # Get logging level, set manually when running pipeline loglevel = os.environ.get("LOGLEVEL", "INFO").upper() if loglevel == "DEBUG": -- GitLab From 14572a8d28001981d53a0d50ffefe018982551be Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 12:06:10 -0700 Subject: [PATCH 05/25] fix syntax again for global var --- stages/check-cves/pipeline_wl_compare.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 370f38b2..d370c099 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -30,7 +30,9 @@ from scanners import anchore from scanners import twistlock import swagger_to_jsonschema - +# add global var for api failures. +# TODO: Remove api_exit_code when converting to using the api instead of the query +api_exit_code = 0 def _connect_to_db(): """ @@ -358,7 +360,7 @@ def _vat_findings_query(im_name, im_version): logging.warning(f"Unknown response from VAT {r.status_code}") logging.warning(r.text) logging.error("Failing the pipeline, please contact the administrators") - api_exit_code = 500 + global api_exit_code = 500 def _vat_approval_query(im_name, im_version): @@ -626,9 +628,6 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma def main(): - # add global var for api failures. - # TODO: Remove api_exit_code when converting to using the api instead of the query - global api_exit_code = 0 # Get logging level, set manually when running pipeline loglevel = os.environ.get("LOGLEVEL", "INFO").upper() if loglevel == "DEBUG": -- GitLab From adb6f6660e46a65dd8eebe42f8137574b2d4d2f1 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 12:10:02 -0700 Subject: [PATCH 06/25] fix syntax one more time --- stages/check-cves/pipeline_wl_compare.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index d370c099..7538892a 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -360,7 +360,8 @@ def _vat_findings_query(im_name, im_version): logging.warning(f"Unknown response from VAT {r.status_code}") logging.warning(r.text) logging.error("Failing the pipeline, please contact the administrators") - global api_exit_code = 500 + global api_exit_code + api_exit_code = 500 def _vat_approval_query(im_name, im_version): @@ -668,6 +669,7 @@ def main(): hardening_manifest=hardening_manifest, lint=args.lint, ) + logging.info(api_exit_code) sys.exit(api_exit_code) -- GitLab From f8b644318279d0452d354abe5e926266f0f3556b Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 12:25:27 -0700 Subject: [PATCH 07/25] add api_exit_code to lint stage --- stages/check-cves/pipeline_wl_compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 7538892a..fce704dc 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -142,7 +142,8 @@ def _pipeline_whitelist_compare(image_name, hardening_manifest, lint=False): whitelist_branch=wl_branch, hardening_manifest=hardening_manifest, ) - sys.exit(0) + logging.info(api_exit_code) + sys.exit(api_exit_code) artifacts_path = os.environ["ARTIFACT_STORAGE"] -- GitLab From 91939f9172fa0f8b358491b6e4f38ae8a2f79e03 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 12:48:40 -0700 Subject: [PATCH 08/25] use exit code 3, and get exit code from python for bash --- stages/check-cves/base.yaml | 6 +++++- stages/check-cves/pipeline_wl_compare.py | 2 +- stages/lint/base.yaml | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index 2ca160d4..b4ed4039 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -20,5 +20,9 @@ check cves: - export PROJ_PATH - pip3 install jsonschema - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" + - | + if [ "$?" == 3 ]; then + exit 3 + fi allow_failure: - exit_codes: 500 + exit_codes: 3 diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index fce704dc..19149c8b 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -362,7 +362,7 @@ def _vat_findings_query(im_name, im_version): logging.warning(r.text) logging.error("Failing the pipeline, please contact the administrators") global api_exit_code - api_exit_code = 500 + api_exit_code = 3 def _vat_approval_query(im_name, im_version): diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 61bb0e3b..1d1829ad 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -63,8 +63,12 @@ wl compare lint: - pip3 install jsonschema - 'mkdir -p "${ARTIFACT_DIR}"' - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' + - | + if [ "$?" == 3 ]; then + exit 3 + fi allow_failure: - exit_codes: 500 + exit_codes: 3 artifacts: paths: - "${ARTIFACT_DIR}/" -- GitLab From 3881e83db1bf73e9e9ebfdd387a2e091d9b30aad Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:04:13 -0700 Subject: [PATCH 09/25] fix bash syntax --- stages/check-cves/base.yaml | 2 +- stages/lint/base.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index b4ed4039..31d7feea 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -22,7 +22,7 @@ check cves: - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" - | if [ "$?" == 3 ]; then - exit 3 + exit 3 fi allow_failure: exit_codes: 3 diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 1d1829ad..2c7413ee 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -65,7 +65,7 @@ wl compare lint: - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' - | if [ "$?" == 3 ]; then - exit 3 + exit 3 fi allow_failure: exit_codes: 3 -- GitLab From 9928c23b8986b8023d35a0594c1a8d64ff401322 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:08:00 -0700 Subject: [PATCH 10/25] add debug --- stages/lint/base.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 2c7413ee..c217978f 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -66,6 +66,8 @@ wl compare lint: - | if [ "$?" == 3 ]; then exit 3 + else + echo "something is wrong" fi allow_failure: exit_codes: 3 -- GitLab From cc0a1737db5531560866252bfc5a5bb2d0fff03f Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:09:15 -0700 Subject: [PATCH 11/25] remove quotes --- stages/lint/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index c217978f..1a823eeb 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -64,7 +64,7 @@ wl compare lint: - 'mkdir -p "${ARTIFACT_DIR}"' - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' - | - if [ "$?" == 3 ]; then + if [ $? == 3 ]; then exit 3 else echo "something is wrong" -- GitLab From 37807ccc29969dde1d97db477f72ba5df0f606be Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:11:44 -0700 Subject: [PATCH 12/25] test case --- stages/lint/base.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 1a823eeb..930490c2 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -69,6 +69,7 @@ wl compare lint: else echo "something is wrong" fi + - exit 3 allow_failure: exit_codes: 3 artifacts: -- GitLab From 4c68a8b742b58eff9046af8bf838c8532f0056e4 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:14:02 -0700 Subject: [PATCH 13/25] make sure changes are being picked up --- stages/lint/base.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 930490c2..62707f49 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -62,6 +62,7 @@ wl compare lint: - export PROJ_PATH - pip3 install jsonschema - 'mkdir -p "${ARTIFACT_DIR}"' + - echo "test" - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' - | if [ $? == 3 ]; then -- GitLab From cb32b16bbc915f10feb3b9c7ea32f3166c01e0c2 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:19:36 -0700 Subject: [PATCH 14/25] validate exit code --- stages/lint/base.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 62707f49..accbb0dd 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -62,15 +62,15 @@ wl compare lint: - export PROJ_PATH - pip3 install jsonschema - 'mkdir -p "${ARTIFACT_DIR}"' - - echo "test" - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' - - | - if [ $? == 3 ]; then - exit 3 - else - echo "something is wrong" - fi - - exit 3 + - echo $? + # - | + # if [ $? == 3 ]; then + # exit 3 + # else + # echo "something is wrong" + # fi + # - exit 3 allow_failure: exit_codes: 3 artifacts: -- GitLab From e9e9e120442397177f290062f1aec2e2c301b6dd Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:36:42 -0700 Subject: [PATCH 15/25] add set +e --- stages/lint/base.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index accbb0dd..b4897cb1 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -62,8 +62,8 @@ wl compare lint: - export PROJ_PATH - pip3 install jsonschema - 'mkdir -p "${ARTIFACT_DIR}"' + - set +e - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' - - echo $? # - | # if [ $? == 3 ]; then # exit 3 -- GitLab From 83a97caeb1d50a80e0f617aa4f9048ff764ce001 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:44:24 -0700 Subject: [PATCH 16/25] test sys exit 1 --- stages/check-cves/pipeline_wl_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 19149c8b..c563d202 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -143,7 +143,7 @@ def _pipeline_whitelist_compare(image_name, hardening_manifest, lint=False): hardening_manifest=hardening_manifest, ) logging.info(api_exit_code) - sys.exit(api_exit_code) + sys.exit(1) artifacts_path = os.environ["ARTIFACT_STORAGE"] -- GitLab From 0c2ad3c02232796ad08f25224a09cf7316716623 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:47:02 -0700 Subject: [PATCH 17/25] remove if case --- stages/check-cves/base.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index 31d7feea..4a53ac35 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -19,10 +19,7 @@ check cves: - PROJ_PATH=$(echo "${CI_PROJECT_PATH}" | sed -e 's/.*dsop\/\(.*\)/\1/') - export PROJ_PATH - pip3 install jsonschema + - set +e - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" - - | - if [ "$?" == 3 ]; then - exit 3 - fi allow_failure: exit_codes: 3 -- GitLab From 16926d9d7aa89a7a1149cc22418b8a07f9b3a02e Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:49:21 -0700 Subject: [PATCH 18/25] sys.exit on correct exit_code for lint stage --- stages/check-cves/pipeline_wl_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index c563d202..19149c8b 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -143,7 +143,7 @@ def _pipeline_whitelist_compare(image_name, hardening_manifest, lint=False): hardening_manifest=hardening_manifest, ) logging.info(api_exit_code) - sys.exit(1) + sys.exit(api_exit_code) artifacts_path = os.environ["ARTIFACT_STORAGE"] -- GitLab From 48cd77199aa5b51ac7548e184e059f1fb20c0d06 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:54:56 -0700 Subject: [PATCH 19/25] test successful exit --- stages/check-cves/pipeline_wl_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 19149c8b..0952df0a 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -143,7 +143,7 @@ def _pipeline_whitelist_compare(image_name, hardening_manifest, lint=False): hardening_manifest=hardening_manifest, ) logging.info(api_exit_code) - sys.exit(api_exit_code) + sys.exit(0) artifacts_path = os.environ["ARTIFACT_STORAGE"] -- GitLab From 1acc06240f9936cf050df4a7c24b1dc3a13a310f Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:55:44 -0700 Subject: [PATCH 20/25] revert test --- stages/check-cves/pipeline_wl_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 0952df0a..19149c8b 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -143,7 +143,7 @@ def _pipeline_whitelist_compare(image_name, hardening_manifest, lint=False): hardening_manifest=hardening_manifest, ) logging.info(api_exit_code) - sys.exit(0) + sys.exit(api_exit_code) artifacts_path = os.environ["ARTIFACT_STORAGE"] -- GitLab From 760e608166515d89b4a62032691a9c671bb8eb1e Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 13:59:04 -0700 Subject: [PATCH 21/25] remove commented code --- stages/lint/base.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index b4897cb1..b4846cab 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -64,13 +64,6 @@ wl compare lint: - 'mkdir -p "${ARTIFACT_DIR}"' - set +e - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' - # - | - # if [ $? == 3 ]; then - # exit 3 - # else - # echo "something is wrong" - # fi - # - exit 3 allow_failure: exit_codes: 3 artifacts: -- GitLab From c8378c17dfde7421db249f6c5b51c70ceac1e9be Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 14:01:43 -0700 Subject: [PATCH 22/25] reformat with black --- stages/check-cves/pipeline_wl_compare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 19149c8b..56ed587d 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -34,6 +34,7 @@ import swagger_to_jsonschema # TODO: Remove api_exit_code when converting to using the api instead of the query api_exit_code = 0 + def _connect_to_db(): """ @return mariadb connection -- GitLab From 07b0796a62e8fe617ce9dd20a2550521a78daa53 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 21:19:04 +0000 Subject: [PATCH 23/25] Api server status hotfix --- scripts/hardening_manifest_yaml/generate.py | 1 + stages/check-cves/base.yaml | 3 +++ stages/check-cves/pipeline_wl_compare.py | 24 ++++++++++++++++----- stages/csv-output/pipeline_csv_gen.py | 1 + stages/lint/base.yaml | 3 +++ 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/scripts/hardening_manifest_yaml/generate.py b/scripts/hardening_manifest_yaml/generate.py index 8baa429e..43256fc8 100644 --- a/scripts/hardening_manifest_yaml/generate.py +++ b/scripts/hardening_manifest_yaml/generate.py @@ -22,6 +22,7 @@ def _fetch_file(url, file, branch="development"): Grabs a raw file from gitlab. """ + assert "/" not in branch url = f"{url}/-/raw/{branch}/{file}" logger.debug(url) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index 007c63e8..4a53ac35 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -19,4 +19,7 @@ check cves: - PROJ_PATH=$(echo "${CI_PROJECT_PATH}" | sed -e 's/.*dsop\/\(.*\)/\1/') - export PROJ_PATH - pip3 install jsonschema + - set +e - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" + allow_failure: + exit_codes: 3 diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index bfa2e868..56ed587d 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -30,6 +30,10 @@ from scanners import anchore from scanners import twistlock import swagger_to_jsonschema +# add global var for api failures. +# TODO: Remove api_exit_code when converting to using the api instead of the query +api_exit_code = 0 + def _connect_to_db(): """ @@ -139,7 +143,8 @@ def _pipeline_whitelist_compare(image_name, hardening_manifest, lint=False): whitelist_branch=wl_branch, hardening_manifest=hardening_manifest, ) - sys.exit(0) + logging.info(api_exit_code) + sys.exit(api_exit_code) artifacts_path = os.environ["ARTIFACT_STORAGE"] @@ -310,11 +315,15 @@ def _get_greylist_file_contents(image_path, branch): def _vat_findings_query(im_name, im_version): logging.info("Running query to vat api") - url = f"{os.environ['VAT_BACKEND_SERVER_ADDRESS']}/internal/container?name={im_name}&tag={im_version}" - logging.info(f"GET {url}") try: - r = requests.get(url) + r = requests.get( + f"{os.environ['VAT_BACKEND_SERVER_ADDRESS']}/internal/container", + params={ + "name": im_name, + "tag": im_version, + }, + ) except requests.exceptions.RequestException as e: logging.warning(f"Could not access VAT API: {url}") logging.warning(e) @@ -353,7 +362,8 @@ def _vat_findings_query(im_name, im_version): logging.warning(f"Unknown response from VAT {r.status_code}") logging.warning(r.text) logging.error("Failing the pipeline, please contact the administrators") - sys.exit(1) + global api_exit_code + api_exit_code = 3 def _vat_approval_query(im_name, im_version): @@ -390,6 +400,7 @@ def _vat_approval_query(im_name, im_version): result = cursor.fetchall() except Error as error: logging.info(error) + sys.exit(1) finally: if conn is not None and conn.is_connected(): conn.close() @@ -437,6 +448,7 @@ def _vat_vuln_query(im_name, im_version): result = cursor.fetchall() except Error as error: logging.info(error) + sys.exit(1) finally: if conn is not None and conn.is_connected(): conn.close() @@ -659,6 +671,8 @@ def main(): hardening_manifest=hardening_manifest, lint=args.lint, ) + logging.info(api_exit_code) + sys.exit(api_exit_code) if __name__ == "__main__": diff --git a/stages/csv-output/pipeline_csv_gen.py b/stages/csv-output/pipeline_csv_gen.py index a61a38e2..836d6e3f 100644 --- a/stages/csv-output/pipeline_csv_gen.py +++ b/stages/csv-output/pipeline_csv_gen.py @@ -346,6 +346,7 @@ def _vat_vuln_query(im_name, im_version): result = cursor.fetchall() except Error as error: logging.info(error) + sys.exit(1) finally: if conn is not None and conn.is_connected(): conn.close() diff --git a/stages/lint/base.yaml b/stages/lint/base.yaml index 01993011..b4846cab 100644 --- a/stages/lint/base.yaml +++ b/stages/lint/base.yaml @@ -62,7 +62,10 @@ wl compare lint: - export PROJ_PATH - pip3 install jsonschema - 'mkdir -p "${ARTIFACT_DIR}"' + - set +e - 'python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" --lint' + allow_failure: + exit_codes: 3 artifacts: paths: - "${ARTIFACT_DIR}/" -- GitLab From 1aa597db11bc35d1d5a4df5bba31714d6c4cbae4 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 14:25:00 -0700 Subject: [PATCH 24/25] remove api failure check from check-cves --- stages/check-cves/base.yaml | 3 --- stages/check-cves/pipeline_wl_compare.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index 4a53ac35..007c63e8 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -19,7 +19,4 @@ check cves: - PROJ_PATH=$(echo "${CI_PROJECT_PATH}" | sed -e 's/.*dsop\/\(.*\)/\1/') - export PROJ_PATH - pip3 install jsonschema - - set +e - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" - allow_failure: - exit_codes: 3 diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 56ed587d..1835ae54 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -671,8 +671,6 @@ def main(): hardening_manifest=hardening_manifest, lint=args.lint, ) - logging.info(api_exit_code) - sys.exit(api_exit_code) if __name__ == "__main__": -- GitLab From b0f042215de0975f486a8024b3531367d99c2c22 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Fri, 26 Feb 2021 14:33:24 -0700 Subject: [PATCH 25/25] remove api error check in check-cves --- stages/check-cves/base.yaml | 3 --- stages/check-cves/pipeline_wl_compare.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/stages/check-cves/base.yaml b/stages/check-cves/base.yaml index 4a53ac35..007c63e8 100644 --- a/stages/check-cves/base.yaml +++ b/stages/check-cves/base.yaml @@ -19,7 +19,4 @@ check cves: - PROJ_PATH=$(echo "${CI_PROJECT_PATH}" | sed -e 's/.*dsop\/\(.*\)/\1/') - export PROJ_PATH - pip3 install jsonschema - - set +e - python3 "${PIPELINE_REPO_DIR}/stages/check-cves/pipeline_wl_compare.py" - allow_failure: - exit_codes: 3 diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 56ed587d..1835ae54 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -671,8 +671,6 @@ def main(): hardening_manifest=hardening_manifest, lint=args.lint, ) - logging.info(api_exit_code) - sys.exit(api_exit_code) if __name__ == "__main__": -- GitLab