From 5059cd1921f0d85c2d1273134d5abeb95d68ee53 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:12:30 -0700 Subject: [PATCH 1/7] fix backwards compatibility to not always load greylist for im_name and im_tag --- stages/check-cves/pipeline_wl_compare.py | 36 +++++++++++------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 32bcccb0..96c0805c 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -334,7 +334,7 @@ def _get_vulns_from_query(row): return vuln_dict -def _next_ancestor(image_path, greylist, hardening_manifest=None): +def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): """ Grabs the parent image path from the current context. Will initially attempt to load a new hardening manifest and then pull the parent image from there. Otherwise it will @@ -347,15 +347,18 @@ def _next_ancestor(image_path, greylist, hardening_manifest=None): # Try to get the parent image out of the local hardening_manifest. if hardening_manifest: - return hardening_manifest["args"]["BASE_IMAGE"] + return (hardening_manifest["args"]["BASE_IMAGE"], hardening_manifest["args"]["BASE_TAG"]) # Try to load the hardening manifest from a remote repo. hm = _load_remote_hardening_manifest(project=image_path) if hm is not None: - return hm["args"]["BASE_IMAGE"] + return (hm["args"]["BASE_IMAGE"], hm["args"]["BASE_TAG"]) try: - return greylist["image_parent_name"] + greylist = _get_greylist_file_contents( + image_path=image_path, branch=whitelist_branch + ) + return (greylist["image_parent_name"],greylist["image_parent_tag"]) except KeyError as e: logging.error("Looks like a hardening_manifest.yaml cannot be found") logging.error( @@ -373,10 +376,6 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma """ total_whitelist = list() - # TODO: remove after 30 day hardening_manifest merge cutoff - greylist = _get_greylist_file_contents( - image_path=image_name, branch=whitelist_branch - ) logging.info(f"Grabbing CVEs for: {image_name}") # get cves from vat result = _vat_vuln_query(os.environ["IMAGE_NAME"], os.environ["IMAGE_VERSION"]) @@ -428,29 +427,26 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma # Use the local hardening manifest to get the first parent. From here *only* the # the master branch should be used for the ancestry. # - parent_image = _next_ancestor( - image_path=image_name, greylist=greylist, hardening_manifest=hardening_manifest + parent_image_name, parent_image_version = _next_ancestor( + image_path=image_name, whitelist_branch=whitelist_branch, hardening_manifest=hardening_manifest ) # get parent cves from VAT - while parent_image: - logging.info(f"Grabbing CVEs for: {parent_image}") - # TODO: remove this after 30 day hardening_manifest merge cutoff - greylist = _get_greylist_file_contents( - image_path=parent_image, branch=whitelist_branch - ) - + while parent_image_name: + logging.info(f"Grabbing CVEs for: {parent_image_name}") + # TODO: remove this after 30 day hardening_manifest merge cutof # TODO: swap this for hardening manifest after 30 day merge cutoff - result = _vat_vuln_query(greylist["image_name"], greylist["image_tag"]) + result = _vat_vuln_query(parent_image_name, parent_image_version) for row in result: vuln_dict = _get_vulns_from_query(row) if vuln_dict["status"] and vuln_dict["status"].lower() == "approve": total_whitelist.append(Vuln(vuln_dict, image_name)) - parent_image = _next_ancestor( + parent_image,parent_image_version = _next_ancestor( image_path=parent_image, - greylist=greylist, + whitelist_branch=whitelist_branch, + hardening_manifest=hardening_manifest, ) logging.info(f"Found {len(total_whitelist)} total whitelisted CVEs") -- GitLab From 09ee992e2ef92aaca30649784d5aa7734ebe36cc Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:13:09 -0700 Subject: [PATCH 2/7] reformat --- stages/check-cves/pipeline_wl_compare.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 96c0805c..b91df556 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -347,7 +347,10 @@ def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): # Try to get the parent image out of the local hardening_manifest. if hardening_manifest: - return (hardening_manifest["args"]["BASE_IMAGE"], hardening_manifest["args"]["BASE_TAG"]) + return ( + hardening_manifest["args"]["BASE_IMAGE"], + hardening_manifest["args"]["BASE_TAG"], + ) # Try to load the hardening manifest from a remote repo. hm = _load_remote_hardening_manifest(project=image_path) @@ -358,7 +361,7 @@ def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): greylist = _get_greylist_file_contents( image_path=image_path, branch=whitelist_branch ) - return (greylist["image_parent_name"],greylist["image_parent_tag"]) + return (greylist["image_parent_name"], greylist["image_parent_tag"]) except KeyError as e: logging.error("Looks like a hardening_manifest.yaml cannot be found") logging.error( @@ -428,7 +431,9 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma # the master branch should be used for the ancestry. # parent_image_name, parent_image_version = _next_ancestor( - image_path=image_name, whitelist_branch=whitelist_branch, hardening_manifest=hardening_manifest + image_path=image_name, + whitelist_branch=whitelist_branch, + hardening_manifest=hardening_manifest, ) # get parent cves from VAT @@ -443,7 +448,7 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma if vuln_dict["status"] and vuln_dict["status"].lower() == "approve": total_whitelist.append(Vuln(vuln_dict, image_name)) - parent_image,parent_image_version = _next_ancestor( + parent_image, parent_image_version = _next_ancestor( image_path=parent_image, whitelist_branch=whitelist_branch, hardening_manifest=hardening_manifest, -- GitLab From d9ebc91b346967b3c01b3ab40341de14a7351691 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:16:06 -0700 Subject: [PATCH 3/7] fix naming convention for consistency --- stages/check-cves/pipeline_wl_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index b91df556..02927d03 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -448,8 +448,8 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma if vuln_dict["status"] and vuln_dict["status"].lower() == "approve": total_whitelist.append(Vuln(vuln_dict, image_name)) - parent_image, parent_image_version = _next_ancestor( - image_path=parent_image, + parent_image_name, parent_image_version = _next_ancestor( + image_path=parent_image_name, whitelist_branch=whitelist_branch, hardening_manifest=hardening_manifest, ) -- GitLab From 77e6b6effa22070a96789558f8fc8883ab6c6a3f Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:21:15 -0700 Subject: [PATCH 4/7] update justifier backwards compat to only pull greylist for im_name and im_tag if hm doesn't exist --- stages/csv-output/justifier.py | 47 ++++++++++++++++------------------ 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/stages/csv-output/justifier.py b/stages/csv-output/justifier.py index 0462683e..f6fb4ae6 100755 --- a/stages/csv-output/justifier.py +++ b/stages/csv-output/justifier.py @@ -102,7 +102,7 @@ def _load_remote_hardening_manifest(project, branch="master"): return None -def _next_ancestor(image_path, greylist, hardening_manifest=None): +def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): """ Grabs the parent image path from the current context. Will initially attempt to load a new hardening manifest and then pull the parent image from there. Otherwise it will @@ -115,17 +115,21 @@ def _next_ancestor(image_path, greylist, hardening_manifest=None): # Try to get the parent image out of the local hardening_manifest. if hardening_manifest: - return hardening_manifest["args"]["BASE_IMAGE"] + return ( + hardening_manifest["args"]["BASE_IMAGE"], + hardening_manifest["args"]["BASE_TAG"], + ) # Try to load the hardening manifest from a remote repo. hm = _load_remote_hardening_manifest(project=image_path) if hm is not None: - logging.debug(hm["args"]["BASE_IMAGE"]) - return hm["args"]["BASE_IMAGE"] + return (hm["args"]["BASE_IMAGE"], hm["args"]["BASE_TAG"]) try: - logging.debug("using greylist for image parent name") - return greylist["image_parent_name"] + greylist = _get_greylist_file_contents( + image_path=image_path, branch=whitelist_branch + ) + return (greylist["image_parent_name"], greylist["image_parent_tag"]) except KeyError as e: logging.error("Looks like a hardening_manifest.yaml cannot be found") logging.error( @@ -287,10 +291,6 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma """ total_whitelist = list() - # TODO: remove after 30 day hardening_manifest merge cutoff - greylist = _get_greylist_file_contents( - image_path=image_name, branch=whitelist_branch - ) logging.info(f"Grabbing CVEs for: {image_name}") # get cves from vat result = _vat_vuln_query(os.environ["IMAGE_NAME"], os.environ["IMAGE_VERSION"]) @@ -320,30 +320,27 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma # Use the local hardening manifest to get the first parent. From here *only* the # the master branch should be used for the ancestry. # - parent_image = _next_ancestor( - image_path=image_name, greylist=greylist, hardening_manifest=hardening_manifest + parent_image_name, parent_image_version = _next_ancestor( + image_path=image_name, + whitelist_branch=whitelist_branch, + hardening_manifest=hardening_manifest, ) # get parent cves from VAT - while parent_image: - logging.info(f"Grabbing CVEs for: {parent_image}") - # TODO: remove this after 30 day hardening_manifest merge cutoff - greylist = _get_greylist_file_contents( - image_path=parent_image, branch=whitelist_branch - ) + while parent_image_name: + logging.info(f"Grabbing CVEs for: {parent_image_name}") + + result = _vat_vuln_query(parent_image_name, parent_image_version) - # TODO: swap this for hardening manifest after 30 day merge cutoff - result = _vat_vuln_query(greylist["image_name"], greylist["image_tag"]) - # logging.debug(result[0]) for row in result: - logging.debug(row) vuln_dict = _get_vulns_from_query(row) if vuln_dict["status"] and vuln_dict["status"].lower() == "approve": total_whitelist.append(vuln_dict) - parent_image = _next_ancestor( - image_path=parent_image, - greylist=greylist, + parent_image_name, parent_image_version = _next_ancestor( + image_path=parent_image_name, + whitelist_branch=whitelist_branch, + hardening_manifest=hardening_manifest, ) logging.info(f"Found {len(total_whitelist)} total whitelisted CVEs") -- GitLab From 8595c89aff2ddc4a94a4a493ed753b112e3bb9e8 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:33:39 -0700 Subject: [PATCH 5/7] remove check for local hardening_manifest to avoid inf loop --- stages/check-cves/pipeline_wl_compare.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index 02927d03..f54c5e45 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -345,12 +345,6 @@ def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): """ - # Try to get the parent image out of the local hardening_manifest. - if hardening_manifest: - return ( - hardening_manifest["args"]["BASE_IMAGE"], - hardening_manifest["args"]["BASE_TAG"], - ) # Try to load the hardening manifest from a remote repo. hm = _load_remote_hardening_manifest(project=image_path) -- GitLab From 971685f3f72711ca48311b0d46badd47792ffe15 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:53:34 -0700 Subject: [PATCH 6/7] remove hardening_manifest from inner loop --- stages/check-cves/pipeline_wl_compare.py | 2 -- stages/csv-output/justifier.py | 1 - 2 files changed, 3 deletions(-) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index f54c5e45..c88d9c49 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -345,7 +345,6 @@ def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): """ - # Try to load the hardening manifest from a remote repo. hm = _load_remote_hardening_manifest(project=image_path) if hm is not None: @@ -445,7 +444,6 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma parent_image_name, parent_image_version = _next_ancestor( image_path=parent_image_name, whitelist_branch=whitelist_branch, - hardening_manifest=hardening_manifest, ) logging.info(f"Found {len(total_whitelist)} total whitelisted CVEs") diff --git a/stages/csv-output/justifier.py b/stages/csv-output/justifier.py index f6fb4ae6..1dc03bde 100755 --- a/stages/csv-output/justifier.py +++ b/stages/csv-output/justifier.py @@ -340,7 +340,6 @@ def _get_complete_whitelist_for_image(image_name, whitelist_branch, hardening_ma parent_image_name, parent_image_version = _next_ancestor( image_path=parent_image_name, whitelist_branch=whitelist_branch, - hardening_manifest=hardening_manifest, ) logging.info(f"Found {len(total_whitelist)} total whitelisted CVEs") -- GitLab From d794eabecc1740d1ed138aa6f049d8e0fbe5ac92 Mon Sep 17 00:00:00 2001 From: Kenneth Maguire Date: Wed, 6 Jan 2021 17:56:57 -0700 Subject: [PATCH 7/7] add check for local hardening manifest --- stages/check-cves/pipeline_wl_compare.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stages/check-cves/pipeline_wl_compare.py b/stages/check-cves/pipeline_wl_compare.py index c88d9c49..d54a1023 100644 --- a/stages/check-cves/pipeline_wl_compare.py +++ b/stages/check-cves/pipeline_wl_compare.py @@ -345,6 +345,13 @@ def _next_ancestor(image_path, whitelist_branch, hardening_manifest=None): """ + # Try to get the parent image out of the local hardening_manifest. + if hardening_manifest: + return ( + hardening_manifest["args"]["BASE_IMAGE"], + hardening_manifest["args"]["BASE_TAG"], + ) + # Try to load the hardening manifest from a remote repo. hm = _load_remote_hardening_manifest(project=image_path) if hm is not None: -- GitLab