Clean up latest code refactoring, update tests #970

johnmhoran · johnmhoran · commit 4389f451d2d0 · 2023-01-20T10:57:28.000-08:00
Reference: #970 Signed-off-by: John M. Horan <johnmhoran@gmail.com>
diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py
@@ -68,7 +68,8 @@ def advisory_data(self):
         """
         advisories = []
 
-        for advisory_page in self.fetch_advisory_pages(self.security_updates_home):
+        # for advisory_page in self.fetch_advisory_pages(self.security_updates_home):
+        for advisory_page in self.fetch_advisory_pages():
             advisories.extend(self.extract_advisories_from_page(advisory_page))
 
         if TRACE:
@@ -179,83 +180,6 @@ def extract_advisories_from_page(self, apache_tomcat_advisory_html):
 
         return advisories
 
-    # XXX: 2023-01-02 Monday 14:48:57.  We need to convert this from class method to
-    # independent function because it's now called by another independent function and thus has no `self`.
-    # def to_version_ranges(self, versions_data, fixed_versions):
-    #     constraints = []
-
-    #     for version_item in versions_data:
-    #         if "to" in version_item:
-    #             version_item_split = version_item.split(" ")
-
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator=">=",
-    #                     version=MavenVersion(version_item_split[0]),
-    #                 )
-    #             )
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator="<=",
-    #                     version=MavenVersion(version_item_split[-1]),
-    #                 )
-    #             )
-
-    #         elif "-" in version_item:
-    #             version_item_split = version_item.split("-")
-
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator=">=",
-    #                     version=MavenVersion(version_item_split[0]),
-    #                 )
-    #             )
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator="<=",
-    #                     version=MavenVersion(version_item_split[-1]),
-    #                 )
-    #             )
-
-    #         else:
-    #             version_item_split = version_item.split(" ")
-
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator="=",
-    #                     version=MavenVersion(version_item_split[0]),
-    #                 )
-    #             )
-
-    #     for fixed_item in fixed_versions:
-    #         if "-" in fixed_item and not any([i.isalpha() for i in fixed_item]):
-    #             fixed_item_split = fixed_item.split(" ")
-
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator=">=",
-    #                     version=MavenVersion(fixed_item_split[0]),
-    #                 ).invert()
-    #             )
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator="<=",
-    #                     version=MavenVersion(fixed_item_split[-1]),
-    #                 ).invert()
-    #             )
-
-    #         else:
-    #             fixed_item_split = fixed_item.split(" ")
-
-    #             constraints.append(
-    #                 VersionConstraint(
-    #                     comparator="=",
-    #                     version=MavenVersion(fixed_item_split[0]),
-    #                 ).invert()
-    #             )
-
-    #     return MavenVersionRange(constraints=constraints)
-
 
 @dataclasses.dataclass(order=True)
 class TomcatAdvisoryData:
@@ -563,224 +487,3 @@ def to_version_ranges(versions_data, fixed_versions):
             )
 
     return MavenVersionRange(constraints=constraints)
-
-    # =====================================================
-
-    # for para in para_list:
-    #     print("type(para) = {}".format(type(para)))
-    #     cve_p = para.find(
-    #         "strong",
-    #         # text=lambda text: text and text.startswith(tuple(vuln_p_list)),
-    #         text=lambda text: text and text.startswith(severities),
-    #     )
-    #     # 2022-12-31 Saturday 20:16:51.  Seems this is False for the missing pair of CVEs?
-    #     if cve_p:
-    #         cve_p_parent = cve_p.find_parent()
-    #         print("\ncve_p = {}\n".format(cve_p))
-    #         print("\ncve_p_parent = {}\n".format(cve_p_parent))
-
-    #         # TODO: 2022-12-31 Saturday 18:58:00.  Why does this omit the 1st pair, CVE-2020-9484 and CVE-2021-25329?
-    #         # Seems cve_p_parent (just above) is also omitted for that data?
-    #         cve_url_list = cve_p_parent.find_all("a")
-    #         print("cve_url_list = {}\n".format(cve_url_list))
-
-    #         cve_id_list = [cve_text.text for cve_text in cve_url_list]
-    #         print("cve_id_list = {}\n".format(cve_id_list))
-
-    #         for cve_id in cve_id_list:
-
-    #             test_dict = {
-    #                 "aliases": [cve_id],
-    #                 "fixed_version": fixed_version,
-    #                 "affected_versions": [],
-    #             }
-
-    #             yield test_dict
-
-    #     else:
-    #         print("\n\nALERT!  This para failed the `if_cve_p` test: {} \n\n".format(para))
-
-
-# ========================================================
-# severity_score = para.text.split(" ")[0]
-# severity_score = severity_score.split(":")[0]
-# print("\nseverity_score = {}\n".format(severity_score))
-
-# print("===")
-
-# better_cve_url_list = para.find_all("a")
-# print("better_cve_url_list = {}\n".format(better_cve_url_list))
-
-# better_cve_id_list = [cve_text.text for cve_text in better_cve_url_list]
-# print("better_cve_id_list = {}\n".format(better_cve_id_list))
-
-# for better_cve_url in para.find_all("a"):
-#     print("better_cve_url = {}\n".format(better_cve_url))
-#     print("better_cve_url.text = {}".format(better_cve_url.text))
-
-# better_nextSiblings = para.find_next_siblings()
-
-# print("===")
-
-# section_parent = para.find_parent()
-
-# cve_url_list = section_parent.find_all("a")
-# print("cve_url_list = {}\n".format(cve_url_list))
-
-# cve_id_list = [cve_text.text for cve_text in cve_url_list]
-# print("cve_id_list = {}\n".format(cve_id_list))
-
-# for cve_url in section_parent.find_all("a"):
-#     print("cve_url = {}\n".format(cve_url))
-#     print("cve_url.text = {}".format(cve_url.text))
-
-# nextSiblings = section_parent.find_next_siblings()
-
-# print("===")
-
-# fixed_commit_list = []
-# affected_versions = []
-
-# for sib in better_nextSiblings:
-#     if "was fixed in" in sib.text or "was fixed with" in sib.text:
-#         print("\nnext sib (was fixed) = {}".format(sib))
-#         fixed_commit_list = sib.find_all("a")
-#         print("\nfixed_commit_list = {}".format(fixed_commit_list))
-
-#     elif "Affects" in sib.text:
-#         print("\nnext sib (affects) = {}\n".format(sib))
-
-#         # 2022-12-27 Tuesday 18:47:28.  We need the list of `sib` elements to examine -- and test -- the raw HTML.
-#         self.record_of_all_affects_elements.append(sib)
-
-#         # 2022-12-27 Tuesday 14:47:51.  We'll examine the affects_string and try to find and remove unwanted alpha and related characters/strings.
-#         # ===
-#         # This version is before stripping/replacing etc.
-#         affects_string = sib.text.split("Affects:")[-1].strip()
-#         print("affects_string = {}\n".format(affects_string))
-#         self.record_of_all_affects_strings.append(affects_string)
-
-#         affected_versions = affects_string.split(", ")
-#         print("> affected_versions = {}\n".format(affected_versions))
-#         self.record_of_all_affected_version_strings.append(affected_versions)
-#         # ===
-#         # This version is with most but not all of the stripping/replacing.
-#         # affects_string = sib.text.split("Affects:")[-1].strip()
-#         # affects_string = affects_string.replace("\n", "")
-#         # affects_string = " ".join(affects_string.split())
-#         # affects_string_no_parens = re.sub(r" ?\([^)]+\)", "", affects_string)
-#         # # print("affects_string = {}\n".format(affects_string))
-#         # # self.record_of_all_affects_strings.append(affects_string)
-#         # print(
-#         #     "affects_string_no_parens = {}\n".format(affects_string_no_parens)
-#         # )
-#         # self.record_of_all_affects_strings.append(affects_string_no_parens)
-
-#         # # affected_versions = affects_string.split(", ")
-#         # affected_versions = affects_string_no_parens.split(", ")
-#         # print("> affected_versions = {}\n".format(affected_versions))
-#         # self.record_of_all_affected_version_strings.append(affected_versions)
-#         # ===
-
-#     elif sib.find_all(
-#         "strong",
-#         text=lambda text: text and text.startswith(tuple(severities)),
-#     ):
-#         break
-
-# # Starting to flesh out this new approach.
-# # for cve_id_record in cve_id_list:
-# #     test_advisory_dict["fixed_versions"] = fixed_versions
-# #     test_advisory_dict["aliases"] = [cve_id_record]
-
-# temp_dict_list = []
-
-# print("\n1.  affected_versions = {}\n".format(affected_versions))
-
-# for better_cve_id_record in better_cve_id_list:
-#     # 2022-12-26 Monday 14:41:11.  This is where `test_advisory_dict = {}` belongs!  Now we have data for the double-CVEs!
-#     test_advisory_dict = {}
-#     test_advisory_dict["fixed_versions"] = fixed_versions
-#     test_advisory_dict["aliases"] = [better_cve_id_record]
-
-#     self.updated_temp_advisory_dict_list.append(test_advisory_dict)
-
-#     print(
-#         "==========================> better_cve_id_record = {}".format(
-#             better_cve_id_record
-#         )
-#     )
-
-#     temp_dict_list.append(test_advisory_dict)
-
-#     # TODO: 2022-12-26 Monday 14:55:49.  Is this where we build the "better" AdvisoryData() objects?
-#     better_severities = []
-#     better_severities.append(
-#         VulnerabilitySeverity(
-#             system=APACHE_TOMCAT,
-#             value=severity_score,
-#             scoring_elements="",
-#         )
-#     )
-
-#     print("\n2.  affected_versions = {}\n".format(affected_versions))
-
-#     affected_version_range = self.to_version_ranges(
-#         # versions_data, fixed_versions
-#         affected_versions,
-#         # TODO: 2022-12-26 Monday 16:01:08.  fix this!
-#         # ["1.1"],
-#         # ["8.5.0 to 8.5.4", " 8.0.0.RC1 to 8.0.36"],
-#         # ["3.0", " 3.1-3.1.1", " 3.2-3.2.1"],
-#         # TODO: 2022-12-26 Monday 17:56:04.  This identified the problem -- a space at the start of the 2nd range!
-#         # ["8.5.0 to 8.5.4", "8.0.0.RC1 to 8.0.36"],
-#         fixed_versions,
-#     )
-#     references = [
-#         Reference(
-#             # url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_id}",
-#             url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={better_cve_id_record}",
-#             # reference_id=cve_id,
-#             reference_id=better_cve_id_record,
-#             # severities=severities,
-#             severities=better_severities,
-#         ),
-#     ]
-
-#     # for commit_url in fixed_in_commits:
-#     for commit_url in fixed_commit_list:
-#         # references.append(Reference(url=commit_url))
-#         # TODO: 2022-12-26 Monday 17:23:38.  Does this fix the error TypeError: Object of type Tag is not JSON serializable?  Yes.
-#         references.append(Reference(url=commit_url["href"]))
-
-#     # 2022-12-26 Monday 15:37:02.  Does this belong here?
-#     affected_packages = []
-
-#     affected_packages.append(
-#         AffectedPackage(
-#             package=PackageURL(
-#                 type="maven",
-#                 namespace="apache",
-#                 name="tomcat",
-#             ),
-#             affected_version_range=affected_version_range,
-#         )
-#     )
-
-#     advisories.append(
-#         AdvisoryData(
-#             aliases=[better_cve_id_record],
-#             summary="",
-#             affected_packages=affected_packages,
-#             references=references,
-#         )
-#     )
-
-# self.temp_list_of_fixed_versions.append(fixed_versions)
-
-# yield AdvisoryData(
-#     aliases=aliases,
-#     summary="",
-#     affected_packages=affected_packages,
-#     references=references,
-# )
diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py
@@ -28,7 +28,7 @@
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_tomcat")
 
-security_updates_home = "https://tomcat.apache.org/security"
+# security_updates_home = "https://tomcat.apache.org/security"
 
 
 # Temp test to flesh out `extract_advisories_from_page()` -- the method .
@@ -311,9 +311,17 @@ def test_advisory_data():
 
 
 def test_fetch_links():
-    retrieved_links = ApacheTomcatImporter().fetch_advisory_links(security_updates_home)
+    # retrieved_links = ApacheTomcatImporter().fetch_advisory_links(security_updates_home)
+    retrieved_links = ApacheTomcatImporter().fetch_advisory_links(
+        "https://tomcat.apache.org/security"
+    )
+
+    generator_result = []
+    for link in retrieved_links:
+        generator_result.append(link)
 
-    assert retrieved_links == [
+    assert generator_result == [
+        "https://tomcat.apache.org/security-11.html",
         "https://tomcat.apache.org/security-10.html",
         "https://tomcat.apache.org/security-9.html",
         "https://tomcat.apache.org/security-8.html",
