Update tests for new extract_advisories_from_page() #970

Reference: #970

Signed-off-by: John M. Horan <[email protected]>

Author: johnmhoran

vulnerabilities/importers/apache_tomcat.py

@@ -25,7 +25,7 @@
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.severity_systems import APACHE_TOMCAT
 
-TRACE = True
+TRACE = False
 
 
 class ApacheTomcatImporter(Importer):
@@ -89,7 +89,6 @@ def debug_advisory_data(self, advisories):
             adv_dict = adv.to_dict()
             temp_advisory_to_dict_list.append(adv_dict)
 
-        # TODO: 2022-12-26 Monday 16:17:38.  With my new ("better") approach, throws error: TypeError: Object of type Tag is not JSON serializable
         with open(
             "apache_tomcat_advisories_to_dict-02.json",
             "w",
@@ -133,6 +132,8 @@ def debug_advisory_data(self, advisories):
             for line in self.record_of_all_affected_version_strings:
                 f.write(f"{line}\n")
 
+    # 2022-12-29 Thursday 13:11:16.  We're in the process of refactoring this method.
+    # See, e.g., function with the same name at the bottom of this file.
     def extract_advisories_from_page(self, apache_tomcat_advisory_html):
         """
         Return a list of AdvisoryData extracted from the HTML text ``apache_tomcat_advisory_html``.
@@ -389,8 +390,6 @@ def to_version_ranges(self, versions_data, fixed_versions):
                 )
 
             elif "-" in version_item:
-                # elif "-" in version_item and not any([i.isalpha() for i in version_item]):
-                # version_item_split = version_item.split(" ")
                 version_item_split = version_item.split("-")
 
                 constraints.append(
@@ -479,6 +478,7 @@ def extract_advisories_from_page(apache_tomcat_advisory_html):
         fixed_version = fixed_version_heading.text.split("Fixed in Apache Tomcat")[-1].strip()
         if TRACE:
             print("fixed_version = {}".format(fixed_version))
+            print("===========================")
 
         # We want to handle the occasional "and" in the fixed version headers, e.g.,
         # <h3 id="Fixed_in_Apache_Tomcat_8.5.5_and_8.0.37"><span class="pull-right">5 September 2016</span> Fixed in Apache Tomcat 8.5.5 and 8.0.37</h3>
@@ -489,6 +489,7 @@ def extract_advisories_from_page(apache_tomcat_advisory_html):
 
         if TRACE:
             print("fixed_versions = {}".format(fixed_versions))
+            print("===========================")
 
         # Each group of fixed-version-related data is contained in a div that immediately follows the h3 element, e.g.,
         # <h3 id="Fixed_in_Apache_Tomcat_8.5.8"><span class="pull-right">8 November 2016</span> Fixed in Apache Tomcat 8.5.8</h3>
@@ -502,20 +503,23 @@ def extract_advisories_from_page(apache_tomcat_advisory_html):
         severities = ("Low:", "Moderate:", "Important:", "High:", "Critical:")
         # A list of groups of paragraphs, each for a single Tomcat Advisory.
         advisory_groups = []
-        current_group = []
+
         for para in fixed_version_paras.find_all("p"):
+            current_group = []
             if para.text.startswith(severities):
-                if current_group:
-                    current_group = []
-                else:
-                    advisory_groups.append(current_group)
-
                 current_group.append(para)
 
-            else:
-                if current_group:
-                    current_group.append(para)
-                else:
-                    pass
+                test_nextSiblings = para.find_next_siblings()
+                for next_sibling in test_nextSiblings:
+                    if not next_sibling.text.startswith(severities):
+                        current_group.append(next_sibling)
+                    elif next_sibling.text.startswith(severities):
+                        break
+
+                advisory_groups.append(current_group)
+
+        if TRACE:
+            print("\ncurrent_group = {}\n".format(current_group))
+            print("\nadvisory_groups = {}\n".format(advisory_groups))
 
         yield TomcatAdvisoryData(fixed_versions=fixed_versions, advisory_groups=advisory_groups)

vulnerabilities/tests/test_apache_tomcat.py

@@ -59,6 +59,8 @@ def test_extract_advisories_from_page():
                     'href="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-30639" '
                     'rel="nofollow">CVE-2021-30639</a></p>',
                     "<p>An error introduced as part of a change to improve " "error handling.</p>",
+                    "<o>Applications that do not use non-blocking I/O are "
+                    "not exposed to this vulnerability.</o>",
                     "<p>This was fixed with commit\n"
                     "        <a "
                     'href="https://github.com/apache/tomcat/commit/b59099e4ca501a039510334ebe1024971cd6f959">b59099e4</a>.</p>',
@@ -200,7 +202,29 @@ def test_extract_advisories_from_page_with_multiple_groups():
                     "The issue was made public\n"
                     "       on 1 March 2021.</p>",
                     "<p>Affects: 10.0.0-M1 to 10.0.0</p>",
-                ]
+                ],
+                [
+                    "<p><strong>Important: Request mix-up with "
+                    "h2c</strong>\n"
+                    "<a "
+                    'href="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-25122" '
+                    'rel="nofollow">CVE-2021-25122</a></p>',
+                    "<p>When responding to new h2c connection requests, "
+                    "Apache Tomcat could\n"
+                    "    duplicate request headers and a limited amount of "
+                    "request body from one\n"
+                    "    request to another meaning user A and user B could "
+                    "both see the results of\n"
+                    "    user A's request.</p>",
+                    "<p>This was fixed with commit\n"
+                    "       <a "
+                    'href="https://github.com/apache/tomcat/commit/dd757c0a893e2e35f8bc1385d6967221ae8b9b9b">dd757c0a</a>.</p>',
+                    "<p>This issue was identified by the Apache Tomcat "
+                    "Security team on 11\n"
+                    "       January 2021. The issue was made public on 1 "
+                    "March 2021.</p>",
+                    "<p>Affects: 10.0.0-M1 to 10.0.0</p>",
+                ],
             ],
             "fixed_versions": ["10.0.2"],
         },
@@ -232,11 +256,40 @@ def test_extract_advisories_from_page_with_multiple_groups():
                     "       on 26 October 2020. The issue was made public "
                     "on 14 January 2021.</p>",
                     "<p>Affects: 10.0.0-M1 to 10.0.0-M9</p>",
-                ]
+                ],
+                [
+                    "<p><strong>Moderate: HTTP/2 request header "
+                    "mix-up</strong>\n"
+                    "<a "
+                    'href="http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-17527" '
+                    'rel="nofollow">CVE-2020-17527</a></p>',
+                    "<p>While investigating issue <a "
+                    'href="https://bz.apache.org/bugzilla/show_bug.cgi?id=64830">64830</a> '
+                    "it was discovered that Apache\n"
+                    "       Tomcat could re-use an HTTP request header "
+                    "value from the previous stream\n"
+                    "       received on an HTTP/2 connection for the "
+                    "request associated with the\n"
+                    "       subsequent stream. While this would most likely "
+                    "lead to an error and the\n"
+                    "       closure of the HTTP/2 connection, it is "
+                    "possible that information could\n"
+                    "       leak between requests.\n"
+                    "    </p>",
+                    "<p>This was fixed with commit\n"
+                    "       <a "
+                    'href="https://github.com/apache/tomcat/commit/8d2fe6894d6e258a6d615d7f786acca80e6020cb">8d2fe689</a>.</p>',
+                    "<p>This issue was identified by the Apache Tomcat "
+                    "Security team on 10\n"
+                    "       November 2020. The issue was made public on 3 "
+                    "December 2020.</p>",
+                    "<p>Affects: 10.0.0-M1 to 10.0.0-M9</p>",
+                ],
             ],
             "fixed_versions": ["10.0.0-M10"],
         },
     ]
+
     results = extract_advisories_from_page(page)
     results = [d.to_dict() for d in results]
     assert results == expected