Adjust the code according to changes in postgres html page

Fixed get_or_create_from_purl

Signed-off-by: Tushar Goel <[email protected]>

Author: TG1999

vulnerabilities/importers/postgresql.py

@@ -99,7 +99,9 @@ def to_advisories(data):
             # in the prior code, this is the only place where cve_id was defined, and presumably
             # there was no error like the error we got:
             # UnboundLocalError: local variable 'cve_id' referenced before assignment
-            cve_id = ref_col.select("nobr")[0].text
+
+            # changed from nobr to .nobr due to html changes
+            cve_id = ref_col.select(".nobr")[0].text
             # This is for the anomaly in https://www.postgresql.org/support/security/8.1/ 's
             # last entry
         except IndexError:
@@ -142,7 +144,7 @@ def to_advisories(data):
 
 
 def find_advisory_urls(page_data):
-    soup = BeautifulSoup(page_data)
+    soup = BeautifulSoup(page_data, features="lxml")
     return {
         urlparse.urljoin("https://www.postgresql.org/", a_tag.attrs["href"])
         for a_tag in soup.select("h3+ p a")

vulnerabilities/models.py

@@ -324,6 +324,20 @@ def get_or_create_from_purl(self, purl: PackageURL):
         ``purl`` PackageURL.
         """
         purl_fields = without_empty_values(purl.to_dict(encode=True))
+
+        # when there are 2 packages one with qualifiers and one without
+        # qualifiers, having all other fields same, this raises MultipleObjectsReturned
+        # so we are filling out the fields with empty value to avoid this
+        for field in PackageURL._fields:
+            # name, type, and version are required fields
+            if field not in purl_fields:
+                if field == "namespace":
+                    purl_fields[field] = ""
+                if field == "qualifiers":
+                    purl_fields[field] = {}
+                if field == "subpath":
+                    purl_fields[field] = ""
+
         package, _ = Package.objects.get_or_create(**purl_fields)
         return package
 

vulnerabilities/tests/test_data/postgresql/advisories.html

@@ -109,7 +109,7 @@ <h2>Known security issues in all supported versions</h2>
 
     <tr>
       <td>
-        <nobr><a href="/support/security/CVE-2020-10733/">CVE-2020-10733</a></nobr><br>
+        <a href="/support/security/CVE-2020-10733/" class="nobr">CVE-2020-10733</a><br>
         <a href="/about/news/postgresql-123-118-1013-9618-and-9522-released-2038/">Announcement</a><br>
       </td>
       <td>12, 11, 10, 9.6</td>
@@ -122,7 +122,7 @@ <h2>Known security issues in all supported versions</h2>
 
       <tr>
         <td>
-          <nobr><a href="/support/security/CVE-2020-1720/">CVE-2020-1720</a></nobr><br>
+          <a href="/support/security/CVE-2020-1720/" class="nobr">CVE-2020-1720</a><br>
           <a href="/about/news/postgresql-122-117-1012-9617-9521-and-9426-released-2011/">Announcement</a><br>
         </td>
         <td>12, 11, 10, 9.6</td>