Revert "Backporting #3963 (#3986)" (#3987)

This reverts commit 87a13c7.

Author: valentijnscholten

dojo/settings/settings.dist.py

@@ -863,7 +863,6 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param
     'Trivy Scan': DEDUPE_ALGO_HASH_CODE,
     'HackerOne Cases': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE,
     'Snyk Scan': DEDUPE_ALGO_HASH_CODE,
-    'Safety Scan': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL,
 }
 
 DUPE_DELETE_MAX_PER_RUN = env('DD_DUPE_DELETE_MAX_PER_RUN')

dojo/tools/safety/parser.py

@@ -18,63 +18,73 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Safety scan (--json) output file can be imported in JSON format."
 
-    def get_safetydb(self):
-        """Grab Safety DB for CVE lookup"""
+    def get_findings(self, json_output, test):
+
+        # Grab Safety DB for CVE lookup
         url = "https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json"
         try:
             response = urllib.request.urlopen(url)
-            return json.load(response)
+            safety_db = json.loads(response.read().decode('utf-8'))
         except urllib.error.URLError as e:
             logger.warn("Error Message: %s", e)
             logger.warn("Could not resolve %s. Fallback is using the offline version from dojo/tools/safety/insecure_full.json.", url)
-            with open("dojo/tools/safety/insecure_full.json", "r") as insecure_full:
-                return json.load(insecure_full)
+            with open("dojo/tools/safety/insecure_full.json", "r") as f:
+                safety_db = json.load(f)
+            f.close()
 
-    def get_findings(self, json_output, test):
-        safety_db = self.get_safetydb()
+        tree = self.parse_json(json_output)
+        return self.get_items(tree, test, safety_db)
 
-        tree = json.load(json_output)
+    def parse_json(self, json_output):
+        data = json_output.read() or '[]'
+        try:
+            json_obj = json.loads(str(data, 'utf-8'))
+        except:
+            json_obj = json.loads(data)
+        tree = {l[4]: {'package': str(l[0]),
+                       'affected': str(l[1]),
+                       'installed': str(l[2]),
+                       'description': str(l[3]),
+                       'id': str(l[4])}
+                for l in json_obj}  # noqa: E741
+        return tree
 
+    def get_items(self, tree, test, safety_db):
         items = {}
-        for node in tree:
-            item_node = {
-                'package': str(node[0]),
-                'affected': str(node[1]),
-                'installed': str(node[2]),
-                'description': str(node[3]),
-                'id': str(node[4])
-            }
-            severity = 'Medium'  # Because Safety doesn't include severity rating
-            cve = None
-            for a in safety_db[item_node['package']]:
-                if a['id'] == 'pyup.io-' + item_node['id']:
-                    if a['cve']:
-                        cve = a['cve']
-            title = item_node['package'] + " (" + item_node['affected'] + ")"
-
-            finding = Finding(title=title + " | " + cve if cve else title,
-                            test=test,
-                            severity=severity,
-                            description="**Description:** " + item_node['description'] +
-                                        "\n**Vulnerable Package:** " + item_node['package'] +
-                                        "\n**Installed Version:** " + item_node['installed'] +
-                                        "\n**Vulnerable Versions:** " + item_node['affected'] +
-                                        "\n**CVE:** " + (cve or "N/A") +
-                                        "\n**ID:** " + item_node['id'],
-                            cve=cve,
-                            cwe=1035,  # Vulnerable Third Party Component
-                            mitigation="No mitigation provided",
-                            references="No reference provided",
-                            active=False,
-                            verified=False,
-                            false_p=False,
-                            duplicate=False,
-                            out_of_scope=False,
-                            mitigated=None,
-                            impact="No impact provided",
-                            component_name=item_node['package'],
-                            component_version=item_node['installed'],
-                            unique_id_from_tool=item_node['id'])
-            items[finding.unique_id_from_tool] = finding
+
+        for key, node in tree.items():
+            item = get_item(node, test, safety_db)
+            items[key] = item
 
         return list(items.values())
+
+
+def get_item(item_node, test, safety_db):
+    severity = 'Info'  # Because Safety doesn't include severity rating
+    cve = ''.join(a['cve'] or ''
+                  for a in safety_db[item_node['package']]
+                  if a['id'] == 'pyup.io-' + item_node['id']) or None
+    title = item_node['package'] + " (" + item_node['affected'] + ")"
+
+    finding = Finding(title=title + " | " + cve if cve else title,
+                      test=test,
+                      severity=severity,
+                      description=item_node['description'] +
+                                  "\n Vulnerable Package: " + item_node['package'] +
+                                  "\n Installed Version: " + item_node['installed'] +
+                                  "\n Vulnerable Versions: " + item_node['affected'] +
+                                  "\n CVE: " + (cve or "N/A") +
+                                  "\n ID: " + item_node['id'],
+                      cve=cve,
+                      cwe=1035,  # Vulnerable Third Party Component
+                      mitigation="No mitigation provided",
+                      references="No reference provided",
+                      active=False,
+                      verified=False,
+                      false_p=False,
+                      duplicate=False,
+                      out_of_scope=False,
+                      mitigated=None,
+                      impact="No impact provided")
+
+    return finding

dojo/unittests/scans/safety/empty.json

@@ -1 +0,0 @@
-[]

dojo/unittests/scans/safety/many_vulns.json

@@ -31,19 +31,4 @@ def test_multiple_cves(self):
         parser = SafetyParser()
         findings = parser.get_findings(testfile, Test())
         self.assertEqual(1, len(findings))
-        for finding in findings:
-            if "37863" == finding.unique_id_from_tool:
-                self.assertIsNone(finding.cve)
-
-    def test_multiple2(self):
-        testfile = open("dojo/unittests/scans/safety/many_vulns.json")
-        parser = SafetyParser()
-        findings = parser.get_findings(testfile, Test())
-        self.assertEqual(5, len(findings))
-        for finding in findings:
-            if "39608" == finding.unique_id_from_tool:
-                self.assertEqual("httplib2", finding.component_name)
-                self.assertEqual("0.18.1", finding.component_version)
-                self.assertEqual("CVE-2021-21240", finding.cve)
-            elif "39525" == finding.unique_id_from_tool:
-                self.assertIsNone(finding.cve)
+        self.assertEqual("CVE-2019-12385", findings[0].cve)