Merge pull request #4001 from DefectDojo/release/1.13.2

madchap · web-flow · commit 1e5412f1afc1 · 2021-03-04T22:59:14.000+01:00
Release: Merge release into master from: release/1.13.2
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -67,7 +67,7 @@ jobs:
 
       # phased startup so we can use the exit code from unit test container
       - name: Start MySQL
-        run: docker-compose up -d
+        run: docker-compose up --no-deps -d mysql
 
       # no celery or initializer needed for unit tests
       - name: Unit tests
diff --git a/components/package.json b/components/package.json
@@ -1,6 +1,6 @@
 {
   "name": "DefectDojo",
-  "version": "1.13.1",
+  "version": "1.13.2",
   "dependencies": {
     "JUMFlot": "jumjum123/JUMFlot#*",
     "bootstrap": "^3.4.0",
diff --git a/dojo/__init__.py b/dojo/__init__.py
@@ -6,6 +6,6 @@
 
 default_app_config = 'dojo.apps.DojoAppConfig'
 
-__version__ = '1.13.1'
+__version__ = '1.13.2'
 __url__ = 'https://github.com/DefectDojo/django-DefectDojo'
 __docs__ = 'http://defectdojo.readthedocs.io/'
diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py
@@ -863,6 +863,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param
     'Trivy Scan': DEDUPE_ALGO_HASH_CODE,
     'HackerOne Cases': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE,
     'Snyk Scan': DEDUPE_ALGO_HASH_CODE,
+    'Safety Scan': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL,
 }
 
 DUPE_DELETE_MAX_PER_RUN = env('DD_DUPE_DELETE_MAX_PER_RUN')
diff --git a/dojo/tools/nessus/parser.py b/dojo/tools/nessus/parser.py
@@ -253,8 +253,8 @@ def get_description_for_scan_types(self, scan_type):
     def get_findings(self, filename, test):
 
         if filename.name.lower().endswith('.xml'):
-            return list(NessusXMLParser().parse(filename, test).values())
+            return NessusXMLParser().get_findings(filename, test)
         elif filename.name.lower().endswith('.csv'):
-            return list(NessusCSVParser().parse(filename, test).values())
+            return NessusCSVParser().get_findings(filename, test)
         else:
             raise ValueError('Unknown File Format')
diff --git a/dojo/tools/openvas_csv/parser.py b/dojo/tools/openvas_csv/parser.py
@@ -144,6 +144,7 @@ def map_column_value(self, finding, column_value):
 
                 finding.unsaved_endpoints = endpoints
 
+        # FIXME manage port and protocole event if it's an IP address
         # URL is an IP so save as an IP endpoint
         elif self.is_valid_ipv4_address(url):
             try:
@@ -317,12 +318,10 @@ def get_findings(self, filename, test):
         dupes = dict()
         chain = self.create_chain()
 
-        if filename is None:
-            return ()
-
-        content = open(filename.temporary_file_path(), 'rb')
-        reportCSV = io.TextIOWrapper(content, encoding='utf-8 ', errors='replace')
-        reader = csv.reader(reportCSV, delimiter=',', quotechar='"')
+        content = filename.read()
+        if type(content) is bytes:
+            content = content.decode('utf-8')
+        reader = csv.reader(io.StringIO(content), delimiter=',', quotechar='"')
 
         row_number = 0
         for row in reader:
@@ -335,7 +334,7 @@ def get_findings(self, filename, test):
 
             column_number = 0
             for column in row:
-                self.chain.process_column(self.column_names[column_number], column, finding)
+                chain.process_column(column_names[column_number], column, finding)
                 column_number += 1
 
             if finding is not None and row_number > 0:
diff --git a/dojo/tools/safety/parser.py b/dojo/tools/safety/parser.py
@@ -18,73 +18,63 @@ def get_label_for_scan_types(self, scan_type):
     def get_description_for_scan_types(self, scan_type):
         return "Safety scan (--json) output file can be imported in JSON format."
 
-    def get_findings(self, json_output, test):
-
-        # Grab Safety DB for CVE lookup
+    def get_safetydb(self):
+        """Grab Safety DB for CVE lookup"""
         url = "https://raw.githubusercontent.com/pyupio/safety-db/master/data/insecure_full.json"
         try:
             response = urllib.request.urlopen(url)
-            safety_db = json.loads(response.read().decode('utf-8'))
+            return json.load(response)
         except urllib.error.URLError as e:
             logger.warn("Error Message: %s", e)
             logger.warn("Could not resolve %s. Fallback is using the offline version from dojo/tools/safety/insecure_full.json.", url)
-            with open("dojo/tools/safety/insecure_full.json", "r") as f:
-                safety_db = json.load(f)
-            f.close()
+            with open("dojo/tools/safety/insecure_full.json", "r") as insecure_full:
+                return json.load(insecure_full)
 
-        tree = self.parse_json(json_output)
-        return self.get_items(tree, test, safety_db)
+    def get_findings(self, json_output, test):
+        safety_db = self.get_safetydb()
 
-    def parse_json(self, json_output):
-        data = json_output.read() or '[]'
-        try:
-            json_obj = json.loads(str(data, 'utf-8'))
-        except:
-            json_obj = json.loads(data)
-        tree = {l[4]: {'package': str(l[0]),
-                       'affected': str(l[1]),
-                       'installed': str(l[2]),
-                       'description': str(l[3]),
-                       'id': str(l[4])}
-                for l in json_obj}  # noqa: E741
-        return tree
+        tree = json.load(json_output)
 
-    def get_items(self, tree, test, safety_db):
         items = {}
-
-        for key, node in tree.items():
-            item = get_item(node, test, safety_db)
-            items[key] = item
+        for node in tree:
+            item_node = {
+                'package': str(node[0]),
+                'affected': str(node[1]),
+                'installed': str(node[2]),
+                'description': str(node[3]),
+                'id': str(node[4])
+            }
+            severity = 'Medium'  # Because Safety doesn't include severity rating
+            cve = None
+            for a in safety_db[item_node['package']]:
+                if a['id'] == 'pyup.io-' + item_node['id']:
+                    if a['cve']:
+                        cve = a['cve']
+            title = item_node['package'] + " (" + item_node['affected'] + ")"
+
+            finding = Finding(title=title + " | " + cve if cve else title,
+                            test=test,
+                            severity=severity,
+                            description="**Description:** " + item_node['description'] +
+                                        "\n**Vulnerable Package:** " + item_node['package'] +
+                                        "\n**Installed Version:** " + item_node['installed'] +
+                                        "\n**Vulnerable Versions:** " + item_node['affected'] +
+                                        "\n**CVE:** " + (cve or "N/A") +
+                                        "\n**ID:** " + item_node['id'],
+                            cve=cve,
+                            cwe=1035,  # Vulnerable Third Party Component
+                            mitigation="No mitigation provided",
+                            references="No reference provided",
+                            active=False,
+                            verified=False,
+                            false_p=False,
+                            duplicate=False,
+                            out_of_scope=False,
+                            mitigated=None,
+                            impact="No impact provided",
+                            component_name=item_node['package'],
+                            component_version=item_node['installed'],
+                            unique_id_from_tool=item_node['id'])
+            items[finding.unique_id_from_tool] = finding
 
         return list(items.values())
-
-
-def get_item(item_node, test, safety_db):
-    severity = 'Info'  # Because Safety doesn't include severity rating
-    cve = ''.join(a['cve'] or ''
-                  for a in safety_db[item_node['package']]
-                  if a['id'] == 'pyup.io-' + item_node['id']) or None
-    title = item_node['package'] + " (" + item_node['affected'] + ")"
-
-    finding = Finding(title=title + " | " + cve if cve else title,
-                      test=test,
-                      severity=severity,
-                      description=item_node['description'] +
-                                  "\n Vulnerable Package: " + item_node['package'] +
-                                  "\n Installed Version: " + item_node['installed'] +
-                                  "\n Vulnerable Versions: " + item_node['affected'] +
-                                  "\n CVE: " + (cve or "N/A") +
-                                  "\n ID: " + item_node['id'],
-                      cve=cve,
-                      cwe=1035,  # Vulnerable Third Party Component
-                      mitigation="No mitigation provided",
-                      references="No reference provided",
-                      active=False,
-                      verified=False,
-                      false_p=False,
-                      duplicate=False,
-                      out_of_scope=False,
-                      mitigated=None,
-                      impact="No impact provided")
-
-    return finding
diff --git a/dojo/unittests/scans/nessus/nessus_v_unknown.xml b/dojo/unittests/scans/nessus/nessus_v_unknown.xml
diff --git a/dojo/unittests/scans/openvas/report-e2759495-f26d-4089-9c56-12a10dc36c9c.csv b/dojo/unittests/scans/openvas/report-e2759495-f26d-4089-9c56-12a10dc36c9c.csv
@@ -0,0 +1,28 @@
+IP,Hostname,Port,Port Protocol,CVSS,Severity,Solution Type,NVT Name,Summary,Specific Result,NVT OID,CVEs,Task ID,Task Name,Timestamp,Result ID,Impact,Solution,Affected Software/OS,Vulnerability Insight,Vulnerability Detection Method,Product Detection Result,BIDs,CERTs,Other References
+10.0.0.8,,22,tcp,4.3,Medium,Mitigation,SSH Weak Encryption Algorithms Supported,The remote SSH server is configured to allow weak encryption algorithms.,"The following weak client-to-server encryption algorithms are supported by the remote service:
+
+aes128-cbc
+aes256-cbc
+
+
+The following weak server-to-client encryption algorithms are supported by the remote service:
+
+aes128-cbc
+aes256-cbc
+
+
+
+",1.3.6.1.4.1.25623.1.0.105611,,c122f831-2481-46d3-97e7-9755e5eeca30,test,2021-02-25T20:01:27Z,ad08dbdb-0d0a-4216-9e09-f5d2f44c1cb9,,Disable the weak encryption algorithms.,,"The `arcfour` cipher is the Arcfour stream cipher with 128-bit keys.
+  The Arcfour cipher is believed to be compatible with the RC4 cipher [SCHNEIER]. Arcfour (and RC4) has problems
+  with weak keys, and should not be used anymore.
+
+  The `none` algorithm specifies that no encryption is to be done.
+  Note that this method provides no confidentiality protection, and it
+  is NOT RECOMMENDED to use it.
+
+  A vulnerability exists in SSH messages that employ CBC mode that may allow an attacker to recover plaintext from a block of ciphertext.","Check if remote ssh service supports Arcfour, none or CBC ciphers.
+Details:
+SSH Weak Encryption Algorithms Supported
+(OID: 1.3.6.1.4.1.25623.1.0.105611)
+Version used: 2020-08-24T08:40:10Z
+",,,,
diff --git a/dojo/unittests/scans/safety/empty.json b/dojo/unittests/scans/safety/empty.json
@@ -0,0 +1 @@
+[]
diff --git a/dojo/unittests/scans/safety/many_vulns.json b/dojo/unittests/scans/safety/many_vulns.json
@@ -0,0 +1,47 @@
+[
+    [
+        "pyyaml",
+        "<5.4",
+        "5.3.1",
+        "A vulnerability was discovered in the PyYAML library in versions before 5.4, where it is susceptible to arbitrary code execution when it processes untrusted YAML files through the full_load method or with the FullLoader loader. Applications that use the library to process untrusted input may be vulnerable to this flaw. This flaw allows an attacker to execute arbitrary code on the system by abusing the python/object/new constructor. This flaw is due to an incomplete fix for CVE-2020-1747. See CVE-2020-14343.",
+        "39611",
+        null,
+        null
+    ],
+    [
+        "jinja2",
+        ">=0.0.0,<2.11.3",
+        "2.11.1",
+        "This affects the package jinja2 from 0.0.0 and before 2.11.3. The ReDOS vulnerability of the regex is mainly due to the sub-pattern [a-zA-Z0-9._-]+.[a-zA-Z0-9._-]+ This issue can be mitigated by Markdown to format user content instead of the urlize filter, or by implementing request timeouts and limiting process memory. See CVE-2020-28493.",
+        "39525",
+        null,
+        null
+    ],
+    [
+        "httplib2",
+        "<0.19.0",
+        "0.18.1",
+        "httplib2 is a comprehensive HTTP client library for Python. In httplib2 before version 0.19.0, a malicious server which responds with long series of \"\\xa0\" characters in the \"www-authenticate\" header may cause Denial of Service (CPU burn while parsing header) of the httplib2 client accessing said server. This is fixed in version 0.19.0 which contains a new implementation of auth headers parsing using the pyparsing library. See CVE-2021-21240.",
+        "39608",
+        null,
+        null
+    ],
+    [
+        "django",
+        "==2.2.17",
+        "2.2.17",
+        "Django 2.2.18 fixes a security issue with severity \"low\" in 2.2.17 (CVE-2021-3281).",
+        "39523",
+        null,
+        null
+    ],
+    [
+        "django",
+        ">=2.2,<2.2.18",
+        "2.2.17",
+        "In Django 2.2 before 2.2.18, 3.0 before 3.0.12, and 3.1 before 3.1.6, the django.utils.archive.extract method (used by \"startapp --template\" and \"startproject --template\") allows directory traversal via an archive with absolute paths or relative paths with dot segments. See CVE-2021-3281.",
+        "39526",
+        null,
+        null
+    ]
+]
diff --git a/dojo/unittests/test_factory.py b/dojo/unittests/test_factory.py
@@ -18,3 +18,10 @@ def test_anchore_one_finding(self):
         findings = parser.get_findings(testfile, Test())
         testfile.close()
         self.assertEqual(1, len(findings))
+
+    def test_nessus(self):
+        testfile = open("dojo/unittests/scans/nessus/nessus_v_unknown.xml")
+        parser = import_parser_factory(testfile, Test(), False, False, 'Nessus Scan')
+        findings = parser.get_findings(testfile, Test())
+        testfile.close()
+        self.assertEqual(32, len(findings))
diff --git a/dojo/unittests/tools/test_nessus_parser.py b/dojo/unittests/tools/test_nessus_parser.py
@@ -1,5 +1,5 @@
 from django.test import TestCase
-from dojo.tools.nessus.parser import NessusXMLParser, NessusCSVParser
+from dojo.tools.nessus.parser import NessusXMLParser, NessusCSVParser, NessusParser
 from dojo.models import Finding, Test, Engagement, Product
 
 
@@ -92,3 +92,20 @@ def test_parse_some_findings_csv_bytes(self):
         testfile = open("dojo/unittests/scans/nessus/nessus_many_vuln2-all.csv", "rb")
         parser = NessusCSVParser()
         findings = parser.get_findings(testfile, self.create_test())
+
+    def test_parse_some_findings_samples(self):
+        """Test that come from samples repo"""
+        testfile = open("dojo/unittests/scans/nessus/nessus_v_unknown.xml")
+        parser = NessusParser()
+        findings = parser.get_findings(testfile, self.create_test())
+        self.assertEqual(32, len(findings))
+        finding = findings[0]
+        self.assertIn(finding.severity, Finding.SEVERITIES)
+        self.assertEqual('Info', finding.severity)
+        self.assertIsNone(finding.cve)
+        self.assertEqual('Nessus Scan Information', finding.title)
+        finding = findings[25]
+        self.assertIn(finding.severity, Finding.SEVERITIES)
+        self.assertEqual('Nessus SYN scanner', finding.title)
+        self.assertEqual('Info', finding.severity)
+        self.assertIsNone(finding.cve)
diff --git a/dojo/unittests/tools/test_openvas_csv_parser.py b/dojo/unittests/tools/test_openvas_csv_parser.py
@@ -1,12 +1,26 @@
 from django.test import TestCase
-# from dojo.tools.openvas_csv.parser import OpenVASUploadCsvParser
-# from dojo.models import Test
+from dojo.tools.openvas_csv.parser import OpenVASCsvParser
+from dojo.models import Test, Engagement, Product
 
 
 class TestOpenVASUploadCsvParser(TestCase):
-    pass
-    # FIXME add unit tests for OpenVAS
-    # def test_openvas_csv_parser_without_file_has_no_findings(self):
-    #     parser = OpenVASUploadCsvParser()
-    #     findings = parser.get_findings(None, Test())
-    #     self.assertEqual(0, len(findings))
+
+    def test_openvas_csv_parser_without_file_has_no_findings(self):
+        with open("dojo/unittests/scans/openvas/report-e2759495-f26d-4089-9c56-12a10dc36c9c.csv") as f:
+            test = Test()
+            test.engagement = Engagement()
+            test.engagement.product = Product()
+            parser = OpenVASCsvParser()
+            findings = parser.get_findings(f, test)
+            self.assertEqual(1, len(findings))
+            # finding
+            self.assertEqual("SSH Weak Encryption Algorithms Supported", findings[0].title)
+            self.assertEqual("Medium", findings[0].severity)
+            # endpoints
+            self.assertEqual(1, len(findings[0].unsaved_endpoints))
+            # endpoint
+            self.assertEqual("10.0.0.8", findings[0].unsaved_endpoints[0].host)
+            # FIXME there is a 'Port protocole' column
+            # self.assertEqual("tcp", findings[0].unsaved_endpoints[0].protocol)
+            # FIXME there is a 'Port' column
+            # self.assertEqual(22, findings[0].unsaved_endpoints[0].port)
diff --git a/dojo/unittests/tools/test_safety_parser.py b/dojo/unittests/tools/test_safety_parser.py
@@ -31,4 +31,19 @@ def test_multiple_cves(self):
         parser = SafetyParser()
         findings = parser.get_findings(testfile, Test())
         self.assertEqual(1, len(findings))
-        self.assertEqual("CVE-2019-12385", findings[0].cve)
+        for finding in findings:
+            if "37863" == finding.unique_id_from_tool:
+                self.assertIsNone(finding.cve)
+
+    def test_multiple2(self):
+        testfile = open("dojo/unittests/scans/safety/many_vulns.json")
+        parser = SafetyParser()
+        findings = parser.get_findings(testfile, Test())
+        self.assertEqual(5, len(findings))
+        for finding in findings:
+            if "39608" == finding.unique_id_from_tool:
+                self.assertEqual("httplib2", finding.component_name)
+                self.assertEqual("0.18.1", finding.component_version)
+                self.assertEqual("CVE-2021-21240", finding.cve)
+            elif "39525" == finding.unique_id_from_tool:
+                self.assertIsNone(finding.cve)
diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
-appVersion: "1.13.1"
+appVersion: "1.13.2"
 description: A Helm chart for Kubernetes to install DefectDojo
 name: defectdojo
 version: 1.5.0
diff --git a/setup.py b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name='DefectDojo',
-    version='1.13.1',
+    version='1.13.2',
     author='Greg Anderson',
     description="Tool for managing vulnerability engagements",
     install_requires=[

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "DefectDojo",`
`3`		`- "version": "1.13.1",`
	`3`	`+ "version": "1.13.2",`
`4`	`4`	`"dependencies": {`
`5`	`5`	`"JUMFlot": "jumjum123/JUMFlot#*",`
`6`	`6`	`"bootstrap": "^3.4.0",`
Original file line number	Diff line number	Diff line change
`@@ -863,6 +863,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param`
`863`	`863`	`'Trivy Scan': DEDUPE_ALGO_HASH_CODE,`
`864`	`864`	`'HackerOne Cases': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL_OR_HASH_CODE,`
`865`	`865`	`'Snyk Scan': DEDUPE_ALGO_HASH_CODE,`
	`866`	`+ 'Safety Scan': DEDUPE_ALGO_UNIQUE_ID_FROM_TOOL,`
`866`	`867`	`}`
`867`	`868`
`868`	`869`	`DUPE_DELETE_MAX_PER_RUN = env('DD_DUPE_DELETE_MAX_PER_RUN')`