Add tests for CVSS and Severity options (#740) (#919)

fixes #740

Author: anthonyharrison

test/test_cli.py

@@ -300,3 +300,138 @@ def test_config_file(self, caplog, filename):
                 pytest.fail(
                     msg="cli option should override logging level specified in config file"
                 )
+
+    @staticmethod
+    def check_string_in_file(filename, string_to_find):
+        # Check if 'string_to_find' is in file
+        fh = open(filename)
+        file_contents = fh.readlines()
+        fh.close()
+        for line in file_contents:
+            if string_to_find in line:
+                return True
+        return False
+
+    def test_severity(self, capsys, caplog):
+        # scan with severity setting to ensure only CVEs above severity threshold are reported
+
+        # Check command line parameters - wrong case
+        with pytest.raises(SystemExit) as e:
+            main(["cve-bin-tool", "-S", "HIGH", self.tempdir])
+        assert e.value.args[0] == -2
+        # Check command line parameters - wrong option
+        with pytest.raises(SystemExit) as e:
+            main(["cve-bin-tool", "-S", "ALL", self.tempdir])
+        assert e.value.args[0] == -2
+
+        my_test_filename = "sevtest.csv"
+
+        if os.path.exists(my_test_filename):
+            os.remove(my_test_filename)
+        with caplog.at_level(logging.DEBUG):
+            main(
+                [
+                    "cve-bin-tool",
+                    "-x",
+                    "-f",
+                    "csv",
+                    "-o",
+                    my_test_filename,
+                    "-S",
+                    "high",
+                    os.path.join(self.tempdir, CURL_7_20_0_RPM),
+                ]
+            )
+        # Verify that no CVEs with a severity of Medium are reported
+        assert not self.check_string_in_file(my_test_filename, "MEDIUM")
+        # Verify that CVEs with a higher severity are reported
+        assert self.check_string_in_file(my_test_filename, "HIGH")
+        caplog.clear()
+        if os.path.exists(my_test_filename):
+            os.remove(my_test_filename)
+
+    def test_CVSS_score(self, capsys, caplog):
+        # scan with severity score to ensure only CVEs above score threshold are reported
+
+        my_test_filename = "sevtest.csv"
+
+        # Check command line parameters. Less than 0 result in default behaviour.
+        if os.path.exists(my_test_filename):
+            os.remove(my_test_filename)
+        with caplog.at_level(logging.DEBUG):
+            main(
+                [
+                    "cve-bin-tool",
+                    "-x",
+                    "-c",
+                    "-1",
+                    "-f",
+                    "csv",
+                    "-o",
+                    my_test_filename,
+                    os.path.join(self.tempdir, CURL_7_20_0_RPM),
+                ]
+            )
+        # Verify that some CVEs with a severity of Medium are reported
+        assert self.check_string_in_file(my_test_filename, "MEDIUM")
+        caplog.clear()
+
+        # Check command line parameters. >10 results in no CVEs being reported (Maximum CVSS score is 10)
+        if os.path.exists(my_test_filename):
+            os.remove(my_test_filename)
+        with caplog.at_level(logging.DEBUG):
+            main(
+                [
+                    "cve-bin-tool",
+                    "-x",
+                    "-c",
+                    "11",
+                    "-f",
+                    "csv",
+                    "-o",
+                    my_test_filename,
+                    os.path.join(self.tempdir, CURL_7_20_0_RPM),
+                ]
+            )
+        # Verify that no CVEs are reported (no file is created)
+        assert not os.path.exists(my_test_filename)
+        caplog.clear()
+
+        with caplog.at_level(logging.DEBUG):
+            main(
+                [
+                    "cve-bin-tool",
+                    "-x",
+                    "-f",
+                    "csv",
+                    "-o",
+                    my_test_filename,
+                    os.path.join(self.tempdir, CURL_7_20_0_RPM),
+                ]
+            )
+        # Verify that CVEs with a severity of Medium are reported
+        assert self.check_string_in_file(my_test_filename, "MEDIUM")
+        caplog.clear()
+        if os.path.exists(my_test_filename):
+            os.remove(my_test_filename)
+
+        # Now check subset
+        with caplog.at_level(logging.DEBUG):
+            main(
+                [
+                    "cve-bin-tool",
+                    "-x",
+                    "-c",
+                    "7",
+                    "-f",
+                    "csv",
+                    "-o",
+                    my_test_filename,
+                    os.path.join(self.tempdir, CURL_7_20_0_RPM),
+                ]
+            )
+        # Verify that no CVEs with a severity of Medium are reported
+        assert not self.check_string_in_file(my_test_filename, "MEDIUM")
+        if os.path.exists(my_test_filename):
+            os.remove(my_test_filename)
+        caplog.clear()