Produce a score of 0 for fatal errors and add fatal to score evaluation (#5521)

Co-authored-by: Daniël van Noord <[email protected]>

Author: jacobtylerwalls

ChangeLog

@@ -54,6 +54,13 @@ Release date: TBA
 * The ``PyLinter`` class will now be initialiazed with a ``TextReporter``
   as its reporter if none is provided.
 
+* Fatal errors now emit a score of 0.0 regardless of whether the linted module
+  contained any statements
+
+  Closes #5451
+
+* ``fatal`` was added to the variables permitted in score evaluation expressions.
+
 * Fix false positive - Allow unpacking of ``self`` in a subclass of ``typing.NamedTuple``.
 
   Closes #5312

doc/faq.rst

@@ -252,7 +252,7 @@ default value by changing the mixin-class-rgx option.
 Even though the final rating Pylint renders is nominally out of ten, there's no
 lower bound on it. By default, the formula to calculate score is ::
 
-    10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+    0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
 However, this option can be changed in the Pylint rc file. If having negative
 values really bugs you, you can set the formula to be the maximum of 0 and the

doc/whatsnew/2.13.rst

@@ -68,7 +68,14 @@ Other Changes
 
   Closes #5065
 
-* The ``PyLinter`` class will now be initialiazed with a ``TextReporter``
+* Fatal errors now emit a score of 0.0 regardless of whether the linted module
+  contained any statements
+
+  Closes #5451
+
+* ``fatal`` was added to the variables permitted in score evaluation expressions.
+
+* The ``PyLinter`` class will now be initialized with a ``TextReporter``
   as its reporter if none is provided.
 
 * The ``testutils`` for unittests now accept ``end_lineno`` and ``end_column``. Tests

examples/pylintrc

@@ -90,11 +90,11 @@ enable=c-extension-no-member
 [REPORTS]
 
 # Python expression which should return a score less than or equal to 10. You
-# have access to the variables 'error', 'warning', 'refactor', and 'convention'
+# have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention' and 'info'
 # which contain the number of messages in each category, as well as 'statement'
 # which is the total number of statements analyzed. This score is used by the
 # global evaluation report (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+evaluation=0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
 # Template used to display messages. This is a python new-style format string
 # used to format the message information. See doc for all details.

pylint/lint/pylinter.py

@@ -764,7 +764,7 @@ def enable_fail_on_messages(self):
                         self.enable(msg.msgid)
                         self.fail_on_symbols.append(msg.symbol)
                     elif msg.msgid[0] in fail_on_cats:
-                        # message starts with a cateogry value, flag (but do not enable) it
+                        # message starts with a category value, flag (but do not enable) it
                         self.fail_on_symbols.append(msg.symbol)
 
     def any_fail_on_issues(self):
@@ -1315,6 +1315,7 @@ def _report_evaluation(self):
         evaluation = self.config.evaluation
         try:
             stats_dict = {
+                "fatal": self.stats.fatal,
                 "error": self.stats.error,
                 "warning": self.stats.warning,
                 "refactor": self.stats.refactor,

pylintrc

@@ -98,11 +98,11 @@ files-output=no
 reports=no
 
 # Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention'
+# and 'info', which contain the number of messages in each category, as
+# well as 'statement', which is the total number of statements analyzed. This
+# score is used by the global evaluation report (RP0004).
+evaluation=0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
 # Template used to display messages. This is a python new-style format string
 # used to format the message information. See doc for all details

tests/test_self.py

@@ -1154,6 +1154,14 @@ def test_fail_on_exit_code(self, args, expected):
         # and errors that are generated they don't affect the exit code.
         self._runtest([path, "--fail-under=-10"] + args, code=expected)
 
+    def test_one_module_fatal_error(self):
+        """
+        Fatal errors in one of several modules linted still exits non-zero.
+        """
+        valid_path = join(HERE, "conftest.py")
+        invalid_path = join(HERE, "garbagePath.py")
+        self._runtest([valid_path, invalid_path], code=1)
+
     @pytest.mark.parametrize(
         "args, expected",
         [