Fix end_lineno on PyPy 3.8 (#1454)

* Fix end_lineno on PyPy 3.8
* Add CI job for PyPy 3.8

Author: cdce8p

.github/workflows/ci.yaml

@@ -336,7 +336,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        python-version: ["pypy-3.6", "pypy-3.7"]
+        python-version: ["pypy-3.6", "pypy-3.7", "pypy-3.8"]
     outputs:
       python-key: ${{ steps.generate-python-key.outputs.key }}
     steps:
@@ -381,7 +381,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["pypy-3.6", "pypy-3.7"]
+        python-version: ["pypy-3.6", "pypy-3.7", "pypy-3.8"]
     steps:
       - name: Check out code from GitHub
         uses: actions/[email protected]

ChangeLog

@@ -18,6 +18,11 @@ Release date: TBA
 
   Closes #1410
 
+* Set ``end_lineno`` and ``end_col_offset`` attributes to ``None`` for all nodes
+  with PyPy 3.8. PyPy 3.8 assigns these attributes inconsistently which could lead
+  to unexpected errors. Overwriting them with ``None`` will cause a fallback
+  to the already supported way of PyPy 3.7.
+
 
 What's New in astroid 2.10.1?
 =============================

astroid/nodes/scoped_nodes/scoped_nodes.py

@@ -57,7 +57,7 @@
 from astroid import bases
 from astroid import decorators as decorators_mod
 from astroid import mixins, util
-from astroid.const import PY38_PLUS, PY39_PLUS
+from astroid.const import IMPLEMENTATION_PYPY, PY38, PY38_PLUS, PY39_PLUS
 from astroid.context import (
     CallContext,
     InferenceContext,
@@ -2399,7 +2399,7 @@ def _newstyle_impl(self, context=None):
     @cached_property
     def fromlineno(self) -> Optional[int]:
         """The first line that this node appears on in the source code."""
-        if not PY38_PLUS:
+        if not PY38_PLUS or PY38 and IMPLEMENTATION_PYPY:
             # For Python < 3.8 the lineno is the line number of the first decorator.
             # We want the class statement lineno. Similar to 'FunctionDef.fromlineno'
             lineno = self.lineno

astroid/rebuilder.py

@@ -54,7 +54,7 @@
 
 from astroid import nodes
 from astroid._ast import ParserModule, get_parser_module, parse_function_type_comment
-from astroid.const import IMPLEMENTATION_PYPY, PY36, PY38, PY38_PLUS, Context
+from astroid.const import IMPLEMENTATION_PYPY, PY36, PY38, PY38_PLUS, PY39_PLUS, Context
 from astroid.manager import AstroidManager
 from astroid.nodes import NodeNG
 from astroid.nodes.utils import Position
@@ -224,7 +224,7 @@ def _fix_doc_node_position(self, node: NodesWithDocsType) -> None:
 
         lineno = node.lineno or 1  # lineno of modules is 0
         end_range: Optional[int] = node.doc_node.lineno
-        if IMPLEMENTATION_PYPY:
+        if IMPLEMENTATION_PYPY and not PY39_PLUS:
             end_range = None
         # pylint: disable-next=unsubscriptable-object
         data = "\n".join(self._data[lineno - 1 : end_range])
@@ -271,6 +271,26 @@ def _fix_doc_node_position(self, node: NodesWithDocsType) -> None:
         node.doc_node.end_lineno = lineno + t.end[0] - 1
         node.doc_node.end_col_offset = t.end[1]
 
+    def _reset_end_lineno(self, newnode: nodes.NodeNG) -> None:
+        """Reset end_lineno and end_col_offset attributes for PyPy 3.8.
+
+        For some nodes, these are either set to -1 or only partially assigned.
+        To keep consistency across astroid and pylint, reset all.
+
+        This has been fixed in PyPy 3.9.
+        For reference, an (incomplete) list of nodes with issues:
+            - ClassDef          - For
+            - FunctionDef       - While
+            - Call              - If
+            - Decorators        - TryExcept
+            - With              - TryFinally
+            - Assign
+        """
+        newnode.end_lineno = None
+        newnode.end_col_offset = None
+        for child_node in newnode.get_children():
+            self._reset_end_lineno(child_node)
+
     def visit_module(
         self, node: "ast.Module", modname: str, modpath: str, package: bool
     ) -> nodes.Module:
@@ -292,6 +312,8 @@ def visit_module(
             doc_node=self.visit(doc_ast_node, newnode),
         )
         self._fix_doc_node_position(newnode)
+        if IMPLEMENTATION_PYPY and PY38:
+            self._reset_end_lineno(newnode)
         return newnode
 
     if sys.version_info >= (3, 10):

tests/unittest_builder.py

@@ -38,11 +38,12 @@
 import tempfile
 import textwrap
 import unittest
+import unittest.mock
 
 import pytest
 
 from astroid import Instance, builder, nodes, test_utils, util
-from astroid.const import PY38_PLUS
+from astroid.const import IMPLEMENTATION_PYPY, PY38, PY38_PLUS, PY39_PLUS
 from astroid.exceptions import (
     AstroidBuildingError,
     AstroidSyntaxError,
@@ -78,9 +79,12 @@ def test_callfunc_lineno(self) -> None:
         self.assertEqual(name.tolineno, 4)
         strarg = callfunc.args[0]
         self.assertIsInstance(strarg, nodes.Const)
-        if hasattr(sys, "pypy_version_info"):
+        if IMPLEMENTATION_PYPY:
             self.assertEqual(strarg.fromlineno, 4)
-            self.assertEqual(strarg.tolineno, 4)
+            if not PY39_PLUS:
+                self.assertEqual(strarg.tolineno, 4)
+            else:
+                self.assertEqual(strarg.tolineno, 5)
         else:
             if not PY38_PLUS:
                 self.assertEqual(strarg.fromlineno, 5)
@@ -183,8 +187,8 @@ class C:
 
         c = ast_module.body[2]
         assert isinstance(c, nodes.ClassDef)
-        if not PY38_PLUS:
-            # Not perfect, but best we can do for Python 3.7
+        if not PY38_PLUS or PY38 and IMPLEMENTATION_PYPY:
+            # Not perfect, but best we can do for Python 3.7 and PyPy 3.8
             # Can't detect closing bracket on new line.
             assert c.fromlineno == 12
         else:

tests/unittest_nodes_lineno.py

@@ -4,11 +4,12 @@
 
 import astroid
 from astroid import builder, nodes
-from astroid.const import PY38_PLUS, PY39_PLUS, PY310_PLUS
+from astroid.const import IMPLEMENTATION_PYPY, PY38, PY38_PLUS, PY39_PLUS, PY310_PLUS
 
 
 @pytest.mark.skipif(
-    PY38_PLUS, reason="end_lineno and end_col_offset were added in PY38"
+    PY38_PLUS and not (PY38 and IMPLEMENTATION_PYPY),
+    reason="end_lineno and end_col_offset were added in PY38",
 )
 class TestEndLinenoNotSet:
     """Test 'end_lineno' and 'end_col_offset' are initialized as 'None' for Python < 3.8."""
@@ -36,7 +37,8 @@ def test_end_lineno_not_set() -> None:
 
 
 @pytest.mark.skipif(
-    not PY38_PLUS, reason="end_lineno and end_col_offset were added in PY38"
+    not PY38_PLUS or PY38 and IMPLEMENTATION_PYPY,
+    reason="end_lineno and end_col_offset were added in PY38",
 )
 class TestLinenoColOffset:
     """Test 'lineno', 'col_offset', 'end_lineno', and 'end_col_offset' for all nodes."""

tests/unittest_scoped_nodes.py

@@ -58,7 +58,7 @@
     util,
 )
 from astroid.bases import BoundMethod, Generator, Instance, UnboundMethod
-from astroid.const import PY38_PLUS, PY310_PLUS, WIN32
+from astroid.const import IMPLEMENTATION_PYPY, PY38, PY38_PLUS, PY310_PLUS, WIN32
 from astroid.exceptions import (
     AttributeInferenceError,
     DuplicateBasesError,
@@ -1291,7 +1291,7 @@ def g2():
         astroid = builder.parse(data)
         self.assertEqual(astroid["g1"].fromlineno, 4)
         self.assertEqual(astroid["g1"].tolineno, 5)
-        if not PY38_PLUS:
+        if not PY38_PLUS or PY38 and IMPLEMENTATION_PYPY:
             self.assertEqual(astroid["g2"].fromlineno, 9)
         else:
             self.assertEqual(astroid["g2"].fromlineno, 10)