Merge remote-tracking branch 'origin/main' into autoformat/spacevisitor

Author: nielsdebruin

.github/workflows/receive-pr.yml

@@ -1,6 +1,9 @@
 name: receive-pr
 
 on:
+  pull_request:
+    types: []
+
 #  pull_request:
 #    types: [opened, synchronize]
 #    branches:

rewrite/rewrite/__init__.py

@@ -7,7 +7,7 @@
 from .execution import ExecutionContext, DelegatingExecutionContext, InMemoryExecutionContext, Recipe, RecipeRunException
 from .markers import *
 from .tree import Checksum, FileAttributes, SourceFile, Tree, PrintOutputCapture, PrinterFactory
-from .utils import random_id
+from .utils import random_id, list_map
 from .visitor import Cursor, TreeVisitor
 from .parser import *
 from .result import *
@@ -21,6 +21,7 @@
     'PrintOutputCapture',
     'PrinterFactory',
     'random_id',
+    'list_map',
     'Cursor',
     'TreeVisitor',
     'ExecutionContext',

rewrite/rewrite/java/extensions.py

@@ -2,6 +2,7 @@
 from typing import Optional, TypeVar, TYPE_CHECKING
 
 from .support_types import J, JRightPadded, JLeftPadded, JContainer, Space
+from .. import list_map
 from ..visitor import Cursor
 from ..tree import Tree
 
@@ -19,7 +20,7 @@ def visit_container(v: 'JavaVisitor', container: Optional[JContainer[J2]], loc:
 
     v.cursor = Cursor(v.cursor, container)
     before = v.visit_space(container.before, loc.before_location, p)
-    js = [v.visit_right_padded(el, loc.element_location, p) for el in container.padding.elements]
+    js = list_map(lambda el: v.visit_right_padded(el, loc.element_location, p), container.padding.elements)
     v.cursor = v.cursor.parent
 
     return container if js is container.padding.elements and before is container.before else JContainer(before, js, container.markers)

rewrite/rewrite/java/visitor.py

@@ -68,5 +68,6 @@
     'AutoFormat',
     'BlankLinesVisitor',
     'NormalizeFormatVisitor',
+    'NormalizeTabsOrSpacesVisitor',
     'SpacesVisitor',
 ]

rewrite/rewrite/python/__init__.py

@@ -8,5 +8,6 @@
     'AutoFormat',
     'BlankLinesVisitor',
     'NormalizeFormatVisitor',
+    'NormalizeTabsOrSpacesVisitor',
     'SpacesVisitor',
 ]

rewrite/rewrite/python/format/__init__.py

@@ -3,6 +3,8 @@
 from .blank_lines import BlankLinesVisitor
 from .normalize_format import NormalizeFormatVisitor
 from .spaces_visitor import SpacesVisitor
+from .normalize_tabs_or_spaces import NormalizeTabsOrSpacesVisitor
+from .. import TabsAndIndentsStyle
 from ..style import BlankLinesStyle, SpacesStyle, IntelliJ
 from ..visitor import PythonVisitor
 from ... import Recipe, Tree, Cursor
@@ -24,9 +26,10 @@ def visit(self, tree: Optional[Tree], p: P, parent: Optional[Cursor] = None) ->
         cu = tree if isinstance(tree, JavaSourceFile) else self._cursor.first_enclosing_or_throw(JavaSourceFile)
 
         tree = NormalizeFormatVisitor(self._stop_after).visit(tree, p, self._cursor.fork())
-        tree = BlankLinesVisitor(cu.get_style(BlankLinesStyle) or IntelliJ.blank_lines(), self._stop_after).visit(tree,
-                                                                                                                  p,
-                                                                                                                  self._cursor.fork())
-        tree = SpacesVisitor(cu.get_style(SpacesStyle) or IntelliJ.spaces(), self._stop_after).visit(tree, p,
-                                                                                                     self._cursor.fork())
+        tree = BlankLinesVisitor(cu.get_style(BlankLinesStyle) or IntelliJ.blank_lines(), self._stop_after).visit(tree, p, self._cursor.fork())
+        tree = SpacesVisitor(cu.get_style(SpacesStyle) or IntelliJ.spaces(), self._stop_after).visit(tree, p, self._cursor.fork())
+        tree = NormalizeTabsOrSpacesVisitor(
+            cu.get_style(TabsAndIndentsStyle) or IntelliJ.tabs_and_indents(),
+            self._stop_after
+        ).visit(tree, p, self._cursor.fork())
         return tree

rewrite/rewrite/python/format/auto_format.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
-from typing import cast, Optional, TypeVar
+from typing import cast, Optional, TypeVar, List
 
-from rewrite import Tree, P, Cursor
+from rewrite import Tree, P, Cursor, list_map
 from rewrite.java import MethodDeclaration, J, Space, ClassDeclaration
 from rewrite.python import PythonVisitor, PyComment
 from rewrite.visitor import T
@@ -65,19 +65,19 @@ def _common_margin(s1, s2):
 def _concatenate_prefix(j: J, prefix: Space) -> J2:
     shift = _common_margin(None, j.prefix.whitespace)
 
-    def modify_comment(c: PyComment):
+    def modify_comment(c: PyComment) -> PyComment:
         if len(shift) == 0:
             return c
         c = c.with_text(c.text.replace('\n', '\n' + shift))
         if '\n' in c.suffix:
             c = c.with_suffix(c.suffix.replace('\n', '\n' + shift))
         return c
 
-    comments = j.prefix.comments + \
-               [modify_comment(cast(PyComment, comment)) for comment in prefix.comments]
+    comments = j.prefix.comments + list_map(modify_comment, cast(List[PyComment], prefix.comments))
 
     new_prefix = j.prefix
     new_prefix = new_prefix.with_whitespace(new_prefix.whitespace + prefix.whitespace)
-    new_prefix = new_prefix.with_comments(comments)
+    if comments:
+        new_prefix = new_prefix.with_comments(comments)
 
     return j.with_prefix(new_prefix)

rewrite/rewrite/python/format/normalize_format.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from typing import cast, Optional, TypeVar, List
+
+from rewrite import Tree, P, Cursor, list_map
+from rewrite.java import J, Space
+from rewrite.python import PythonVisitor, PyComment, TabsAndIndentsStyle
+from rewrite.visitor import T
+
+J2 = TypeVar('J2', bound=J)
+
+
+# TODO consider supporting multiline string literals
+class NormalizeTabsOrSpacesVisitor(PythonVisitor):
+
+    def __init__(self, style: TabsAndIndentsStyle, stop_after: Tree = None):
+        self._stop_after = stop_after
+        self._style = style
+        self._stop = False
+
+    def visit_space(self, space: Space, loc, p):
+        if not space or space is Space.EMPTY:
+            return space
+
+        s = space.with_whitespace(self.normalize_after_first_newline(space.whitespace))
+        return s.with_comments(list_map(self.process_comment, cast(List[PyComment], s.comments)))
+
+    def process_comment(self, comment: PyComment) -> PyComment:
+        return comment.with_suffix(self.normalize_after_first_newline(comment.suffix))
+
+    def normalize_after_first_newline(self, text: str) -> str:
+        first_newline = text.find('\n')
+        if first_newline >= 0 and first_newline != len(text) - 1:
+            return text[:first_newline + 1] + self.normalize(text[first_newline + 1:], False)
+        return text
+
+    def normalize(self, text: str, is_comment: bool) -> str:
+        if not text:
+            return text
+
+        if ' ' not in text if self._style.use_tab_character else '\t' not in text:
+            return text
+
+        text_builder = []
+        consecutive_spaces = 0
+        in_margin = True
+        i = 0
+
+        while i < len(text):
+            c = text[i]
+
+            if c in '\n\r':
+                in_margin = True
+                consecutive_spaces = 0
+
+            elif in_margin:
+                if self._style.use_tab_character and c == ' ' and (
+                        not is_comment or (i + 1 < len(text) and text[i + 1] != '*')):
+                    end_idx = i + self._style.tab_size
+                    if text[i:end_idx] == ' ' * self._style.tab_size:
+                        text_builder.append('\t')
+                        i += self._style.tab_size
+                        continue
+
+                elif not self._style.use_tab_character and c == '\t':
+                    text_builder.append(' ' * (self._style.tab_size - (consecutive_spaces % self._style.tab_size)))
+                    consecutive_spaces = 0
+                    i += 1
+                    continue
+
+            text_builder.append(c)
+            in_margin = in_margin and c.isspace()
+            consecutive_spaces = consecutive_spaces + 1 if c.isspace() else 0
+            i += 1
+
+        return ''.join(text_builder)
+
+    def post_visit(self, tree: T, p: P) -> Optional[T]:
+        if self._stop_after and tree == self._stop_after:
+            self._stop = True
+        return tree
+
+    def visit(self, tree: Optional[Tree], p: P, parent: Optional[Cursor] = None) -> Optional[T]:
+        return tree if self._stop else super().visit(tree, p, parent)

rewrite/rewrite/python/format/normalize_tabs_or_spaces.py

@@ -1,7 +1,7 @@
 from typing import cast, TypeVar, Union
 
-from rewrite import SourceFile, TreeVisitor
-from .extensions import *
+from rewrite import SourceFile, TreeVisitor, list_map
+from . import extensions
 from .support_types import *
 from .tree import *
 from rewrite.java import JavaVisitor
@@ -51,7 +51,7 @@ def visit_chained_assignment(self, chained_assignment: ChainedAssignment, p: P)
             return temp_statement
         chained_assignment = cast(ChainedAssignment, temp_statement)
         chained_assignment = chained_assignment.with_markers(self.visit_markers(chained_assignment.markers, p))
-        chained_assignment = chained_assignment.padding.with_variables([self.visit_right_padded(v, PyRightPadded.Location.CHAINED_ASSIGNMENT_VARIABLES, p) for v in chained_assignment.padding.variables])
+        chained_assignment = chained_assignment.padding.with_variables(list_map(lambda v: self.visit_right_padded(v, PyRightPadded.Location.CHAINED_ASSIGNMENT_VARIABLES, p), chained_assignment.padding.variables))
         chained_assignment = chained_assignment.with_assignment(self.visit_and_cast(chained_assignment.assignment, Expression, p))
         return chained_assignment
 
@@ -92,8 +92,8 @@ def visit_type_hint(self, type_hint: TypeHint, p: P) -> J:
     def visit_compilation_unit(self, compilation_unit: CompilationUnit, p: P) -> J:
         compilation_unit = compilation_unit.with_prefix(self.visit_space(compilation_unit.prefix, Space.Location.COMPILATION_UNIT_PREFIX, p))
         compilation_unit = compilation_unit.with_markers(self.visit_markers(compilation_unit.markers, p))
-        compilation_unit = compilation_unit.padding.with_imports([self.visit_right_padded(v, JRightPadded.Location.IMPORT, p) for v in compilation_unit.padding.imports])
-        compilation_unit = compilation_unit.padding.with_statements([self.visit_right_padded(v, PyRightPadded.Location.COMPILATION_UNIT_STATEMENTS, p) for v in compilation_unit.padding.statements])
+        compilation_unit = compilation_unit.padding.with_imports(list_map(lambda v: self.visit_right_padded(v, JRightPadded.Location.IMPORT, p), compilation_unit.padding.imports))
+        compilation_unit = compilation_unit.padding.with_statements(list_map(lambda v: self.visit_right_padded(v, PyRightPadded.Location.COMPILATION_UNIT_STATEMENTS, p), compilation_unit.padding.statements))
         compilation_unit = compilation_unit.with_eof(self.visit_space(compilation_unit.eof, Space.Location.COMPILATION_UNIT_EOF, p))
         return compilation_unit
 
@@ -180,7 +180,7 @@ def visit_formatted_string(self, formatted_string: FormattedString, p: P) -> J:
             return temp_expression
         formatted_string = cast(FormattedString, temp_expression)
         formatted_string = formatted_string.with_markers(self.visit_markers(formatted_string.markers, p))
-        formatted_string = formatted_string.with_parts([self.visit_and_cast(v, Expression, p) for v in formatted_string.parts])
+        formatted_string = formatted_string.with_parts(list_map(lambda v: self.visit_and_cast(v, Expression, p), formatted_string.parts))
         return formatted_string
 
     def visit_formatted_string_value(self, value: FormattedString.Value, p: P) -> J:
@@ -223,7 +223,7 @@ def visit_comprehension_expression(self, comprehension_expression: Comprehension
         comprehension_expression = cast(ComprehensionExpression, temp_expression)
         comprehension_expression = comprehension_expression.with_markers(self.visit_markers(comprehension_expression.markers, p))
         comprehension_expression = comprehension_expression.with_result(self.visit_and_cast(comprehension_expression.result, Expression, p))
-        comprehension_expression = comprehension_expression.with_clauses([self.visit_and_cast(v, ComprehensionExpression.Clause, p) for v in comprehension_expression.clauses])
+        comprehension_expression = comprehension_expression.with_clauses(list_map(lambda v: self.visit_and_cast(v, ComprehensionExpression.Clause, p), comprehension_expression.clauses))
         comprehension_expression = comprehension_expression.with_suffix(self.visit_space(comprehension_expression.suffix, PySpace.Location.COMPREHENSION_EXPRESSION_SUFFIX, p))
         return comprehension_expression
 
@@ -239,7 +239,7 @@ def visit_comprehension_clause(self, clause: ComprehensionExpression.Clause, p:
         clause = clause.padding.with_async(self.visit_right_padded(clause.padding.async_, PyRightPadded.Location.COMPREHENSION_EXPRESSION_CLAUSE_ASYNC, p))
         clause = clause.with_iterator_variable(self.visit_and_cast(clause.iterator_variable, Expression, p))
         clause = clause.padding.with_iterated_list(self.visit_left_padded(clause.padding.iterated_list, PyLeftPadded.Location.COMPREHENSION_EXPRESSION_CLAUSE_ITERATED_LIST, p))
-        clause = clause.with_conditions([self.visit_and_cast(v, ComprehensionExpression.Condition, p) for v in clause.conditions])
+        clause = clause.with_conditions(list_map(lambda v: self.visit_and_cast(v, ComprehensionExpression.Condition, p), clause.conditions))
         return clause
 
     def visit_type_alias(self, type_alias: TypeAlias, p: P) -> J:
@@ -270,7 +270,7 @@ def visit_union_type(self, union_type: UnionType, p: P) -> J:
             return temp_expression
         union_type = cast(UnionType, temp_expression)
         union_type = union_type.with_markers(self.visit_markers(union_type.markers, p))
-        union_type = union_type.padding.with_types([self.visit_right_padded(v, PyRightPadded.Location.UNION_TYPE_TYPES, p) for v in union_type.padding.types])
+        union_type = union_type.padding.with_types(list_map(lambda v: self.visit_right_padded(v, PyRightPadded.Location.UNION_TYPE_TYPES, p), union_type.padding.types))
         return union_type
 
     def visit_variable_scope(self, variable_scope: VariableScope, p: P) -> J:
@@ -280,7 +280,7 @@ def visit_variable_scope(self, variable_scope: VariableScope, p: P) -> J:
             return temp_statement
         variable_scope = cast(VariableScope, temp_statement)
         variable_scope = variable_scope.with_markers(self.visit_markers(variable_scope.markers, p))
-        variable_scope = variable_scope.padding.with_names([self.visit_right_padded(v, PyRightPadded.Location.VARIABLE_SCOPE_NAMES, p) for v in variable_scope.padding.names])
+        variable_scope = variable_scope.padding.with_names(list_map(lambda v: self.visit_right_padded(v, PyRightPadded.Location.VARIABLE_SCOPE_NAMES, p), variable_scope.padding.names))
         return variable_scope
 
     def visit_del(self, del_: Del, p: P) -> J:
@@ -290,7 +290,7 @@ def visit_del(self, del_: Del, p: P) -> J:
             return temp_statement
         del_ = cast(Del, temp_statement)
         del_ = del_.with_markers(self.visit_markers(del_.markers, p))
-        del_ = del_.padding.with_targets([self.visit_right_padded(v, PyRightPadded.Location.DEL_TARGETS, p) for v in del_.padding.targets])
+        del_ = del_.padding.with_targets(list_map(lambda v: self.visit_right_padded(v, PyRightPadded.Location.DEL_TARGETS, p), del_.padding.targets))
         return del_
 
     def visit_special_parameter(self, special_parameter: SpecialParameter, p: P) -> J: