Format generated Python parser module (#347)

* debt: Format generated Python parser module

- Removes configuring exclusion of any Python modules from formatting
- Ensure entire library is formatted correctly
- Drop redundant `or False` logic from conditionals
- Drop redundant `else` conditionals after `return` within `if` clause
- Skipped formatting on `state_comment` string
  - Challenging to generate formatted Python multiline string from C#
- Migrate Python formatter from `black` to `ruff`
- Faster tooling
- Better handling with linting rule conflicts
- Enables extending to hundreds of linting rules
- Aligns with pytest-bdd (pytest-dev/pytest-bdd#758)

Co-authored-by: M.P. Korstanje <[email protected]>

Author: kieran-ryan

.pre-commit-config.yaml

@@ -1,11 +1,10 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
-  # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: c53240a7f974b3707e13eac6710542cc96a2d61a  # frozen: 24.10.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 89c421dff2e1026ba12cdb9ebd731f4a83aa8021  # frozen: v0.8.6
     hooks:
-      - id: black
+      - id: ruff-format
         files: "^python/"
   - repo: https://github.com/asottile/pyupgrade
     rev: 19364aa1b2ac289ce75e6bbe93994f7b4b9425f6  # frozen: v3.19.0

python/gherkin-python.razor

@@ -4,21 +4,31 @@
   switch(production.Type)
   {
     case ProductionRuleType.Start:
-      @:          self.start_rule(context, '@production.RuleName')
+      @:  self.start_rule(context, "@production.RuleName")
       break;
     case ProductionRuleType.End:
-      @:          self.end_rule(context, '@production.RuleName')
+      @:  self.end_rule(context, "@production.RuleName")
       break;
     case ProductionRuleType.Process:
-      @:          self.build(context, token)
+      @:  self.build(context, token)
       break;
   }
 }
 @helper HandleParserError(IEnumerable<string> expectedTokens, State state)
-{<text>        state_comment = "State: @state.Id - @Raw(state.Comment)"
-        token.detach
-        expected_tokens = ["@Raw(string.Join("\", \"", expectedTokens))"]
-        error = UnexpectedEOFException(token, expected_tokens, state_comment) if token.eof() else UnexpectedTokenException(token, expected_tokens, state_comment)
+{<text>        state_comment = "State: @state.Id - @Raw(state.Comment)"  # fmt: skip
+        token.detach()
+        expected_tokens = [
+    </text>
+@foreach (var token in expectedTokens)
+        {<text>        "@token",
+    </text>
+    }
+<text>    ]
+        error = (
+            UnexpectedEOFException(token, expected_tokens, state_comment)
+            if token.eof()
+            else UnexpectedTokenException(token, expected_tokens, state_comment)
+        )
         if self.stop_at_first_error:
             raise error
         self.add_error(context, error)
@@ -30,23 +40,28 @@ from __future__ import annotations
 
 from collections import deque
 from collections.abc import Callable
-from typing import cast, TypeVar
+from typing import TypeVar, cast
 
 from .ast_builder import AstBuilder
+from .errors import (
+    CompositeParserException,
+    ParserException,
+    UnexpectedEOFException,
+    UnexpectedTokenException,
+)
+from .parser_types import GherkinDocument
 from .token import Token
 from .token_matcher import TokenMatcher
 from .token_scanner import TokenScanner
-from .parser_types import GherkinDocument
-from .errors import UnexpectedEOFException, UnexpectedTokenException, ParserException, CompositeParserException
 
-_T = TypeVar('_T')
-_U = TypeVar('_U')
-_V = TypeVar('_V')
+_T = TypeVar("_T")
+_U = TypeVar("_U")
+_V = TypeVar("_V")
 
 RULE_TYPE = [
-    'None',
+    "None",
     @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule))
-    {<text>    '@rule.Name.Replace("#", "_")',  # @rule.ToString(true)
+    {<text>    "@rule.Name.Replace('#', '_')",  # @rule.ToString(true)
 </text>}
 ]
 
@@ -75,18 +90,18 @@ class @(Model.ParserClassName):
         token_scanner_or_str: TokenScanner | str,
         token_matcher: TokenMatcher | None = None,
     ) -> GherkinDocument:
-        token_scanner = TokenScanner(token_scanner_or_str) if isinstance(token_scanner_or_str, str) else token_scanner_or_str
+        token_scanner = (
+            TokenScanner(token_scanner_or_str)
+            if isinstance(token_scanner_or_str, str)
+            else token_scanner_or_str
+        )
         self.ast_builder.reset()
         if token_matcher is None:
             token_matcher = TokenMatcher()
         token_matcher.reset()
-        context = ParserContext(
-            token_scanner,
-            token_matcher,
-            deque(),
-            [])
+        context = ParserContext(token_scanner, token_matcher, deque(), [])
 
-        self.start_rule(context, '@Model.RuleSet.StartRule.Name')
+        self.start_rule(context, "@Model.RuleSet.StartRule.Name")
         state = 0
         token = None
         while True:
@@ -95,7 +110,7 @@ class @(Model.ParserClassName):
             if token.eof():
                 break
 
-        self.end_rule(context, '@Model.RuleSet.StartRule.Name')
+        self.end_rule(context, "@Model.RuleSet.StartRule.Name")
 
         if context.errors:
             raise CompositeParserException(context.errors)
@@ -123,8 +138,7 @@ class @(Model.ParserClassName):
     def read_token(self, context: ParserContext) -> Token:
         if context.token_queue:
             return context.token_queue.popleft()
-        else:
-            return context.token_scanner.read()
+        return context.token_scanner.read()
 @foreach(var rule in Model.RuleSet.TokenRules)
 {<text>
     def match_@(rule.Name.Replace("#", ""))(self, context: ParserContext, token: Token) -> bool:
@@ -133,20 +147,24 @@ class @(Model.ParserClassName):
         @:if token.eof():
         @:    return False
         }
-        return self.handle_external_error(context, False, token, context.token_matcher.match_@(rule.Name.Replace("#", "")))</text>
+        return self.handle_external_error(
+            context, False, token, context.token_matcher.match_@(rule.Name.Replace("#", ""))
+        )
+</text>
 }
 
     def match_token(self, state: int, token: Token, context: ParserContext) -> int:
-        state_map: dict[int, Callable[[Token, ParserContext], int]]= {
+        state_map: dict[int, Callable[[Token, ParserContext], int]] = {
         @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
         {
         @:    @state.Id: self.match_token_at_@(state.Id),
         }
         }
-        if state in state_map:
-            return state_map[state](token, context)
-        else:
+
+        if state not in state_map:
             raise RuntimeError("Unknown state: " + str(state))
+
+        return state_map[state](token, context)
 @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
 {<text>
     # @Raw(state.Comment)
@@ -157,39 +175,61 @@ class @(Model.ParserClassName):
             if (transition.LookAheadHint != null)
             {
             @:if self.lookahead_@(transition.LookAheadHint.Id)(context, token):
-            }
                 foreach(var production in transition.Productions)
                 {
-                    @CallProduction(production)
+                    <text>        </text>@CallProduction(production)
                 }
                 @:return @transition.TargetState
+            } else
+            {
+                foreach(var production in transition.Productions)
+                {
+                    <text>    </text>@CallProduction(production)
+                }
+            @:return @transition.TargetState
+            }
         }
 
-        @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state)</text>
+        @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state)
+
+</text>
 }
 @foreach(var lookAheadHint in Model.RuleSet.LookAheadHints)
 {
 <text>
     def lookahead_@(lookAheadHint.Id)(self, context: ParserContext, currentToken: Token) -> bool:
-        currentToken.detach
+        currentToken.detach()
         token = None
         queue = []
         match = False
-        while True:
+        # Effectively do-while
+        continue_lookahead = True
+        while continue_lookahead:
             token = self.read_token(context)
-            token.detach
+            token.detach()
             queue.append(token)
-
-            if (@foreach(var tokenType in lookAheadHint.ExpectedTokens) {<text>self.@MatchToken(tokenType) or </text>}False):
+            @foreach(var tokenType in lookAheadHint.ExpectedTokens) {
+            <text>
+            if self.@MatchToken(tokenType):
                 match = True
-                break
+                break</text>
+            }
+
+
+            continue_lookahead = False
+            @foreach(var tokenType in lookAheadHint.Skip) {
+            <text>
+            if self.@MatchToken(tokenType):
+                continue_lookahead = True
+                continue</text>
+            }
 
-            if not (@foreach(var tokenType in lookAheadHint.Skip) {<text>self.@MatchToken(tokenType) or </text>}False):
-                break
 
         context.token_queue.extend(queue)
 
-        return match</text>
+        return match
+</text>
+
 }
 
     # private

python/gherkin/ast_node.py

@@ -9,7 +9,6 @@
 
 
 class AstNode:
-
     def __init__(self, rule_type: str) -> None:
         self.rule_type = rule_type
         self._sub_items: defaultdict[str, list[object]] = defaultdict(list)

python/gherkin/dialect.py

@@ -33,7 +33,6 @@
 
 
 class Dialect:
-
     @classmethod
     def for_name(cls, name: str) -> Self | None:
         return cls(DIALECTS[name]) if name in DIALECTS else None