java: Remove redundant ITokenScanner interface and constructors

mpkorstanje · mpkorstanje · commit e8513d450a7b · 2025-04-05T02:11:53.000+02:00
The parser always reads the entire Gherkin document as a String so there
is no need for intermediate steps involving a reader. However, the
`BufferedReader` is still much more efficient than the `Scanner` so
we do use that.
diff --git a/java/gherkin-java.razor b/java/gherkin-java.razor
@@ -70,12 +70,12 @@ class @Model.ParserClassName<T> {
     private final Builder<T> builder;
 
     static class ParserContext {
-        final ITokenScanner tokenScanner;
-        final ITokenMatcher tokenMatcher;
+        final TokenScanner tokenScanner;
+        final TokenMatcher tokenMatcher;
         final Queue<Token> tokenQueue;
         final List<ParserException> errors;
 
-        ParserContext(ITokenScanner tokenScanner, ITokenMatcher tokenMatcher, Queue<Token> tokenQueue, List<ParserException> errors) {
+        ParserContext(TokenScanner tokenScanner, TokenMatcher tokenMatcher, Queue<Token> tokenQueue, List<ParserException> errors) {
             this.tokenScanner = tokenScanner;
             this.tokenMatcher = tokenMatcher;
             this.tokenQueue = tokenQueue;
@@ -88,26 +88,18 @@ class @Model.ParserClassName<T> {
     }
 
     T parse(String source, String uri) {
-        return parse(new StringReader(source), uri);
-    }
-
-    T parse(Reader source, String uri) {
         return parse(new TokenScanner(source), uri);
     }
 
-    T parse(ITokenScanner tokenScanner, String uri) {
+    T parse(TokenScanner tokenScanner, String uri) {
         return parse(tokenScanner, new TokenMatcher(), uri);
     }
 
-    T parse(String source, ITokenMatcher tokenMatcher, String uri) {
-        return parse(new StringReader(source), tokenMatcher, uri);
-    }
-
-    T parse(Reader source, ITokenMatcher tokenMatcher, String uri) {
+    T parse(String source, TokenMatcher tokenMatcher, String uri) {
         return parse(new TokenScanner(source), tokenMatcher, uri);
     }
 
-    T parse(ITokenScanner tokenScanner, ITokenMatcher tokenMatcher, String uri) {
+    T parse(TokenScanner tokenScanner, TokenMatcher tokenMatcher, String uri) {
         builder.reset(uri);
         tokenMatcher.reset();
 
@@ -288,11 +280,7 @@ class @Model.ParserClassName<T> {
         void reset(String uri);
     }
 
-    interface ITokenScanner {
-        Token read();
-    }
-
-    interface ITokenMatcher {
+    interface TokenMatcher {
         @foreach(var rule in Model.RuleSet.TokenRules)
         {
         @:boolean match_@(rule.Name.Replace("#", ""))(Token token);
diff --git a/java/src/main/java/io/cucumber/gherkin/GherkinTokenMatcher.java b/java/src/main/java/io/cucumber/gherkin/GherkinTokenMatcher.java
@@ -5,6 +5,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import io.cucumber.gherkin.Parser.TokenMatcher;
 import io.cucumber.messages.types.StepKeywordType;
 
 import static io.cucumber.gherkin.GherkinLanguageConstants.COMMENT_PREFIX;
@@ -15,27 +16,26 @@
 import static io.cucumber.gherkin.GherkinLanguageConstants.TITLE_KEYWORD_SEPARATOR;
 import static io.cucumber.gherkin.Locations.COLUMN_OFFSET;
 import static io.cucumber.gherkin.Locations.atColumn;
-import static io.cucumber.gherkin.Parser.ITokenMatcher;
 import static io.cucumber.gherkin.Parser.TokenType;
 
-class TokenMatcher implements ITokenMatcher {
+class GherkinTokenMatcher implements TokenMatcher {
 
     private static final Pattern LANGUAGE_PATTERN = Pattern.compile("^#\\s*language\\s*:\\s*([a-zA-Z\\-_]+)\\s*$");
     private final GherkinDialectProvider dialectProvider;
     private GherkinDialect currentDialect;
     private String activeDocStringSeparator = null;
     private int indentToRemove = 0;
 
-    TokenMatcher(GherkinDialectProvider dialectProvider) {
+    GherkinTokenMatcher(GherkinDialectProvider dialectProvider) {
         this.dialectProvider = dialectProvider;
         reset();
     }
 
-    TokenMatcher() {
+    GherkinTokenMatcher() {
         this(new GherkinDialectProvider());
     }
 
-    TokenMatcher(String defaultDialectName) {
+    GherkinTokenMatcher(String defaultDialectName) {
         this(new GherkinDialectProvider(defaultDialectName));
     }
 
diff --git a/java/src/main/java/io/cucumber/gherkin/Parser.java b/java/src/main/java/io/cucumber/gherkin/Parser.java
@@ -84,12 +84,12 @@ enum RuleType {
     private final Builder<T> builder;
 
     static class ParserContext {
-        final ITokenScanner tokenScanner;
-        final ITokenMatcher tokenMatcher;
+        final TokenScanner tokenScanner;
+        final TokenMatcher tokenMatcher;
         final Queue<Token> tokenQueue;
         final List<ParserException> errors;
 
-        ParserContext(ITokenScanner tokenScanner, ITokenMatcher tokenMatcher, Queue<Token> tokenQueue, List<ParserException> errors) {
+        ParserContext(TokenScanner tokenScanner, TokenMatcher tokenMatcher, Queue<Token> tokenQueue, List<ParserException> errors) {
             this.tokenScanner = tokenScanner;
             this.tokenMatcher = tokenMatcher;
             this.tokenQueue = tokenQueue;
@@ -102,26 +102,18 @@ static class ParserContext {
     }
 
     T parse(String source, String uri) {
-        return parse(new StringReader(source), uri);
-    }
-
-    T parse(Reader source, String uri) {
         return parse(new TokenScanner(source), uri);
     }
 
-    T parse(ITokenScanner tokenScanner, String uri) {
-        return parse(tokenScanner, new TokenMatcher(), uri);
-    }
-
-    T parse(String source, ITokenMatcher tokenMatcher, String uri) {
-        return parse(new StringReader(source), tokenMatcher, uri);
+    T parse(TokenScanner tokenScanner, String uri) {
+        return parse(tokenScanner, new GherkinTokenMatcher(), uri);
     }
 
-    T parse(Reader source, ITokenMatcher tokenMatcher, String uri) {
+    T parse(String source, TokenMatcher tokenMatcher, String uri) {
         return parse(new TokenScanner(source), tokenMatcher, uri);
     }
 
-    T parse(ITokenScanner tokenScanner, ITokenMatcher tokenMatcher, String uri) {
+    T parse(TokenScanner tokenScanner, TokenMatcher tokenMatcher, String uri) {
         builder.reset(uri);
         tokenMatcher.reset();
 
@@ -3831,11 +3823,7 @@ interface Builder<T> {
         void reset(String uri);
     }
 
-    interface ITokenScanner {
-        Token read();
-    }
-
-    interface ITokenMatcher {
+    interface TokenMatcher {
         boolean match_EOF(Token token);
         boolean match_Empty(Token token);
         boolean match_Comment(Token token);
diff --git a/java/src/main/java/io/cucumber/gherkin/TokenScanner.java b/java/src/main/java/io/cucumber/gherkin/TokenScanner.java
@@ -4,36 +4,34 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.Reader;
+import java.io.StringReader;
 
 import static io.cucumber.gherkin.Token.createEOF;
 import static io.cucumber.gherkin.Token.createGherkinLine;
 
 /**
- * <p>
- * The scanner reads a gherkin doc (typically read from a .feature file) and creates a token
- * for each line. The tokens are passed to the parser, which outputs an AST (Abstract Syntax Tree).</p>
- * <p>
- * If the scanner sees a # language header, it will reconfigure itself dynamically to look for
- * Gherkin keywords for the associated language. The keywords are defined in gherkin-languages.json.</p>
+ * The scanner reads a gherkin doc (typically read from a .feature file) and
+ * creates a token for each line. The tokens are passed to the parser, which
+ * outputs an AST (Abstract Syntax Tree).</p>
  */
-class TokenScanner implements Parser.ITokenScanner {
+class TokenScanner {
 
     private final BufferedReader reader;
     private int lineNumber;
 
-    TokenScanner(Reader source) {
-        this.reader = new BufferedReader(source);
+    TokenScanner(String source) {
+        this.reader = new BufferedReader(new StringReader(source));
     }
 
-    @Override
-    public Token read() {
+    Token read() {
         try {
             String rawText = reader.readLine();
-            Location location = Locations.atLine(++lineNumber);
             if (rawText == null) {
+                // Don't optimistically increment the line number
+                Location location = Locations.atLine(++lineNumber);
                 return createEOF(location);
             }
+            Location location = Locations.atLine(++lineNumber);
             return createGherkinLine(rawText, location);
         } catch (IOException e) {
             throw new RuntimeException(e);
diff --git a/java/src/test/java/io/cucumber/gherkin/GenerateTokens.java b/java/src/test/java/io/cucumber/gherkin/GenerateTokens.java
@@ -1,19 +1,19 @@
 package io.cucumber.gherkin;
 
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.nio.charset.StandardCharsets;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
 
 public final class GenerateTokens {
-    public static void main(String[] args) throws FileNotFoundException {
+    public static void main(String[] args) throws IOException {
         TokenFormatterBuilder builder = new TokenFormatterBuilder();
         Parser<String> parser = new Parser<>(builder);
-        TokenMatcher matcher = new TokenMatcher();
+        GherkinTokenMatcher matcher = new GherkinTokenMatcher();
         for (String fileName : args) {
-            Reader in = new InputStreamReader(new FileInputStream(fileName), StandardCharsets.UTF_8);
-            String result = parser.parse(in, matcher, fileName);
+            byte[] bytes = readAllBytes(Paths.get(fileName));
+            String result = parser.parse(new String(bytes, UTF_8), matcher, fileName);
             Stdio.out.print(result);
             Stdio.out.flush(); // print doesn't autoflush
         }
diff --git a/java/src/test/java/io/cucumber/gherkin/GherkinDocumentBuilderTest.java b/java/src/test/java/io/cucumber/gherkin/GherkinDocumentBuilderTest.java
@@ -21,7 +21,7 @@ class GherkinDocumentBuilderTest {
     @Test
     void is_reusable() {
         Parser<GherkinDocument> parser = new Parser<>(new GherkinDocumentBuilder(idGenerator, "test.feature"));
-        TokenMatcher matcher = new TokenMatcher();
+        GherkinTokenMatcher matcher = new GherkinTokenMatcher();
 
         GherkinDocument d1 = parser.parse("Feature: 1", matcher, "1.feature");
         GherkinDocument d2 = parser.parse("Feature: 2", matcher, "2.feature");
diff --git a/java/src/test/java/io/cucumber/gherkin/ParserTest.java b/java/src/test/java/io/cucumber/gherkin/ParserTest.java
@@ -10,7 +10,7 @@ class ParserTest {
 
     @Test
     void change_default_language() {
-        TokenMatcher matcher = new TokenMatcher("no");
+        GherkinTokenMatcher matcher = new GherkinTokenMatcher("no");
         IdGenerator idGenerator = new IncrementingIdGenerator();
         Parser<GherkinDocument> parser = new Parser<>(new GherkinDocumentBuilder(idGenerator, "test.feature"));
 
