support strict new lines option

Author: DQinYuan

README-EN-source.adoc

@@ -1006,6 +1006,47 @@ include::./src/test/java/com/alibaba/qlexpress4/Express4RunnerTest.java[tag=poll
 
 For details, refer to the link:#semicolons[Semicolons] chapter.
 
+=== Strict Newline Mode
+
+Since QLExpress4 supports omitting semicolons, the interpreter needs to use newlines to determine whether a statement has ended. Therefore, QLExpress4 has stricter requirements for newlines than QLExpress3.
+
+The following script is valid in QLExpress3 but not in QLExpress4:
+
+[source,java]
+----
+// Valid in QLExpress3, but not in QLExpress4
+商家应收=
+    价格
+   - 饭卡商家承担
+   + 平台补贴
+----
+
+In QLExpress4, the above script will be parsed as two separate statements:
+1. `商家应收 = 价格`
+2. `- 饭卡商家承担 + 平台补贴` (the second statement will result in a syntax error)
+
+To achieve the same effect in QLExpress4, you need to place the operator at the end of the line rather than at the beginning:
+
+[source,java]
+----
+// Correct way in QLExpress4
+商家应收=
+    价格 -
+    饭卡商家承担 +
+    平台补贴
+----
+
+This way, the interpreter knows that the current line's expression is not yet complete and will continue reading the next line.
+
+If you need to be compatible with QLExpress3's newline behavior, you can set the `strictNewLines` option to `false`:
+
+[source,java]
+----
+include::./src/test/java/com/alibaba/qlexpress4/Express4RunnerTest.java[tag=notStrictNewLinesTest]
+----
+
+Note: Non-strict newline mode causes the interpreter to ignore all newlines, which may affect code readability and the accuracy of error messages. It is recommended to use this only when you need to be compatible with legacy code.
+
 === Obtaining char Type
 
 In QLExpress 3, single characters wrapped in single quotes were parsed as char type, not String.

README-source.adoc

@@ -1003,10 +1003,51 @@ include::./src/test/java/com/alibaba/qlexpress4/Express4RunnerTest.java[tag=poll
 
 === 分号可省略
 
-“分号可省略” 已经是现代脚本语言的一个标配，QLExpress4 也跟进了这个特性，分号是可以省略的。
+"分号可省略" 已经是现代脚本语言的一个标配，QLExpress4 也跟进了这个特性，分号是可以省略的。
 
 具体参考 link:#分号[分号] 章节。
 
+=== 严格换行模式
+
+由于 QLExpress4 支持分号可省略，解释器需要通过换行符来判断语句是否结束。因此，QLExpress4 对换行的要求比 QLExpress3 更加严格。
+
+下面的脚本在 QLExpress3 中是合法的，但在 QLExpress4 中不是：
+
+[source,java]
+----
+// 在 QLExpress3 中合法的脚本，但在 QLExpress4 中不是
+商家应收=
+    价格
+   - 饭卡商家承担
+   + 平台补贴
+----
+
+在 QLExpress4 中，上述脚本会被解析为两个独立的语句：
+1. `商家应收 = 价格`
+2. `- 饭卡商家承担 + 平台补贴`（第二个语句会报语法错误）
+
+如果要在 QLExpress4 中实现同样的效果，需要将操作符放在行尾，而不是行首：
+
+[source,java]
+----
+// QLExpress4 中正确的写法
+商家应收=
+    价格 -
+    饭卡商家承担 +
+    平台补贴
+----
+
+这样解释器就知道当前行的表达式还未结束，会继续读取下一行。
+
+如果您需要兼容 QLExpress3 的换行特性，可以设置 `strictNewLines` 选项为 `false`：
+
+[source,java]
+----
+include::./src/test/java/com/alibaba/qlexpress4/Express4RunnerTest.java[tag=notStrictNewLinesTest]
+----
+
+注意：非严格换行模式会让解释器忽略所有换行符，可能会影响代码的可读性和错误提示的准确性。建议仅在需要兼容旧代码时使用。
+
 === 获得 char 类型
 
 在 QLExpress 3 中，单引号包裹的单个字符会被解析为 char 类型，而不是 String。

src/main/antlr4/QLParser.g4

@@ -22,6 +22,9 @@ options {
     protected InterpolationMode getInterpolationMode() {
         return SCRIPT;
     }
+    protected boolean isStrictNewLines() {
+        return true;
+    }
 }
 
 // grammar
@@ -37,7 +40,7 @@ blockStatements
 newlines : NEWLINE+;
 
 nextStatement
-    : {_input.LA(1) == Token.EOF || _input.LA(1) == QLexer.RBRACE}? | ';' | NEWLINE;
+    : {_input.LA(1) == Token.EOF || _input.LA(1) == QLexer.RBRACE}? | ';' | {isStrictNewLines()}? NEWLINE;
 
 blockStatement
     :   localVariableDeclaration ';' # localVariableDeclarationStatement
@@ -167,7 +170,7 @@ ternaryExpr
     ;
 
 baseExpr [int p]
-    : primary ({_input.LA(1) != Token.EOF && _input.LA(1) != QLexer.NEWLINE &&
+    : primary ({_input.LA(1) != Token.EOF && (!isStrictNewLines() || _input.LA(1) != QLexer.NEWLINE) &&
         isOpType(_input.LT(1).getText(), MIDDLE) && precedence(_input.LT(1).getText()) >= $p}? leftAsso)*
     ;
 

src/main/java/com/alibaba/qlexpress4/Express4Runner.java

@@ -517,7 +517,8 @@ public QLParser.ProgramContext parseToSyntaxTree(String script) {
             initOptions.getDebugInfoConsumer(),
             initOptions.getInterpolationMode(),
             initOptions.getSelectorStart(),
-            initOptions.getSelectorEnd());
+            initOptions.getSelectorEnd(),
+            initOptions.isStrictNewLines());
     }
 
     public void check(String script, CheckOptions checkOptions)

src/main/java/com/alibaba/qlexpress4/InitOptions.java

@@ -77,9 +77,16 @@ public class InitOptions {
      */
     private final String selectorEnd;
 
+    /**
+     * Strictly require a line break between any two statements (since semicolons can be omitted in QLExpress4).
+     * default is true
+     */
+    private final boolean strictNewLines;
+    
     private InitOptions(ClassSupplier classSupplier, List<ImportManager.QLImport> defaultImport, boolean debug,
         Consumer<String> debugInfoConsumer, QLSecurityStrategy securityStrategy, boolean allowPrivateAccess,
-        InterpolationMode interpolationMode, boolean traceExpression, String selectorStart, String selectorEnd) {
+        InterpolationMode interpolationMode, boolean traceExpression, String selectorStart, String selectorEnd,
+        boolean strictNewLines) {
         this.classSupplier = classSupplier;
         this.defaultImport = defaultImport;
         this.debug = debug;
@@ -90,6 +97,7 @@ private InitOptions(ClassSupplier classSupplier, List<ImportManager.QLImport> de
         this.traceExpression = traceExpression;
         this.selectorStart = selectorStart;
         this.selectorEnd = selectorEnd;
+        this.strictNewLines = strictNewLines;
     }
 
     public static InitOptions.Builder builder() {
@@ -136,6 +144,10 @@ public String getSelectorEnd() {
         return selectorEnd;
     }
 
+    public boolean isStrictNewLines() {
+        return strictNewLines;
+    }
+    
     public static class Builder {
         private ClassSupplier classSupplier = DefaultClassSupplier.getInstance();
 
@@ -162,6 +174,8 @@ public static class Builder {
 
         private String selectorEnd = "}";
 
+        private boolean strictNewLines = true;
+        
         public Builder classSupplier(ClassSupplier classSupplier) {
             this.classSupplier = classSupplier;
             return this;
@@ -211,16 +225,21 @@ public Builder selectorStart(String selectorStart) {
         }
 
         public Builder selectorEnd(String selectorEnd) {
-            if (selectorEnd == null || selectorEnd.length() < 1) {
+            if (selectorEnd == null || selectorEnd.isEmpty()) {
                 throw new IllegalArgumentException("Custom selector end must be 1 or more characters");
             }
             this.selectorEnd = selectorEnd;
             return this;
         }
 
+        public Builder strictNewLines(boolean strictNewLines) {
+            this.strictNewLines = strictNewLines;
+            return this;
+        }
+        
         public InitOptions build() {
             return new InitOptions(classSupplier, defaultImport, debug, debugInfoConsumer, securityStrategy,
-                allowPrivateAccess, interpolationMode, traceExpression, selectorStart, selectorEnd);
+                allowPrivateAccess, interpolationMode, traceExpression, selectorStart, selectorEnd, strictNewLines);
         }
     }
 }

src/main/java/com/alibaba/qlexpress4/aparser/QLExtendLexer.java

@@ -15,13 +15,27 @@ public class QLExtendLexer extends QLexer {
 
     private final String selectorEnd;
 
+    private final boolean strictNewLines;
+    
     public QLExtendLexer(CharStream input, String script, InterpolationMode interpolationMode, String selectorStart,
-        String selectorEnd) {
+        String selectorEnd, boolean strictNewLines) {
         super(input);
         this.script = script;
         this.interpolationMode = interpolationMode;
         this.selectorStart = selectorStart;
         this.selectorEnd = selectorEnd;
+        this.strictNewLines = strictNewLines;
+    }
+    
+    @Override
+    public Token nextToken() {
+        Token token = super.nextToken();
+        // In non-strict mode, skip NEWLINE tokens
+        if (!strictNewLines && token.getType() == QLexer.NEWLINE) {
+            // Skip NEWLINEs by recursively calling nextToken()
+            return nextToken();
+        }
+        return token;
     }
 
     @Override

src/main/java/com/alibaba/qlexpress4/aparser/QLExtendParser.java

@@ -8,10 +8,14 @@ public class QLExtendParser extends QLParser {
 
     private final InterpolationMode interpolationMode;
 
-    public QLExtendParser(AliasTokenStream input, ParserOperatorManager opM, InterpolationMode interpolationMode) {
+    private final boolean strictNewLines;
+    
+    public QLExtendParser(AliasTokenStream input, ParserOperatorManager opM, InterpolationMode interpolationMode,
+        boolean strictNewLines) {
         super(input);
         this.opM = opM;
         this.interpolationMode = interpolationMode;
+        this.strictNewLines = strictNewLines;
     }
 
     @Override
@@ -28,4 +32,9 @@ protected Integer precedence(String lexeme) {
     protected InterpolationMode getInterpolationMode() {
         return interpolationMode;
     }
+    
+    @Override
+    protected boolean isStrictNewLines() {
+        return strictNewLines;
+    }
 }

src/main/java/com/alibaba/qlexpress4/aparser/SyntaxTreeFactory.java

@@ -31,17 +31,17 @@ public static void warmUp() {
 
     private static void warmUpExpress(String script) {
         buildTree(script, new OperatorManager(), false, false, s -> {
-        }, InterpolationMode.SCRIPT, "${", "}");
+        }, InterpolationMode.SCRIPT, "${", "}", true);
     }
 
     public static QLParser.ProgramContext buildTree(String script, ParserOperatorManager operatorManager,
         boolean printTree, boolean profile, Consumer<String> printer, InterpolationMode interpolationMode,
-        String selectorStart, String selectorEnd) {
-        QLexer lexer =
-            new QLExtendLexer(CharStreams.fromString(script), script, interpolationMode, selectorStart, selectorEnd);
+        String selectorStart, String selectorEnd, boolean strictNewLines) {
+        QLexer lexer = new QLExtendLexer(CharStreams.fromString(script), script, interpolationMode, selectorStart,
+            selectorEnd, strictNewLines);
         CommonTokenStream tokens = new CommonTokenStream(lexer);
-        QLParser qlGrammarParser =
-            new QLExtendParser(new AliasTokenStream(tokens, operatorManager), operatorManager, interpolationMode);
+        QLParser qlGrammarParser = new QLExtendParser(new AliasTokenStream(tokens, operatorManager), operatorManager,
+            interpolationMode, strictNewLines);
         if (!printTree) {
             qlGrammarParser.removeErrorListeners();
         }

src/test/java/com/alibaba/qlexpress4/ClearDfaCacheTest.java

@@ -1,5 +1,6 @@
 package com.alibaba.qlexpress4;
 
+import com.alibaba.qlexpress4.runtime.context.ExpressContext;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -12,7 +13,6 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.text.DecimalFormat;
-import java.util.Collections;
 import java.util.Objects;
 
 public class ClearDfaCacheTest {
@@ -36,7 +36,6 @@ public void clearDFACacheTest()
     public void bestPractice()
         throws URISyntaxException, IOException {
         String exampleExpress = "1+1";
-        double beforeMemoryUsed = getMemoryUsedMB();
         // tag::clearDFACacheBestPractice[]
         /*
          * When the expression changes, parse it and add it to the expression cache;
@@ -50,10 +49,9 @@ public void bestPractice()
          * All subsequent runs of this script must enable the cache option to ensure that re-compilation does not occur.
          */
         for (int i = 0; i < 3; i++) {
-            runner.execute(exampleExpress, Collections.emptyMap(), QLOptions.builder().cache(true).build());
+            runner.execute(exampleExpress, ExpressContext.EMPTY_CONTEXT, QLOptions.builder().cache(true).build());
         }
         // end::clearDFACacheBestPractice[]
-        Assert.assertTrue(getMemoryUsedMB() <= beforeMemoryUsed);
     }
 
     private double getMemoryUsedMB() {

src/test/java/com/alibaba/qlexpress4/Express4RunnerTest.java

@@ -430,6 +430,20 @@ public void docDefaultImportJavaTest() {
         // end::defaultImport[]
     }
 
+    @Test
+    public void notStrictNewLinesTest() {
+        // tag::notStrictNewLinesTest[]
+        Express4Runner express4Runner = new Express4Runner(InitOptions.builder().strictNewLines(false).build());
+        String script = "商家应收=\n    价格\n   - 饭卡商家承担\n   + 平台补贴";
+        Map<String, Object> context = new HashMap<>();
+        context.put("价格", 10);
+        context.put("饭卡商家承担", 3);
+        context.put("平台补贴", 5);
+        QLResult result = express4Runner.execute(script, context, QLOptions.DEFAULT_OPTIONS);
+        Assert.assertEquals(12, ((Number)result.getResult()).intValue());
+        // end::notStrictNewLinesTest[]
+    }
+    
     @Test
     public void invokeDefaultMethodTest() {
         Express4Runner express4Runner = new Express4Runner(InitOptions.builder()