Add support for multiline strings in compiler directives

Author: cirras

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Support for multiline string literals within compiler directives.
 - Support for the `TEXTBLOCK` directive.
 - **API:** `CompilerDirectiveParser` can now return a new `TextBlockDirective` type.
 - **API:** `CheckVerifier::withCompilerVersion` method.

delphi-checks-testkit/src/main/java/au/com/integradev/delphi/checks/verifier/CheckVerifierImpl.java

@@ -280,7 +280,9 @@ private List<Issue> execute() {
     var sensorContext = SensorContextTester.create(FileUtils.getTempDirectory());
     sensorContext.settings().setProperty(DelphiProperties.TEST_TYPE_KEY, "Test.TTestSuite");
 
-    var compilerDirectiveParser = new CompilerDirectiveParserImpl(Platform.WINDOWS);
+    var compilerDirectiveParser =
+        new CompilerDirectiveParserImpl(
+            Platform.WINDOWS, file.getTextBlockLineEndingModeRegistry());
 
     var checkRegistrar = mock(MasterCheckRegistrar.class);
     when(checkRegistrar.getRuleKey(check))

delphi-frontend/src/main/java/au/com/integradev/delphi/executor/DelphiChecksExecutor.java

@@ -57,7 +57,8 @@ public DelphiChecksExecutor(
   @Override
   public void execute(Context context, DelphiInputFile delphiFile) {
     Platform platform = delphiProjectHelper.getToolchain().platform;
-    CompilerDirectiveParser compilerDirectiveParser = new CompilerDirectiveParserImpl(platform);
+    CompilerDirectiveParser compilerDirectiveParser =
+        new CompilerDirectiveParserImpl(platform, delphiFile.getTextBlockLineEndingModeRegistry());
     Function<DelphiCheck, DelphiCheckContext> createCheckContext =
         check ->
             new DelphiCheckContextImpl(

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessor.java

@@ -51,7 +51,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonar.plugins.communitydelphi.api.directive.CompilerDirective;
-import org.sonar.plugins.communitydelphi.api.directive.CompilerDirectiveParser;
 import org.sonar.plugins.communitydelphi.api.directive.ConditionalDirective;
 import org.sonar.plugins.communitydelphi.api.directive.SwitchDirective.SwitchKind;
 import org.sonar.plugins.communitydelphi.api.directive.TextBlockDirective.LineEndingKind;
@@ -152,7 +151,7 @@ private void processToken(Token token) {
     tokenIndex++;
 
     if (token.getType() == DelphiLexer.TkCompilerDirective) {
-      CompilerDirectiveParser parser = new CompilerDirectiveParserImpl(platform);
+      var parser = new CompilerDirectiveParserImpl(platform, getTextBlockLineEndingModeRegistry());
       DelphiToken directiveToken = new DelphiTokenImpl(token);
       parser.parse(directiveToken).ifPresent(this::processDirective);
     } else if (!parentDirective.isEmpty()) {

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/directive/CompilerDirectiveParserImpl.java

@@ -22,6 +22,7 @@
 import static au.com.integradev.delphi.preprocessor.directive.CompilerDirectiveParserImpl.DirectiveBracketType.PAREN;
 
 import au.com.integradev.delphi.compiler.Platform;
+import au.com.integradev.delphi.preprocessor.TextBlockLineEndingModeRegistry;
 import au.com.integradev.delphi.preprocessor.directive.expression.Expression;
 import au.com.integradev.delphi.preprocessor.directive.expression.ExpressionLexer;
 import au.com.integradev.delphi.preprocessor.directive.expression.ExpressionLexer.ExpressionLexerError;
@@ -46,7 +47,6 @@
 
 public class CompilerDirectiveParserImpl implements CompilerDirectiveParser {
   private static final ExpressionLexer EXPRESSION_LEXER = new ExpressionLexer();
-  private static final ExpressionParser EXPRESSION_PARSER = new ExpressionParser();
 
   private static final char END_OF_INPUT = '\0';
 
@@ -56,6 +56,7 @@ enum DirectiveBracketType {
   }
 
   private final Platform platform;
+  private final TextBlockLineEndingModeRegistry textBlockLineEndingModeRegistry;
 
   // Parser state
   private String data;
@@ -65,8 +66,10 @@ enum DirectiveBracketType {
   private DelphiToken token;
   private DirectiveBracketType directiveBracketType;
 
-  public CompilerDirectiveParserImpl(Platform platform) {
+  public CompilerDirectiveParserImpl(
+      Platform platform, TextBlockLineEndingModeRegistry textBlockLineEndingModeRegistry) {
     this.platform = platform;
+    this.textBlockLineEndingModeRegistry = textBlockLineEndingModeRegistry;
   }
 
   @Override
@@ -300,12 +303,17 @@ private Expression readExpression() {
 
     try {
       var tokens = EXPRESSION_LEXER.lex(input.toString());
-      return EXPRESSION_PARSER.parse(tokens);
+      return expressionParser().parse(tokens);
     } catch (ExpressionLexerError | ExpressionParserError e) {
       throw new CompilerDirectiveParserError(e, token);
     }
   }
 
+  private ExpressionParser expressionParser() {
+    int index = token.getIndex();
+    return new ExpressionParser(textBlockLineEndingModeRegistry.getLineEndingMode(index));
+  }
+
   private boolean isEndOfDirective(char character) {
     boolean result = false;
 

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/directive/expression/ExpressionLexer.java

@@ -87,8 +87,12 @@ public List<Token> lex(String data) {
   }
 
   private char peekChar() {
-    if (position < data.length()) {
-      return data.charAt(position);
+    return peekChar(0);
+  }
+
+  private char peekChar(int offset) {
+    if (position + offset < data.length()) {
+      return data.charAt(position + offset);
     }
     return END_OF_INPUT;
   }
@@ -123,7 +127,7 @@ private Token readToken() {
       } else if (SYNTAX_CHARACTERS.containsKey(character)) {
         return readSyntaxToken();
       } else if (character == '\'') {
-        return readSingleQuoteString();
+        return readString();
       } else {
         throw new ExpressionLexerError("Unexpected character: '" + character + "'");
       }
@@ -182,25 +186,83 @@ private Token readIdentifier() {
     return new Token(type, text);
   }
 
-  private Token readSingleQuoteString() {
-    getChar();
+  private Token readString() {
+    Token result = readMultilineString();
+    if (result == null) {
+      result = readSingleLineString();
+    }
+    return result;
+  }
+
+  private Token readSingleLineString() {
     StringBuilder value = new StringBuilder();
+    value.append(getChar());
+
     char character;
 
     while ((character = getChar()) != END_OF_INPUT) {
+      value.append(character);
       if (character == '\'') {
-        if (peekChar() != '\'') {
+        if (peekChar() == '\'') {
+          value.append(getChar());
+        } else {
           break;
         }
-        getChar();
-      } else {
-        value.append(character);
       }
     }
 
     return new Token(TokenType.STRING, value.toString());
   }
 
+  private Token readMultilineString() {
+    int lookahead = lookaheadMultilineString();
+    if (lookahead == 0) {
+      return null;
+    }
+
+    String value = data.substring(position, position + lookahead);
+    position += lookahead;
+
+    return new Token(TokenType.MULTILINE_STRING, value);
+  }
+
+  private int lookaheadMultilineString() {
+    int startQuotes = lookaheadSingleQuotes(0);
+    if (startQuotes >= 3 && (startQuotes % 2 == 1) && isNewLine(peekChar(startQuotes))) {
+      int i = startQuotes;
+      while (true) {
+        switch (peekChar(++i)) {
+          case '\'':
+            int quotes = Math.min(startQuotes, lookaheadSingleQuotes(i));
+            i += quotes;
+            if (quotes == startQuotes) {
+              return i;
+            }
+            break;
+
+          case END_OF_INPUT:
+            return 0;
+
+          default:
+            // do nothing
+        }
+      }
+    }
+    return 0;
+  }
+
+  private int lookaheadSingleQuotes(int i) {
+    int result = 0;
+    while (peekChar(i++) == '\'') {
+      ++result;
+    }
+    return result;
+  }
+
+  private boolean isNewLine(int c) {
+    return c == '\r' || c == '\n';
+  }
+
   private static boolean isHexDigit(char character) {
     character = Character.toLowerCase(character);
     return Character.isDigit(character) || (character >= 'a' && character <= 'f');

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/directive/expression/ExpressionParser.java

@@ -39,14 +39,19 @@
 import static au.com.integradev.delphi.preprocessor.directive.expression.Token.TokenType.XOR;
 import static java.util.Objects.requireNonNullElse;
 
+import au.com.integradev.delphi.preprocessor.TextBlockLineEndingMode;
 import au.com.integradev.delphi.preprocessor.directive.expression.Token.TokenType;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Deque;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
+import org.apache.commons.lang3.StringUtils;
 
 public class ExpressionParser {
   public static class ExpressionParserError extends RuntimeException {
@@ -70,10 +75,16 @@ public static class ExpressionParserError extends RuntimeException {
   private static final ImmutableSet<TokenType> UNARY_OPERATORS =
       Sets.immutableEnumSet(PLUS, MINUS, NOT);
 
+  private final TextBlockLineEndingMode textBlockLineEndingMode;
+
   // Parser state
   private List<Token> tokens;
   private int position;
 
+  public ExpressionParser(TextBlockLineEndingMode textBlockLineEndingMode) {
+    this.textBlockLineEndingMode = textBlockLineEndingMode;
+  }
+
   public Expression parse(List<Token> tokens) {
     this.tokens = tokens;
     this.position = 0;
@@ -169,6 +180,7 @@ private Expression parsePrimary() {
     Token token = peekToken();
     switch (token.getType()) {
       case STRING:
+      case MULTILINE_STRING:
       case INTEGER:
       case REAL:
         return parseLiteral();
@@ -191,7 +203,64 @@ private Expression parsePrimary() {
 
   private Expression parseLiteral() {
     Token token = getToken();
-    return Expressions.literal(token.getType(), token.getText());
+
+    String text;
+    switch (token.getType()) {
+      case STRING:
+        text = evaluateString(token.getText());
+        break;
+      case MULTILINE_STRING:
+        text = evaluateMultilineString(token.getText(), textBlockLineEndingMode);
+        break;
+      default:
+        text = token.getText();
+    }
+
+    return Expressions.literal(token.getType(), text);
+  }
+
+  private static String evaluateString(String text) {
+    text = text.substring(1, text.length() - 1);
+    text = text.replace("''", "'");
+    return text;
+  }
+
+  private String evaluateMultilineString(String text, TextBlockLineEndingMode lineEndingMode) {
+    Deque<String> lines = text.lines().collect(Collectors.toCollection(ArrayDeque<String>::new));
+
+    lines.removeFirst();
+
+    String last = lines.removeLast();
+    String indentation = readLeadingWhitespace(last);
+
+    String lineEnding;
+    switch (lineEndingMode) {
+      case CR:
+        lineEnding = "\r";
+        break;
+      case LF:
+        lineEnding = "\n";
+        break;
+      default:
+        lineEnding = "\r\n";
+    }
+
+    return lines.stream()
+        .map(line -> StringUtils.removeStart(line, indentation))
+        .collect(Collectors.joining(lineEnding));
+  }
+
+  private static String readLeadingWhitespace(String input) {
+    StringBuilder result = new StringBuilder();
+    for (int i = 0; i < input.length(); ++i) {
+      char c = input.charAt(i);
+      if (c <= 0x20 || c == 0x3000) {
+        result.append(c);
+      } else {
+        break;
+      }
+    }
+    return result.toString();
   }
 
   private Expression parseIdentifier() {

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/directive/expression/Expressions.java

@@ -186,6 +186,7 @@ private static ExpressionValue createValue(TokenType type, String text) {
         case REAL:
           return createReal(doubleFromTextWithDigitSeparators(text));
         case STRING:
+        case MULTILINE_STRING:
           return createString(text);
         default:
           throw new AssertionError("Unhandled literal expression type: " + type.name());

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/directive/expression/Token.java

@@ -25,6 +25,7 @@ enum TokenType {
     REAL,
     IDENTIFIER,
     STRING,
+    MULTILINE_STRING,
     EQUALS,
     NOT_EQUALS,
     LESS_THAN,

delphi-frontend/src/test/java/au/com/integradev/delphi/preprocessor/directive/CompilerDirectiveParserTest.java

@@ -20,6 +20,7 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
 
 import au.com.integradev.delphi.antlr.DelphiLexer;
 import au.com.integradev.delphi.antlr.ast.token.DelphiTokenImpl;
@@ -53,7 +54,7 @@ class CompilerDirectiveParserTest {
 
   @BeforeEach
   void setup() {
-    parser = new CompilerDirectiveParserImpl(Platform.WINDOWS);
+    parser = new CompilerDirectiveParserImpl(Platform.WINDOWS, mock());
   }
 
   @Test