Also add Unicode support to HCL parser

Author: knutwannheden

rewrite-hcl/src/main/java/org/openrewrite/hcl/internal/HclParserVisitor.java

@@ -48,7 +48,14 @@ public class HclParserVisitor extends HCLParserBaseVisitor<Hcl> {
     @Nullable
     private final FileAttributes fileAttributes;
 
+    /**
+     * Track position within the file by character
+     */
     private int cursor = 0;
+    /**
+     * Track parsing position within the file by Unicode code point
+     */
+    private int codePointCursor = 0;
 
     public HclParserVisitor(Path path, String source, Charset charset, boolean charsetBomMarked, @Nullable FileAttributes fileAttributes) {
         this.path = path;
@@ -146,7 +153,7 @@ public Hcl visitBinaryOp(HCLParser.BinaryOpContext ctx) {
                     break;
             }
             Space opPrefix = Space.format(prefix(ctx.binaryOperator()));
-            cursor = ctx.binaryOperator().getStop().getStopIndex() + 1;
+            advanceCursor(ctx.binaryOperator().getStop().getStopIndex() + 1);
 
             if (c.unaryOp() != null) {
                 right = (Expression) visit(c.operation() != null ? c.operation() : c.exprTerm(0));
@@ -380,7 +387,7 @@ private List<Expression> visitHeredocTemplateExpressions(List<HCLParser.HeredocT
             if (part.heredocLiteral() != null) {
                 Space prefix = Space.format(prefix(part.heredocLiteral()));
                 String value = part.heredocLiteral().getText();
-                cursor = part.heredocLiteral().getStop().getStopIndex() + 1;
+                advanceCursor(part.heredocLiteral().getStop().getStopIndex() + 1);
                 expressions.add(new Hcl.Literal(randomId(), prefix, Markers.EMPTY, value, value));
             } else if (part.templateInterpolation() != null) {
                 Space prefix = Space.format(prefix(part.templateInterpolation()));
@@ -623,7 +630,7 @@ private List<Expression> visitTemplateExpressions(List<HCLParser.QuotedTemplateP
             if (part.stringLiteral() != null) {
                 Space prefix = Space.format(prefix(part.stringLiteral()));
                 String value = part.stringLiteral().getText();
-                cursor = part.stringLiteral().getStop().getStopIndex() + 1;
+                advanceCursor(part.stringLiteral().getStop().getStopIndex() + 1);
                 expressions.add(new Hcl.Literal(randomId(), prefix, Markers.EMPTY, value, value));
             } else if (part.templateInterpolation() != null) {
                 Space prefix = Space.format(prefix(part.templateInterpolation()));
@@ -738,12 +745,12 @@ private String prefix(ParserRuleContext ctx) {
 
     private String prefix(Token token) {
         int start = token.getStartIndex();
-        if (start < cursor) {
+        if (start < codePointCursor) {
             return "";
         }
-        String prefix = source.substring(cursor, start);
-        cursor = start;
-        return prefix;
+        int oldCursor = cursor;
+        advanceCursor(start);
+        return source.substring(oldCursor, cursor);
     }
 
     private String prefix(@Nullable TerminalNode terminalNode) {
@@ -757,15 +764,15 @@ private String prefix(@Nullable TerminalNode terminalNode) {
 
         T t = conversion.apply(ctx, prefix(ctx));
         if (ctx.getStop() != null) {
-            cursor = ctx.getStop().getStopIndex() + (Character.isWhitespace(source.charAt(ctx.getStop().getStopIndex())) ? 0 : 1);
+            advanceCursor(ctx.getStop().getStopIndex() + (Character.isWhitespace(source.charAt(ctx.getStop().getStopIndex())) ? 0 : 1));
         }
 
         return t;
     }
 
     private String skip(TerminalNode node) {
         String prefix = prefix(node);
-        cursor = node.getSymbol().getStopIndex() + 1;
+        advanceCursor(node.getSymbol().getStopIndex() + 1);
         return prefix;
     }
 
@@ -785,7 +792,9 @@ private Space sourceBefore(String untilDelim, @Nullable Character stop) {
         }
 
         String prefix = source.substring(cursor, delimIndex);
-        cursor += prefix.length() + untilDelim.length(); // advance past the delimiter
+        int codePointsInPrefix = prefix.codePointCount(0, prefix.length());
+        // All HCL delimiters are ASCII, so length == code point count
+        advanceCursor(codePointCursor + codePointsInPrefix + untilDelim.length());
         return Space.format(prefix);
     }
 
@@ -832,4 +841,17 @@ private int positionOfNext(String untilDelim, @Nullable Character stop) {
 
         return delimIndex > source.length() - untilDelim.length() ? -1 : delimIndex;
     }
+
+    /**
+     *  Advance both the cursor and the code point cursor
+     */
+    @SuppressWarnings("UnusedReturnValue")
+    private int advanceCursor(int newCodePointIndex) {
+        if (newCodePointIndex <= codePointCursor) {
+            return cursor;
+        }
+        cursor = source.offsetByCodePoints(cursor, newCodePointIndex - codePointCursor);
+        codePointCursor = newCodePointIndex;
+        return cursor;
+    }
 }

rewrite-hcl/src/test/java/org/openrewrite/hcl/HclParserTest.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2025 the original author or authors.
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * https://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.openrewrite.hcl;
+
+import org.junit.jupiter.api.Test;
+import org.openrewrite.test.RewriteTest;
+
+import static org.openrewrite.hcl.Assertions.hcl;
+
+public class HclParserTest implements RewriteTest {
+
+    @Test
+    void unicode() {
+        rewriteRun(
+          hcl(
+            """
+              tags = /*👇*/{
+                git_file =/*👇*/ "terraform/aws/👇.tf"
+                git_repo /*👇*/= "terragoat"
+              }
+              """
+          )
+        );
+    }
+}