Merge branch 'topic/lktql_frontend' into 'master'

Introduce the Lkt frontend to LKQL

See merge request eng/libadalang/langkit-query-language!563

Author: Killian Perlin

lkql/lkql.lkt

@@ -10,7 +10,6 @@ lexer lkql_lexer {
     Coma <- ","
     SemiCol <- ";"
     Colon <- ":"
-    UnderScore <- "_"
     Eq <- "="
     EqEq <- "=="
     Neq <- "!="
@@ -65,7 +64,7 @@ lexer lkql_lexer {
     @symbol() Pass <- "pass"
     @symbol() RunPass <- "run_pass"
     Integer <- p"[0-9]+"
-    @symbol() Identifier <- p"[a-z][A-Za-z0-9_]*"
+    @symbol() Identifier <- p"[a-z_][A-Za-z0-9_]*"
     @symbol() UpperIdentifier <- p"[A-Za-z][A-Za-z0-9_]*"
     String <- p"\"(\\\\.|[^\"])*\""
     @trivia()

lkql_jit/builtins_annotations/src/main/java/com/adacore/lkql_jit/annotations/BuiltInProcessor.java

@@ -103,35 +103,47 @@ public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment
             stream.println("import com.adacore.lkql_jit.utils.LKQLTypesHelper;");
             stream.println("import com.adacore.lkql_jit.built_ins.BuiltInFunctionValue;");
             stream.println("import com.adacore.lkql_jit.built_ins.BuiltInMethodFactory;");
+            stream.println("import org.graalvm.collections.Pair;");
 
             for (var p : builtInPackages) {
                 stream.println("import " + p.getKey() + "." + p.getValue() + ";");
             }
             stream.println("public class AllBuiltIns {");
 
             stream.println(
-                "    static private List<BuiltInFunctionValue> allFunctionsCache = null;"
+                "    static private Map<String, Pair<Integer, BuiltInFunctionValue>>" +
+                " allFunctionsCache = null;"
             );
             stream.println(
                 "    static private Map<String, Map<String, BuiltInMethodFactory>>" +
                 " allMethodsCache = null;"
             );
 
             // allFunctions method
-            stream.println("    public static List<BuiltInFunctionValue> allFunctions() {");
-            stream.println("        if (allFunctionsCache != null) {");
-            stream.println("            return allFunctionsCache;");
-            stream.println("        }");
-            stream.println("        allFunctionsCache = Stream.of(");
+            stream.println(
+                "public static Map<String, Pair<Integer, BuiltInFunctionValue>> functions() {"
+            );
+            stream.println("    if (allFunctionsCache != null) {");
+            stream.println("        return allFunctionsCache;");
+            stream.println("    }");
+            stream.println("    allFunctionsCache = new HashMap<>();");
+            stream.println("    var idx = 0;");
+            stream.println("    var fns = Stream.of(");
             stream.println(
                 builtInPackages
                     .stream()
                     .map(p -> "            Arrays.stream(" + p.getValue() + ".getFunctions())")
                     .collect(Collectors.joining(",\n")) +
                 ")"
             );
-            stream.println("           .reduce(Stream::concat).orElseGet(Stream::empty)");
-            stream.println("           .collect(Collectors.toList());");
+
+            stream.println(".reduce(Stream::concat).orElseGet(Stream::empty).toList();");
+            stream.println("    for (var fn : fns) {");
+            stream.println(
+                "        allFunctionsCache.put(fn.getExecutableName(), Pair.create(idx, fn));"
+            );
+            stream.println("        idx += 1;");
+            stream.println("     }");
             stream.println("        return allFunctionsCache;");
             stream.println("    }");
 

lkql_jit/cli/src/main/java/com/adacore/lkql_jit/cli/LKQLLauncher.java

@@ -172,7 +172,7 @@ protected int executeScript(Context.Builder contextBuilder) {
         // Create the context and run the script in it
         try (Context context = contextBuilder.build()) {
             if (this.args.script != null) {
-                final Source source = Source.newBuilder("lkql", new File(this.args.script)).build();
+                Source source = Source.newBuilder("lkql", new File(this.args.script)).build();
                 context.eval(source);
             }
 

lkql_jit/cli/src/main/java/com/adacore/lkql_jit/cli/LKQLRefactor.java

@@ -7,14 +7,11 @@
 
 import static com.adacore.liblkqllang.Liblkqllang.*;
 
-import java.io.PrintWriter;
-import java.nio.charset.StandardCharsets;
+import com.adacore.lkql_jit.options.Refactorings.LKQLToLkt;
+import com.adacore.lkql_jit.options.Refactorings.Refactoring;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.Callable;
-import java.util.function.Consumer;
-import java.util.function.Predicate;
 import picocli.CommandLine;
 
 /**
@@ -51,213 +48,58 @@ public class LKQLRefactor implements Callable<Integer> {
     @CommandLine.Option(names = { "-i", "--inplace" }, description = "Rewrite the files in place")
     public boolean inPlace;
 
-    private enum refactoringKind {
+    private enum RefactoringKind {
         IS_TO_COLON,
+        TO_LKQL_V2,
     }
 
     @CommandLine.Option(
         names = { "-r", "--refactoring" },
         description = "Refactoring to run. Valid values: ${COMPLETION-CANDIDATES}",
         required = true
     )
-    private refactoringKind refactoring;
-
-    /** Kind of actions that can be attached to a token. */
-    private enum actionKind {
-        APPEND,
-        PREPEND,
-        REPLACE,
-    }
-
-    /**
-     * Action record, encapsulating an action that can be attached to a token. When applied, the
-     * action will either append the given text, prepend the given text, or replace the token text
-     * with the given text, depending on the kind.
-     *
-     * <p>To remove a token, simply use a REPLACE action with an empty text.
-     */
-    private record Action(actionKind kind, String text) {}
-
-    /**
-     * Global map of actions that will be applied on tokens. To append an action, use the addAction
-     * method.
-     */
-    private final HashMap<String, List<Action>> actions = new HashMap<>();
-
-    /**
-     * Helper to create a unique Id for a token. TODO: This method exists only because the
-     * 'hashCode' method on Tokens is wrong. This should be fixed at the Langkit level. See
-     * eng/libadalang/langkit#857.
-     *
-     * @return the id as a string
-     */
-    private String getTokenId(Token token) {
-        return (
-            token.unit.getFileName() +
-            (token.isTrivia() ? "trivia " + token.triviaIndex : token.tokenIndex) +
-            token.kind
-        );
-    }
-
-    /** Add an action to the list of actions to apply */
-    public void addAction(Token token, Action action) {
-        List<Action> actionList;
-        var tokenId = getTokenId(token);
-        if (actions.containsKey(tokenId)) {
-            actionList = actions.get(tokenId);
-        } else {
-            actionList = new ArrayList<>();
-            actions.put(tokenId, actionList);
-        }
-
-        actionList.add(action);
-    }
-
-    /**
-     * Helper for refactor writers: Returns the first token that satisfies the predicate 'pred',
-     * iterating on tokens from 'fromTok' (including 'fromTok').
-     *
-     * <p>If no token is found, return the null token.
-     */
-    public static Token firstWithPred(Token fromTok, Predicate<Token> pred) {
-        var curTok = fromTok;
-        while (!curTok.isNone() && !pred.test(curTok)) {
-            curTok = curTok.next();
-        }
-        return curTok;
-    }
-
-    /**
-     * Internal method: rewrite a unit by printing all the tokens via the 'write' callback. Apply
-     * the actions at the same time.
-     */
-    private void printAllTokens(AnalysisUnit unit, Consumer<String> write) {
-        for (var tok = unit.getFirstToken(); !tok.isNone(); tok = tok.next()) {
-            var tokActions = actions.get(getTokenId(tok));
-            if (tokActions != null) {
-                var replaceActions = tokActions
-                    .stream()
-                    .filter(c -> c.kind == actionKind.REPLACE)
-                    .toList();
-                assert replaceActions.size() <= 1 : "Only one replace action per token";
-
-                var prependActions = tokActions
-                    .stream()
-                    .filter(c -> c.kind == actionKind.PREPEND)
-                    .toList();
-
-                var appendActions = tokActions
-                    .stream()
-                    .filter(c -> c.kind == actionKind.APPEND)
-                    .toList();
-
-                for (var action : prependActions) {
-                    write.accept(action.text);
-                }
-
-                if (!replaceActions.isEmpty()) {
-                    write.accept(replaceActions.get(0).text);
-                } else {
-                    write.accept(tok.getText());
-                }
-
-                for (var action : appendActions) {
-                    write.accept(action.text);
-                }
-            } else {
-                write.accept(tok.getText());
-            }
-        }
-    }
-
-    /**
-     * Internal method: rewrite a unit by printing all the tokens, either to stdout, or to the
-     * original file, depending on the value of the '-i' command line flag.
-     */
-    private void printAllTokens(AnalysisUnit unit) {
-        try {
-            if (this.inPlace) {
-                var writer = new PrintWriter(unit.getFileName(), StandardCharsets.UTF_8);
-                printAllTokens(unit, writer::print);
-                writer.close();
-            } else {
-                printAllTokens(unit, System.out::print);
-            }
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * Helper for findAll. Visit all children of 'node', calling 'cons' on each of them. TODO: Hoist
-     * in Java bindings (see eng/libadalang/langkit#859)
-     */
-    private static void visitChildren(LkqlNode node, Consumer<LkqlNode> cons) {
-        if (node == null || node.isNone()) {
-            return;
-        }
-
-        for (var c : node.children()) {
-            if (c != null && !c.isNone()) {
-                cons.accept(c);
-                visitChildren(c, cons);
-            }
-        }
-    }
-
-    /**
-     * Helper for refactor writers: Find all nodes that are children of root and which satisfies the
-     * predicate 'pred'. TODO: Hoist in Java bindings (see eng/libadalang/langkit#859)
-     */
-    public static List<LkqlNode> findAll(LkqlNode root, Predicate<LkqlNode> pred) {
-        var result = new ArrayList<LkqlNode>();
-        visitChildren(root, c -> {
-            if (pred.test(c)) {
-                result.add(c);
-            }
-        });
-        return result;
-    }
+    private RefactoringKind refactoring;
 
     /**
      * Return the refactoring corresponding to enum value passed on command line. This is where
      * concrete refactorings are implemented.
      */
-    public Consumer<AnalysisUnit> getRefactoring() {
-        switch (refactoring) {
-            case IS_TO_COLON:
-                return unit -> {
-                    for (var det : findAll(unit.getRoot(), n -> n instanceof NodePatternDetail)) {
-                        var tokIs = firstWithPred(
-                            det.tokenStart(),
-                            t -> t.kind == TokenKind.LKQL_IS
-                        );
-
-                        if (!tokIs.isNone()) {
-                            // Replace "is" -> ":"
-                            addAction(tokIs, new Action(actionKind.REPLACE, ":"));
-
-                            // Get rid of previous token if it is a whitespace
-                            var prev = tokIs.previous();
-                            if (prev.kind == TokenKind.LKQL_WHITESPACE) {
-                                addAction(tokIs.previous(), new Action(actionKind.REPLACE, ""));
-                            }
+    public Refactoring getRefactoring(AnalysisUnit unit) {
+        return switch (refactoring) {
+            case IS_TO_COLON -> (Refactoring.State state) -> {
+                for (var det : Refactoring.findAll(unit.getRoot(), n ->
+                    n instanceof NodePatternDetail
+                )) {
+                    var tokIs = Refactoring.firstWithPred(
+                        det.tokenStart(),
+                        t -> t.kind == TokenKind.LKQL_IS
+                    );
+
+                    if (!tokIs.isNone()) {
+                        // Replace "is" -> ":"
+                        state.replace(tokIs, ":");
+
+                        // Get rid of previous token if it is a whitespace
+                        var prev = tokIs.previous();
+                        if (Refactoring.isWhitespace(prev)) {
+                            state.delete(tokIs.previous());
                         }
                     }
-                };
-        }
-        return null;
+                }
+            };
+            case TO_LKQL_V2 -> LKQLToLkt.instantiate();
+        };
     }
 
     @Override
     public Integer call() {
-        var refactoring = getRefactoring();
         var ctx = AnalysisContext.create();
         for (var file : files) {
             var unit = ctx.getUnitFromFile(file);
-            refactoring.accept(unit);
-            printAllTokens(unit);
-            actions.clear();
+            var refactoring = getRefactoring(unit);
+            var state = new Refactoring.State(unit);
+            refactoring.applyRefactor(state);
+            state.printAllTokens(this.inPlace);
         }
         return 0;
     }

lkql_jit/language/pom.xml

@@ -94,6 +94,12 @@
             <version>${config.liblkqllangVersion}</version>
         </dependency>
 
+        <dependency>
+            <groupId>com.adacore</groupId>
+            <artifactId>liblktlang</artifactId>
+            <version>${config.liblktlangVersion}</version>
+        </dependency>
+
         <dependency>
             <groupId>com.adacore</groupId>
             <artifactId>libadalang</artifactId>