Allow yaml-test load template from json to add dependencies (#3452)

now users can put dependencies in src/test/proto, and `load schema
template` command will automatically load these dependencies.

Author: pengpeng-lu

yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/CommandUtil.java

@@ -27,6 +27,10 @@
 import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate;
 import com.apple.foundationdb.relational.yamltests.generated.schemainstance.SchemaInstanceOuterClass;
 
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
 import com.google.protobuf.Descriptors;
 import com.google.protobuf.util.JsonFormat;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -39,10 +43,15 @@
 import java.lang.reflect.Method;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.StringTokenizer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Util class for yaml-tests commands.
@@ -93,13 +102,39 @@ public static Map<String, IndexState> fromIndexStateProto(Map<String, SchemaInst
 
     private static RecordMetaData loadRecordMetaDataFromJson(String jsonFileName) {
         RecordMetaDataProto.MetaData.Builder builder = RecordMetaDataProto.MetaData.newBuilder();
+        List<String> deps = new ArrayList<>();
         try {
-            String json = Files.readString(Paths.get(jsonFileName), StandardCharsets.UTF_8);
-            JsonFormat.parser().ignoringUnknownFields().merge(json, builder);
+            String jsonStr = Files.readString(Paths.get(jsonFileName), StandardCharsets.UTF_8);
+            JsonFormat.parser().ignoringUnknownFields().merge(jsonStr, builder);
+            JsonObject obj = JsonParser.parseString(jsonStr).getAsJsonObject();
+            JsonArray dependencyArray = obj.getAsJsonObject("records").getAsJsonArray("dependency");
+            for (JsonElement element : dependencyArray) {
+                String curDep = element.getAsString();
+                if (!"record_metadata.proto".equals(curDep) && !"record_metadata_options.proto".equals(curDep)) {
+                    deps.add(curDep);
+                }
+            }
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        return RecordMetaData.build(builder.build());
+
+        List<Descriptors.FileDescriptor> fileDescriptors = new ArrayList<>();
+        for (String dep: deps) {
+            try {
+                String fullClassName = getFullClassName(dep);
+                Class<?> act = Class.forName(fullClassName);
+                Method method = act.getMethod("getDescriptor");
+                fileDescriptors.add((Descriptors.FileDescriptor) method.invoke(null));
+            } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException |
+                     IOException | ClassNotFoundException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        return RecordMetaData.newBuilder()
+                .addDependencies(fileDescriptors.toArray(new Descriptors.FileDescriptor[0]))
+                .setRecords(builder.build())
+                .getRecordMetaData();
     }
 
     private static Pair<String, String> parseLoadTemplateString(String loadCommandString) {
@@ -115,6 +150,48 @@ private static Pair<String, String> parseLoadTemplateString(String loadCommandSt
         return new ImmutablePair<>(first, second);
     }
 
+    private static String getFullClassName(String protoFileName) throws IOException {
+        String fullProtoFileName = "src/test/proto/" + protoFileName;
+        final Path path = Paths.get(fullProtoFileName);
+        String content = Files.readString(path);
+
+        // Match 'package my.package.name;'
+        Pattern packagePattern = Pattern.compile("package\\s+([\\w\\.]+);");
+        Matcher packageMatcher = packagePattern.matcher(content);
+
+        String protoPackage;
+        if (packageMatcher.find()) {
+            protoPackage = packageMatcher.group(1);
+        } else {
+            throw new IllegalArgumentException("unable to find package name in proto file:" + fullProtoFileName);
+        }
+
+        // Match 'option java_package = "com.example";'
+        Pattern javaPackagePattern = Pattern.compile("option\\s+java_package\\s*=\\s*\"([^\"]+)\";");
+        Matcher javaPackageMatcher = javaPackagePattern.matcher(content);
+
+        String javaPackage = null;
+        if (javaPackageMatcher.find()) {
+            javaPackage = javaPackageMatcher.group(1);
+        }
+
+        // Final package name: java_package if exists, otherwise proto package
+        String effectivePackage = javaPackage != null ? javaPackage : protoPackage;
+
+        // Match java_outer_classname
+        Pattern outerClassPattern = Pattern.compile("option\\s+java_outer_classname\\s*=\\s*\"([^\"]+)\";");
+        Matcher outerClassMatcher = outerClassPattern.matcher(content);
+
+        String outerClassName;
+        if (outerClassMatcher.find()) {
+            outerClassName = outerClassMatcher.group(1);
+        } else {
+            // fallback: default outer class if not explicitly defined
+            outerClassName = protoFileName.replace(".proto", "").replaceAll("[^A-Za-z0-9]", "");
+        }
+        return effectivePackage + "." + outerClassName;
+    }
+
     /**
      * Utility class that encapsulates <a href="https://en.wikipedia.org/wiki/ANSI_escape_code">ANSI escape sequences</a> for colors.
      */

yaml-tests/src/main/resources/standard_tests_metadata.json

@@ -1,7 +1,8 @@
 {
   "records": {
     "dependency": [
-      "record_metadata_options.proto"
+      "record_metadata_options.proto",
+      "test_student.proto"
     ],
     "message_type": [
       {
@@ -32,6 +33,12 @@
             "name": "COL2",
             "number": 3,
             "type": "TYPE_INT64"
+          },
+          {
+            "name": "student",
+            "number": 4,
+            "type": "TYPE_MESSAGE",
+            "type_name": "com.apple.foundationdb.relational.yamltests.generated.teststudent.Student"
           }
         ]
       }

yaml-tests/src/test/java/YamlIntegrationTests.java

@@ -55,13 +55,13 @@ public void standardTestsWithProto(YamlTest.Runner runner) throws Exception {
     }
 
     @TestTemplate
-    public void fieldIndexTestsProto(YamlTest.Runner runner) throws Exception {
-        runner.runYamsql("field-index-tests-proto.yamsql");
+    public void standardTestsWithMetaData(YamlTest.Runner runner) throws Exception {
+        runner.runYamsql("standard-tests-metadata.yamsql");
     }
 
     @TestTemplate
-    public void standardTestsWithMetaData(YamlTest.Runner runner) throws Exception {
-        runner.runYamsql("standard-tests-metadata.yamsql");
+    public void fieldIndexTestsProto(YamlTest.Runner runner) throws Exception {
+        runner.runYamsql("field-index-tests-proto.yamsql");
     }
 
     @TestTemplate

yaml-tests/src/main/proto/standard_tests.proto renamed to yaml-tests/src/test/proto/standard_tests.proto

@@ -31,11 +31,13 @@ package com.apple.foundationdb.relational.yamltests.generated.standardtests;
 option java_outer_classname = "StandardTestsProto";
 
 import "record_metadata_options.proto";
+import "test_student.proto";
 
 message T1 {
   int64 ID = 1 [(com.apple.foundationdb.record.field).primary_key = true];
   int64 COL1 = 2;
   int64 COL2 = 3;
+  com.apple.foundationdb.relational.yamltests.generated.teststudent.Student student = 4;
 }
 
 message RecordTypeUnion {

yaml-tests/src/test/proto/test_student.proto

@@ -0,0 +1,36 @@
+/*
+ * test_student.proto
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+// make sure to use this package naming convention:
+// com.apple.foundationdb.relational.yamltests.generated.<name_of_test_suite>
+// adding "generated" is important to exclude the generated Java file from Jacoco reports.
+// suffixing the namespace with your test name is important for segregating similarly-named
+// generated-Java-classes (such as RecordTypeUnion) into separate packages, otherwise you
+// get an error from `protoc`.
+package com.apple.foundationdb.relational.yamltests.generated.teststudent;
+
+option java_outer_classname = "TestStudentProto";
+
+message Student {
+  int64 ID = 1;
+  string name = 2;
+}

yaml-tests/src/test/resources/disabled-planner-rewrites/standard-tests-metadata.yamsql

@@ -32,19 +32,19 @@ setup:
   connect: "jdbc:embed:/FRL/STANDARD_METADATA_YAML?schema=TEST"
   steps:
     - query: INSERT INTO T1
-        VALUES (1, 10, 1),
-        (2, 10, 2),
-        (3, 10, 3),
-        (4, 10, 4),
-        (5, 10, 5),
-        (6, 20, 6),
-        (7, 20, 7),
-        (8, 20, 8),
-        (9, 20, 9),
-        (10, 20, 10),
-        (11, 20, 11),
-        (12, 20, 12),
-        (13, 20, 13)
+                VALUES (1, 10, 1, (1, 'Albert')),
+                       (2, 10, 2, (2, 'Bob')),
+                       (3, 10, 3, (3, 'Charles')),
+                       (4, 10, 4, (4, 'Dan')),
+                       (5, 10, 5, (5, 'Evelyn')),
+                       (6, 20, 6, (6, 'Fiona')),
+                       (7, 20, 7, (7, 'George')),
+                       (8, 20, 8, (8, 'Harry')),
+                       (9, 20, 9, (9, 'Isabelle')),
+                       (10, 20, 10, (10, 'Jay')),
+                       (11, 20, 11, (11, 'Kelly')),
+                       (12, 20, 12, (12, 'Liam')),
+                       (13, 20, 13, (13, 'Monica'))
 ---
 test_block:
   connect: "jdbc:embed:/FRL/STANDARD_METADATA_YAML?schema=TEST"
@@ -54,31 +54,17 @@ test_block:
       DISABLE_PLANNER_REWRITING: true
   tests:
     -
-      - query: select * from (select * from (select * from T1) as x where ID = 5) as y;
+      - query: select id, col1, col2 from (select * from (select * from T1) as x where ID = 5) as y;
       - result: [{ID: !l 5, !l 10, !l 5}]
     -
-      - query: select * from (select * from (select * from T1) as x) as y where ID = 5;
+      - query: select id, col1, col2 from (select * from (select * from T1) as x) as y where ID = 5;
       - result: [{ID: !l 5, !l 10, !l 5}]
     -
       - query: select count(*) from (select * from (select * from (select * from T1  where ID = 5) as x) as y) as z;
-      - supported_version: 4.1.9.0
       - explain: "SCAN([EQUALS promote(@c23 AS LONG)]) | MAP (_ AS _0) | AGG (count_star(*) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)"
       - result: [{!l 1}]
     -
-      # Copy of above query to simulate force continuations mode, which does not work prior to 4.1.9.0 due to
-      # https://github.com/FoundationDB/fdb-record-layer/issues/3096. Can remove once we no longer with testing mixed-mode
-      # capabilities with versions older than 4.1.9.0
-      - query: select count(*) from (select * from (select * from (select * from T1  where ID = 5) as x) as y) as z;
-      - maxRows: 1
-      - initialVersionLessThan: 4.1.9.0
-      - result: [{!l 1}]
-      - result: [{!l 0}]
-      - result: [{!l 1}] # ad infinitum
-      - initialVersionAtLeast: 4.1.9.0
-      - result: [{!l 1}]
-      - result: []
-    -
-      - query: select * from (select * from (select * from (select * from T1  where ID > 10) as x) as y) as z;
+      - query: select id, col1, col2 from (select * from (select * from (select * from T1  where ID > 10) as x) as y) as z;
       - result: [{ID: !l 11, !l 20, !l 11}, {ID: !l 12, !l 20, !l 12}, {ID: !l 13, !l 20, !l 13}]
 ---
 setup: