Implement Compilable SQL functions (#3307)

This introduces compilable SQL functions, as defined, for the most part,
in the SQL standard. The functions currently have the following
characteristics:

* non-temporary, defined, and serialized as part of the schema template.
* lazily compiled, the deserialized form of a SQL function only captures
its SQL definition, and compiled the first time it is referenced in a
SQL query.
* functions can nest, no support for recursive functions.
* they support (named) parameters, with expression default values (no
support for scalar queries).
* functions can be invoked either with named arguments, or unnamed
arguments, (no support for mixing named and unnamed arguments).
* the body of the function is currently limited to a single SQL
statement.

**Example**

```sql
    create table t1(col1 bigint, col2 string, col3 integer, primary key(col1))
    create function f1 ( in a bigint, in b string )
       as select col1, col2 from t1 where col1 < a and col2 = b
    create function f2 ( in k bigint )
       as select col1, col2, col3 from t1 where col3 = k
    create function f3 ( a bigint, b string, c bigint) as select A.col1, A.col2, B.col3 from f1(a, b) A, f2(c) B
```

When the above DDL is parsed, the functions are visited, and serialized
to disk along with the rest of the metadata, the serialized form of
compilable SQL functions comprises their name, and their description,
i.e. their SQL definition.

```sql
insert into t1 values
        (100, 'a', 1),
        (101, 'b', 2),
        (102, 'b', 2),
        (103, 'b', 2),
        (104, 'c', 2),
        (105, 'd', 4)
```

Invoking the functions can be done in two ways:

1. Using unnamed arguments

```sql
select * from f3(103, 'b', 4)
101, 'b', 4
102, 'b', 4
```

2. Using named arguments

```sql
select * from f3(a => 103, b => 'b', c => 4)
101, 'b', 4
102, 'b', 4
```

It is possible to define a function with default parameter values, for
example we could've defined the function `f3` as the following:

```sql
    create function f3 ( in a bigint default 42, in b string default 'b', in c bigint default '4') as select A.col1, A.col2, B.col3 from f1(a, b) A, f2(c) B
```

if we did so, then we could've invoked `f3` as the following:

```sql
select * from f3()
101, 'b', 4
102, 'b', 4
```

This fixes #3325

Author: hatyo

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -721,6 +721,11 @@ public Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMapFromNames(@
         return getFieldDescriptorMap(recordTypeNames.stream().map(this::getRecordType));
     }
 
+    @Nonnull
+    public Map<String, UserDefinedFunction> getUserDefinedFunctionMap() {
+        return userDefinedFunctionMap;
+    }
+
     @Nonnull
     public static Map<String, Descriptors.FieldDescriptor> getFieldDescriptorMapFromTypes(@Nonnull final Collection<RecordType> recordTypes) {
         if (recordTypes.size() == 1) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java

@@ -229,8 +229,9 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa
             }
         }
         for (RecordMetaDataProto.PUserDefinedFunction function: metaDataProto.getUserDefinedFunctionsList()) {
-            UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
-                    PlanHashable.CURRENT_FOR_CONTINUATION), function);
+            final UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(
+                    new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
+                            PlanHashable.CURRENT_FOR_CONTINUATION), function);
             userDefinedFunctionMap.put(func.getFunctionName(), func);
         }
         if (metaDataProto.hasSplitLongRecords()) {

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java

@@ -31,7 +31,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
-import com.apple.foundationdb.record.query.plan.cascades.values.FunctionCatalog;
+import com.apple.foundationdb.record.query.plan.cascades.values.BuiltInFunctionCatalog;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
 import com.apple.foundationdb.record.util.HashUtils;
 import com.apple.foundationdb.record.util.ServiceLoaderProvider;
@@ -276,7 +276,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
     protected Value resolveAndEncapsulateFunction(@Nonnull final String functionName,
                                                   @Nonnull final List<? extends Value> argumentValues) {
         final BuiltInFunction<?> builtInFunction =
-                FunctionCatalog.resolve(functionName, argumentValues.size())
+                BuiltInFunctionCatalog.resolve(functionName, argumentValues.size())
                         .orElseThrow(() -> new RecordCoreArgumentException("unknown function",
                                 LogMessageKeys.FUNCTION, getName()));
         return (Value)builtInFunction.encapsulate(argumentValues);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BitmapAggregateIndexExpansionVisitor.java

@@ -31,7 +31,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.predicates.Placeholder;
 import com.apple.foundationdb.record.query.plan.cascades.values.ArithmeticValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
-import com.apple.foundationdb.record.query.plan.cascades.values.FunctionCatalog;
+import com.apple.foundationdb.record.query.plan.cascades.values.BuiltInFunctionCatalog;
 import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.NumericAggregationValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
@@ -86,8 +86,8 @@ protected NonnullPair<Quantifier, List<Placeholder>> constructGroupBy(@Nonnull f
         }
 
 
-        final var bitmapConstructAggFunc = FunctionCatalog.getFunctionSingleton(NumericAggregationValue.BitmapConstructAggFn.class).orElseThrow();
-        final var bitmapBitPositionFunc = FunctionCatalog.getFunctionSingleton(ArithmeticValue.BitmapBitPositionFn.class).orElseThrow();
+        final var bitmapConstructAggFunc = BuiltInFunctionCatalog.getFunctionSingleton(NumericAggregationValue.BitmapConstructAggFn.class).orElseThrow();
+        final var bitmapBitPositionFunc = BuiltInFunctionCatalog.getFunctionSingleton(ArithmeticValue.BitmapBitPositionFn.class).orElseThrow();
         final String sizeArgument = index.getOption(IndexOptions.BITMAP_VALUE_ENTRY_SIZE_OPTION);
         final int entrySize = sizeArgument != null ? Integer.parseInt(sizeArgument) : BitmapValueIndexMaintainer.DEFAULT_ENTRY_SIZE;
         final var entrySizeValue = LiteralValue.ofScalar(entrySize);
@@ -101,7 +101,7 @@ protected NonnullPair<Quantifier, List<Placeholder>> constructGroupBy(@Nonnull f
                 .stream()
                 .map(Column::getValue)
                 .collect(ImmutableList.toImmutableList());
-        final var bitmapBitPosition = FunctionCatalog.getFunctionSingleton(ArithmeticValue.BitmapBucketOffsetFn.class).orElseThrow();
+        final var bitmapBitPosition = BuiltInFunctionCatalog.getFunctionSingleton(ArithmeticValue.BitmapBucketOffsetFn.class).orElseThrow();
         final var implicitGroupingValue = (Value)bitmapBitPosition.encapsulate(ImmutableList.of(argument, entrySizeValue));
         final var placeHolder = Placeholder.newInstanceWithoutRanges(implicitGroupingValue, newParameterAlias());
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java

@@ -20,14 +20,14 @@
 
 package com.apple.foundationdb.record.query.plan.cascades;
 
+import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Typed;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
+import java.util.Map;
 
 /**
  * Main interface for defining a built-in function that can be evaluated against a number of arguments.
@@ -64,46 +64,6 @@ protected BuiltInFunction(@Nonnull final String functionName, @Nonnull final Lis
         this.encapsulationFunction = encapsulationFunction;
     }
 
-    /**
-     * Checks whether the provided list of argument types matches the list of function's parameter types.
-     *
-     * @param argumentTypes The argument types list.
-     * @return if the arguments type match, an {@link Optional} containing <code>this</code> instance, otherwise
-     * and empty {@link Optional}.
-     */
-    @SuppressWarnings("java:S3776")
-    @Nonnull
-    public Optional<BuiltInFunction<T>> validateCall(@Nonnull final List<Type> argumentTypes) {
-        int numberOfArguments = argumentTypes.size();
-
-        final List<Type> functionParameterTypes = getParameterTypes();
-
-        if (numberOfArguments > functionParameterTypes.size() && !hasVariadicSuffix()) {
-            return Optional.empty();
-        }
-
-        // check the type codes of the fixed parameters
-        for (int i = 0; i < functionParameterTypes.size(); i ++) {
-            final Type typeI = functionParameterTypes.get(i);
-            if (typeI.getTypeCode() != Type.TypeCode.ANY && typeI.getTypeCode() != argumentTypes.get(i).getTypeCode()) {
-                return Optional.empty();
-            }
-        }
-
-        if (hasVariadicSuffix()) { // This is variadic function, match the rest of arguments, if any.
-            final Type functionVariadicSuffixType = Objects.requireNonNull(getVariadicSuffixType());
-            if (functionVariadicSuffixType.getTypeCode() != Type.TypeCode.ANY) {
-                for (int i = getParameterTypes().size(); i < numberOfArguments; i++) {
-                    if (argumentTypes.get(i).getTypeCode() != functionVariadicSuffixType.getTypeCode()) {
-                        return Optional.empty();
-                    }
-                }
-            }
-        }
-
-        return Optional.of(this);
-    }
-
     @Nonnull
     public EncapsulationFunction<T> getEncapsulationFunction() {
         return encapsulationFunction;
@@ -114,4 +74,10 @@ public EncapsulationFunction<T> getEncapsulationFunction() {
     public Typed encapsulate(@Nonnull final List<? extends Typed> arguments) {
         return encapsulationFunction.encapsulate(this, arguments);
     }
+
+    @Nonnull
+    @Override
+    public Typed encapsulate(@Nonnull final Map<String, ? extends Typed> namedArguments) {
+        throw new RecordCoreException("built-in functions do not support named argument calling conventions");
+    }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CatalogedFunction.java

@@ -23,18 +23,29 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Typed;
+import com.google.common.base.Functions;
 import com.google.common.base.Verify;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Streams;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Collection;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * Main interface for defining all functions that can be evaluated against a number of arguments.
  * Two major sub interfaces inherit this interface: {@link BuiltInFunction} and {@link UserDefinedFunction}
- * {@link BuiltInFunction} represents all functions that are built-in, and stored in code, while {@link UserDefinedFunction} represents all functions defined by users, and stored in {@link com.apple.foundationdb.record.RecordMetaDataProto.MetaData}
+ * {@link BuiltInFunction} represents all functions that are built-in, and stored in code, while
+ * {@link UserDefinedFunction} represents all functions defined by users, and stored in
+ * {@link com.apple.foundationdb.record.RecordMetaDataProto.MetaData}
  */
 @API(API.Status.EXPERIMENTAL)
 public abstract class CatalogedFunction {
@@ -44,18 +55,39 @@ public abstract class CatalogedFunction {
     @Nonnull
     protected final List<Type> parameterTypes;
 
+    @Nonnull
+    protected final BiMap<String, Integer> parameterNamesMap;
+
+    @Nonnull
+    protected final List<Optional<? extends Typed>> parameterDefaults;
+
     /**
      * The type of the function's variadic parameters (if any).
      */
     @Nullable
     private final Type variadicSuffixType;
 
-    protected CatalogedFunction(@Nonnull final String functionName, @Nonnull final List<Type> parameterTypes, @Nullable final Type variadicSuffixType) {
+    protected CatalogedFunction(@Nonnull final String functionName, @Nonnull final List<Type> parameterTypes,
+                                @Nullable final Type variadicSuffixType) {
         this.functionName = functionName;
         this.parameterTypes = ImmutableList.copyOf(parameterTypes);
+        this.parameterDefaults = ImmutableList.of();
+        this.parameterNamesMap = ImmutableBiMap.of();
         this.variadicSuffixType = variadicSuffixType;
     }
 
+    protected CatalogedFunction(@Nonnull final String functionName, @Nonnull final List<String> parameterNames,
+                                @Nonnull final List<Type> parameterTypes,
+                                @Nonnull final List<Optional<? extends Typed>> parameterDefaults) {
+        Verify.verify(parameterNames.size() == parameterTypes.size());
+        this.functionName = functionName;
+        this.parameterTypes = ImmutableList.copyOf(parameterTypes);
+        this.parameterNamesMap = IntStream.range(0, parameterNames.size()).boxed().collect(ImmutableBiMap.toImmutableBiMap(parameterNames::get,
+                Functions.identity()));
+        this.parameterDefaults = ImmutableList.copyOf(parameterDefaults);
+        this.variadicSuffixType = null; // no support yet with named parameters.
+    }
+
     @Nonnull
     public String getFunctionName() {
         return functionName;
@@ -66,6 +98,20 @@ public List<Type> getParameterTypes() {
         return parameterTypes;
     }
 
+    public boolean hasNamedParameters() {
+        return !parameterNamesMap.isEmpty();
+    }
+
+    @Nonnull
+    public String getParameterName(int index) {
+        return parameterNamesMap.inverse().get(index);
+    }
+
+    @Nonnull
+    public Collection<String> getParameterNames() {
+        return parameterNamesMap.keySet();
+    }
+
     @Nullable
     public Type getVariadicSuffixType() {
         return variadicSuffixType;
@@ -76,7 +122,7 @@ public boolean hasVariadicSuffix() {
     }
 
     @Nonnull
-    public Type resolveParameterType(int index) {
+    public Type conputeParameterType(int index) {
         Verify.verify(index >= 0, "unexpected negative parameter index");
         if (index < parameterTypes.size()) {
             return parameterTypes.get(index);
@@ -89,15 +135,69 @@ public Type resolveParameterType(int index) {
     }
 
     @Nonnull
-    public List<Type> resolveParameterTypes(int numberOfArguments) {
+    public Type conputeParameterType(@Nonnull final String parameterName) {
+        return conputeParameterType(getParamIndex(parameterName));
+    }
+
+    @Nonnull
+    public List<Type> getParameterTypes(int numberOfArguments) {
         Verify.verify(numberOfArguments > 0, "unexpected number of arguments");
         final ImmutableList.Builder<Type> resultBuilder = ImmutableList.builder();
         for (int i = 0; i < numberOfArguments; i ++) {
-            resultBuilder.add(resolveParameterType(i));
+            resultBuilder.add(conputeParameterType(i));
         }
         return resultBuilder.build();
     }
 
+    public int getParamIndex(@Nonnull final String parameter) {
+        return parameterNamesMap.get(parameter);
+    }
+
+    public Optional<? extends Typed> getDefaultValue(@Nonnull final String paramName) {
+        return parameterDefaults.get(getParamIndex(paramName));
+    }
+
+    public Optional<? extends Typed> getDefaultValue(int paramIndex) {
+        return parameterDefaults.get(paramIndex);
+    }
+
+    public boolean hasDefaultValue(@Nonnull final String paramName) {
+        return getDefaultValue(paramName).isPresent();
+    }
+
+    public boolean hasDefaultValue(int paramIndex) {
+        return getDefaultValue(paramIndex).isPresent();
+    }
+
+    /**
+     * Validates the function invocation using named arguments. It checks whether the argument names match parameter
+     * names, and that any missing parameter has a default value. It effectively leaves all the work related to handling
+     * permisslbe implicit promotions to the actual function invocation.
+     * @param namedArgumentsTypeMap a list of named arguments.
+     * @return if the arguments type match, an {@link Optional} containing <code>this</code> instance, otherwise
+     * and empty {@link Optional}.
+     */
+    @Nonnull
+    public Optional<? extends CatalogedFunction> validateCall(@Nonnull final Map<String, ? extends Typed> namedArgumentsTypeMap) {
+        if (parameterNamesMap.isEmpty()) {
+            return Optional.empty();
+        }
+        final var argumentNames = namedArgumentsTypeMap.keySet();
+        final var parameterNames = parameterNamesMap.keySet();
+        final var unknownArgs = Sets.difference(argumentNames, parameterNames);
+        if (!unknownArgs.isEmpty()) {
+            return Optional.empty();
+        }
+        final var missingParams = Sets.difference(parameterNames, argumentNames);
+        for (final var missingParam : missingParams) {
+            if (!hasDefaultValue(missingParam)) {
+                return Optional.empty();
+            }
+        }
+        return Optional.of(this);
+    }
+
+    @SuppressWarnings("UnstableApiUsage")
     @Nonnull
     @Override
     public String toString() {
@@ -109,9 +209,18 @@ public String toString() {
             }
         }
 
+        final var parameterNames = parameterNamesMap.keySet();
+        if (!parameterNamesMap.isEmpty()) {
+            return functionName + "(" + Streams.zip(parameterNames.stream(), parameterTypes.stream(),
+                    (name, type) -> name + " -> " + type).collect(Collectors.joining(",")) + variadicSuffixString + ")";
+        }
+
         return functionName + "(" + parameterTypes.stream().map(Object::toString).collect(Collectors.joining(",")) + variadicSuffixString + ")";
     }
 
     @Nonnull
     public abstract Typed encapsulate(@Nonnull List<? extends Typed> arguments);
+
+    @Nonnull
+    public abstract Typed encapsulate(@Nonnull Map<String, ? extends Typed> namedArguments);
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java

@@ -22,6 +22,7 @@
 
 import com.apple.foundationdb.record.PlanDeserializer;
 import com.apple.foundationdb.record.PlanSerializationContext;
+import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.RecordMetaDataProto;
 import com.apple.foundationdb.record.planprotos.PMacroFunctionValue;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Typed;
@@ -32,6 +33,7 @@
 
 import javax.annotation.Nonnull;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -78,6 +80,13 @@ public RecordMetaDataProto.PUserDefinedFunction toProto(@Nonnull PlanSerializati
                 .build();
     }
 
+
+    @Nonnull
+    @Override
+    public Typed encapsulate(@Nonnull final Map<String, ? extends Typed> namedArguments) {
+        throw new RecordCoreException("macro functions do not support named argument calling conventions");
+    }
+
     @Nonnull
     public static MacroFunction fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PMacroFunctionValue functionValue) {
         return new MacroFunction(