diff --git a/presto-main-base/pom.xml b/presto-main-base/pom.xml index f724e286320a5..91b0dd413daa9 100644 --- a/presto-main-base/pom.xml +++ b/presto-main-base/pom.xml @@ -88,6 +88,11 @@ bootstrap + + com.facebook.airlift + http-client + + io.airlift aircompressor diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java index 40de75c838169..364aafb54881e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java @@ -15,6 +15,7 @@ import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.spi.function.BuiltInType; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.Signature; @@ -30,12 +31,16 @@ public class BuiltInFunctionHandle implements FunctionHandle { private final Signature signature; + private final BuiltInType builtInType; @JsonCreator - public BuiltInFunctionHandle(@JsonProperty("signature") Signature signature) + public BuiltInFunctionHandle( + @JsonProperty("signature") Signature signature, + @JsonProperty("builtInType") BuiltInType builtInType) { this.signature = requireNonNull(signature, "signature is null"); checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); + this.builtInType = builtInType; } @JsonProperty @@ -62,6 +67,13 @@ public List getArgumentTypes() return signature.getArgumentTypes(); } + @JsonProperty + @Override + public BuiltInType getBuiltInType() + { + return builtInType; + } + @Override public CatalogSchemaName getCatalogSchemaName() { @@ -78,13 +90,14 @@ public boolean equals(Object o) return false; } BuiltInFunctionHandle that = (BuiltInFunctionHandle) o; - return Objects.equals(signature, that.signature); + return Objects.equals(signature, that.signature) + && Objects.equals(builtInType, that.builtInType); } @Override public int hashCode() { - return Objects.hash(signature); + return Objects.hash(signature, builtInType); } @Override diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInNativeFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInNativeFunctionNamespaceManager.java new file mode 100644 index 0000000000000..b1317fccaed1b --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInNativeFunctionNamespaceManager.java @@ -0,0 +1,112 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.BuiltInType; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.google.common.util.concurrent.UncheckedExecutionException; + +import javax.inject.Provider; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static com.facebook.presto.spi.function.FunctionImplementationType.CPP; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Throwables.throwIfInstanceOf; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class BuiltInNativeFunctionNamespaceManager + extends BuiltInSpecialFunctionNamespaceManager +{ + private static final Logger log = Logger.get(BuiltInNativeFunctionNamespaceManager.class); + + private final Provider nativeFunctionRegistryToolProvider; + + public BuiltInNativeFunctionNamespaceManager(FunctionAndTypeManager functionAndTypeManager, Provider nativeFunctionRegistryToolProvider) + { + super(functionAndTypeManager); + this.nativeFunctionRegistryToolProvider = requireNonNull(nativeFunctionRegistryToolProvider, "nativeFunctionRegistryToolProvider is null"); + } + + public synchronized void registerNativeFunctions() + { + // only register functions once + if (!this.functions.list().isEmpty()) { + return; + } + + List allNativeFunctions = nativeFunctionRegistryToolProvider + .get() + .getNativeFunctionSignatureMap() + .getUDFSignatureMap() + .entrySet() + .stream() + .flatMap(entry -> entry.getValue().stream() + .map(metaInfo -> NativeSidecarFunctionUtil.createSqlInvokedFunction(entry.getKey(), metaInfo))) + .collect(Collectors.toList()); + this.functions = new FunctionMap(this.functions, allNativeFunctions); + } + + @Override + public FunctionHandle getFunctionHandle(Signature signature) + { + return new BuiltInFunctionHandle(signature, BuiltInType.NATIVE); + } + + @Override + public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + { + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + Signature signature = ((BuiltInFunctionHandle) functionHandle).getSignature(); + SpecializedFunctionKey functionKey; + try { + functionKey = specializedFunctionKeyCache.getUnchecked(signature); + } + catch (UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), PrestoException.class); + throw e; + } + SqlFunction function = functionKey.getFunction(); + checkArgument(function instanceof SqlInvokedFunction, "BuiltInPluginFunctionNamespaceManager only support SqlInvokedFunctions"); + SqlInvokedFunction sqlFunction = (SqlInvokedFunction) function; + List argumentNames = sqlFunction.getParameters().stream().map(Parameter::getName).collect(toImmutableList()); + return new FunctionMetadata( + signature.getName(), + signature.getArgumentTypes(), + argumentNames, + signature.getReturnType(), + signature.getKind(), + sqlFunction.getRoutineCharacteristics().getLanguage(), + CPP, + function.isDeterministic(), + function.isCalledOnNullInput(), + sqlFunction.getVersion(), + sqlFunction.getComplexTypeFunctionDescriptor()); + } + + @Override + protected synchronized void checkForNamingConflicts(Collection functions) + { + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInPluginFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInPluginFunctionNamespaceManager.java new file mode 100644 index 0000000000000..6faac2379a0b9 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInPluginFunctionNamespaceManager.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.BuiltInType; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.google.common.util.concurrent.UncheckedExecutionException; + +import java.util.Collection; +import java.util.List; + +import static com.facebook.presto.spi.function.FunctionImplementationType.SQL; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Throwables.throwIfInstanceOf; +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class BuiltInPluginFunctionNamespaceManager + extends BuiltInSpecialFunctionNamespaceManager +{ + public BuiltInPluginFunctionNamespaceManager(FunctionAndTypeManager functionAndTypeManager) + { + super(functionAndTypeManager); + } + + public synchronized void registerPluginFunctions(List functions) + { + checkForNamingConflicts(functions); + this.functions = new FunctionMap(this.functions, functions); + } + + @Override + public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) + { + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + Signature signature = ((BuiltInFunctionHandle) functionHandle).getSignature(); + SpecializedFunctionKey functionKey; + try { + functionKey = specializedFunctionKeyCache.getUnchecked(signature); + } + catch (UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), PrestoException.class); + throw e; + } + SqlFunction function = functionKey.getFunction(); + checkArgument(function instanceof SqlInvokedFunction, "BuiltInPluginFunctionNamespaceManager only support SqlInvokedFunctions"); + SqlInvokedFunction sqlFunction = (SqlInvokedFunction) function; + List argumentNames = sqlFunction.getParameters().stream().map(Parameter::getName).collect(toImmutableList()); + return new FunctionMetadata( + signature.getName(), + signature.getArgumentTypes(), + argumentNames, + signature.getReturnType(), + signature.getKind(), + sqlFunction.getRoutineCharacteristics().getLanguage(), + SQL, + function.isDeterministic(), + function.isCalledOnNullInput(), + sqlFunction.getVersion(), + sqlFunction.getComplexTypeFunctionDescriptor()); + } + + @Override + public FunctionHandle getFunctionHandle(Signature signature) + { + return new BuiltInFunctionHandle(signature, BuiltInType.PLUGIN); + } + + @Override + protected synchronized void checkForNamingConflicts(Collection functions) + { + for (SqlFunction function : functions) { + for (SqlFunction existingFunction : this.functions.list()) { + checkArgument(!function.getSignature().equals(existingFunction.getSignature()), "Function already registered: %s", function.getSignature()); + } + } + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java new file mode 100644 index 0000000000000..1c99e334734f0 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java @@ -0,0 +1,137 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.metadata; + +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.util.concurrent.UncheckedExecutionException; + +import java.util.Collection; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Throwables.throwIfInstanceOf; +import static java.util.Collections.emptyList; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.HOURS; + +public abstract class BuiltInSpecialFunctionNamespaceManager +{ + protected volatile FunctionMap functions = new FunctionMap(); + protected final FunctionAndTypeManager functionAndTypeManager; + protected final Supplier cachedFunctions = + Suppliers.memoize(this::createFunctionMap); + protected final LoadingCache specializedFunctionKeyCache; + protected final LoadingCache specializedScalarCache; + + public BuiltInSpecialFunctionNamespaceManager(FunctionAndTypeManager functionAndTypeManager) + { + this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); + specializedFunctionKeyCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, HOURS) + .build(CacheLoader.from(this::doGetSpecializedFunctionKey)); + specializedScalarCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, HOURS) + .build(CacheLoader.from(key -> { + checkArgument( + key.getFunction() instanceof SqlInvokedFunction, + "Unsupported scalar function class: %s", + key.getFunction().getClass()); + return new SqlInvokedScalarFunctionImplementation(((SqlInvokedFunction) key.getFunction()).getBody()); + })); + } + + public Collection getFunctions(QualifiedObjectName functionName) + { + if (functions.list().isEmpty() || + (!functionName.getCatalogSchemaName().equals(functionAndTypeManager.getDefaultNamespace()))) { + return emptyList(); + } + return cachedFunctions.get().get(functionName); + } + + public List listFunctions() + { + return cachedFunctions.get().list(); + } + + public abstract FunctionHandle getFunctionHandle(Signature signature); + + public abstract FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); + + public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHandle functionHandle) + { + checkArgument(functionHandle instanceof BuiltInFunctionHandle, "Expect BuiltInFunctionHandle"); + return getScalarFunctionImplementation(((BuiltInFunctionHandle) functionHandle).getSignature()); + } + + private ScalarFunctionImplementation getScalarFunctionImplementation(Signature signature) + { + checkArgument(signature.getKind() == SCALAR, "%s is not a scalar function", signature); + checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); + + try { + return specializedScalarCache.getUnchecked(getSpecializedFunctionKey(signature)); + } + catch (UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), PrestoException.class); + throw e; + } + } + + private synchronized FunctionMap createFunctionMap() + { + Optional> functionNamespaceManager = + functionAndTypeManager.getServingFunctionNamespaceManager(functionAndTypeManager.getDefaultNamespace()); + checkArgument(functionNamespaceManager.isPresent(), "Cannot find function namespace for catalog '%s'", functionAndTypeManager.getDefaultNamespace().getCatalogName()); + checkForNamingConflicts(functionNamespaceManager.get().listFunctions(Optional.empty(), Optional.empty())); + return functions; + } + + protected abstract void checkForNamingConflicts(Collection functions); + + private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) + { + return functionAndTypeManager.getSpecializedFunctionKey(signature, getFunctions(signature.getName())); + } + + private SpecializedFunctionKey getSpecializedFunctionKey(Signature signature) + { + try { + return specializedFunctionKeyCache.getUnchecked(signature); + } + catch (UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), PrestoException.class); + throw e; + } + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java index 27faec86a4256..f3a67d573100a 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java @@ -225,6 +225,7 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.AggregationFunctionImplementation; import com.facebook.presto.spi.function.AlterRoutineCharacteristics; +import com.facebook.presto.spi.function.BuiltInType; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.FunctionNamespaceManager; @@ -292,9 +293,6 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableListMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; import com.google.common.util.concurrent.UncheckedExecutionException; import com.google.errorprone.annotations.ThreadSafe; import io.airlift.slice.Slice; @@ -1132,7 +1130,7 @@ public Collection getFunctions(Optional transactionHandle, Signature signature) { - return new BuiltInFunctionHandle(signature); + return new BuiltInFunctionHandle(signature, BuiltInType.NONE); } @Override @@ -1395,44 +1393,6 @@ private static class EmptyTransactionHandle { } - private static class FunctionMap - { - private final Multimap functions; - - public FunctionMap() - { - functions = ImmutableListMultimap.of(); - } - - public FunctionMap(FunctionMap map, Iterable functions) - { - this.functions = ImmutableListMultimap.builder() - .putAll(map.functions) - .putAll(Multimaps.index(functions, function -> function.getSignature().getName())) - .build(); - - // Make sure all functions with the same name are aggregations or none of them are - for (Map.Entry> entry : this.functions.asMap().entrySet()) { - Collection values = entry.getValue(); - long aggregations = values.stream() - .map(function -> function.getSignature().getKind()) - .filter(kind -> kind == AGGREGATE) - .count(); - checkState(aggregations == 0 || aggregations == values.size(), "'%s' is both an aggregation and a scalar function", entry.getKey()); - } - } - - public List list() - { - return ImmutableList.copyOf(functions.values()); - } - - public Collection get(QualifiedObjectName name) - { - return functions.get(name); - } - } - /** * TypeSignature but has overridden equals(). Here, we compare exact signature of any underlying distinct * types. Some distinct types may have extra information on their lazily loaded parents, and same parent diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/ForNativeFunctionRegistryInfo.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/ForNativeFunctionRegistryInfo.java new file mode 100644 index 0000000000000..93f4cd5b4e040 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/ForNativeFunctionRegistryInfo.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; + +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@BindingAnnotation +public @interface ForNativeFunctionRegistryInfo +{ +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java index 343b4e94d1d9f..c59e4d480f4eb 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.airlift.log.Logger; import com.facebook.presto.Session; import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.Page; @@ -35,9 +36,12 @@ import com.facebook.presto.operator.window.WindowFunctionSupplier; import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.function.AggregationFunctionImplementation; import com.facebook.presto.spi.function.AlterRoutineCharacteristics; +import com.facebook.presto.spi.function.BuiltInType; import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionImplementationType; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.FunctionNamespaceManager; @@ -77,6 +81,8 @@ import org.weakref.jmx.Managed; import org.weakref.jmx.Nested; +import javax.inject.Provider; + import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -97,6 +103,7 @@ import static com.facebook.presto.metadata.FunctionSignatureMatcher.constructFunctionNotFoundErrorMessage; import static com.facebook.presto.metadata.SessionFunctionHandle.SESSION_NAMESPACE; import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; +import static com.facebook.presto.spi.StandardErrorCode.AMBIGUOUS_FUNCTION_CALL; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_MISSING; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; @@ -129,6 +136,7 @@ public class FunctionAndTypeManager implements FunctionMetadataManager, TypeManager { private static final Pattern DEFAULT_NAMESPACE_PREFIX_PATTERN = Pattern.compile("[a-z]+\\.[a-z]+"); + private static final Logger log = Logger.get(FunctionAndTypeManager.class); private final TransactionManager transactionManager; private final BlockEncodingSerde blockEncodingSerde; private final BuiltInTypeAndFunctionNamespaceManager builtInTypeAndFunctionNamespaceManager; @@ -143,9 +151,12 @@ public class FunctionAndTypeManager private final LoadingCache functionCache; private final CacheStatsMBean cacheStatsMBean; private final boolean nativeExecution; + private final boolean isBuiltInSidecarFunctionsEnabled; private final CatalogSchemaName defaultNamespace; private final AtomicReference servingTypeManager; private final AtomicReference>> servingTypeManagerParametricTypesSupplier; + private final BuiltInNativeFunctionNamespaceManager builtInNativeFunctionNamespaceManager; + private final BuiltInPluginFunctionNamespaceManager builtInPluginFunctionNamespaceManager; @Inject public FunctionAndTypeManager( @@ -154,7 +165,8 @@ public FunctionAndTypeManager( FeaturesConfig featuresConfig, FunctionsConfig functionsConfig, HandleResolver handleResolver, - Set types) + Set types, + Provider nativeFunctionRegistryToolProvider) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); @@ -174,9 +186,12 @@ public FunctionAndTypeManager( this.functionSignatureMatcher = new FunctionSignatureMatcher(this); this.typeCoercer = new TypeCoercer(functionsConfig, this); this.nativeExecution = featuresConfig.isNativeExecutionEnabled(); + this.isBuiltInSidecarFunctionsEnabled = featuresConfig.isBuiltInSidecarFunctionsEnabled(); this.defaultNamespace = configureDefaultNamespace(functionsConfig.getDefaultNamespacePrefix()); this.servingTypeManager = new AtomicReference<>(builtInTypeAndFunctionNamespaceManager); this.servingTypeManagerParametricTypesSupplier = new AtomicReference<>(this::getServingTypeManagerParametricTypes); + this.builtInNativeFunctionNamespaceManager = new BuiltInNativeFunctionNamespaceManager(this, nativeFunctionRegistryToolProvider); + this.builtInPluginFunctionNamespaceManager = new BuiltInPluginFunctionNamespaceManager(this); } public static FunctionAndTypeManager createTestFunctionAndTypeManager() @@ -187,7 +202,8 @@ public static FunctionAndTypeManager createTestFunctionAndTypeManager() new FeaturesConfig(), new FunctionsConfig(), new HandleResolver(), - ImmutableSet.of()); + ImmutableSet.of(), + () -> null); } public FunctionAndTypeResolver getFunctionAndTypeResolver() @@ -345,8 +361,16 @@ public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) if (functionHandle.getCatalogSchemaName().equals(SESSION_NAMESPACE)) { return ((SessionFunctionHandle) functionHandle).getFunctionMetadata(); } + if (functionHandle.getBuiltInType().equals(BuiltInType.PLUGIN)) { + return builtInPluginFunctionNamespaceManager.getFunctionMetadata(functionHandle); + } + if (functionHandle.getBuiltInType().equals(BuiltInType.NATIVE)) { + return builtInNativeFunctionNamespaceManager.getFunctionMetadata(functionHandle); + } + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionHandle.getCatalogSchemaName()); checkArgument(functionNamespaceManager.isPresent(), "Cannot find function namespace for '%s'", functionHandle.getCatalogSchemaName()); + return functionNamespaceManager.get().getFunctionMetadata(functionHandle); } @@ -436,6 +460,18 @@ public void registerBuiltInFunctions(List functions) builtInTypeAndFunctionNamespaceManager.registerBuiltInFunctions(functions); } + public void registerNativeFunctions() + { + if (isBuiltInSidecarFunctionsEnabled) { + builtInNativeFunctionNamespaceManager.registerNativeFunctions(); + } + } + + public void registerPluginFunctions(List functions) + { + builtInPluginFunctionNamespaceManager.registerPluginFunctions(functions); + } + /** * likePattern / escape is an opportunistic optimization push down to function namespace managers. * Not all function namespace managers can handle it, thus the returned function list could @@ -453,12 +489,16 @@ public List listFunctions(Session session, Optional likePat functions.addAll(functionNamespaceManagers.get( defaultNamespace.getCatalogName()).listFunctions(likePattern, escape).stream() .collect(toImmutableList())); + functions.addAll(builtInNativeFunctionNamespaceManager.listFunctions().stream().collect(toImmutableList())); + functions.addAll(builtInPluginFunctionNamespaceManager.listFunctions().stream().collect(toImmutableList())); } else { functions.addAll(SessionFunctionUtils.listFunctions(session.getSessionFunctions())); functions.addAll(functionNamespaceManagers.values().stream() .flatMap(manager -> manager.listFunctions(likePattern, escape).stream()) .collect(toImmutableList())); + functions.addAll(builtInNativeFunctionNamespaceManager.listFunctions().stream().collect(toImmutableList())); + functions.addAll(builtInPluginFunctionNamespaceManager.listFunctions().stream().collect(toImmutableList())); } return functions.build().stream() @@ -486,7 +526,7 @@ public Collection getFunctions(Session session, Qualified Optional transactionHandle = session.getTransactionId().map( id -> transactionManager.getFunctionNamespaceTransaction(id, functionName.getCatalogName())); - return functionNamespaceManager.get().getFunctions(transactionHandle, functionName); + return getFunctions(functionName, transactionHandle, functionNamespaceManager.get()); } public void createFunction(SqlInvokedFunction function, boolean replace) @@ -601,6 +641,13 @@ public ScalarFunctionImplementation getScalarFunctionImplementation(FunctionHand if (functionHandle.getCatalogSchemaName().equals(SESSION_NAMESPACE)) { return ((SessionFunctionHandle) functionHandle).getScalarFunctionImplementation(); } + if (functionHandle.getBuiltInType().equals(BuiltInType.PLUGIN)) { + return builtInPluginFunctionNamespaceManager.getScalarFunctionImplementation(functionHandle); + } + if (functionHandle.getBuiltInType().equals(BuiltInType.NATIVE)) { + return builtInNativeFunctionNamespaceManager.getScalarFunctionImplementation(functionHandle); + } + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionHandle.getCatalogSchemaName()); checkArgument(functionNamespaceManager.isPresent(), "Cannot find function namespace for '%s'", functionHandle.getCatalogSchemaName()); return functionNamespaceManager.get().getScalarFunctionImplementation(functionHandle); @@ -709,13 +756,7 @@ public FunctionHandle lookupFunction(QualifiedObjectName functionName, List candidates = functionNamespaceManager.get().getFunctions(Optional.empty(), functionName); - Optional match = functionSignatureMatcher.match(candidates, parameterTypes, false); - if (!match.isPresent()) { - throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, candidates)); - } - - return functionNamespaceManager.get().getFunctionHandle(Optional.empty(), match.get()); + return getMatchingFunctionHandle(functionName, Optional.empty(), functionNamespaceManager.get(), parameterTypes, false); } public FunctionHandle lookupCast(CastType castType, Type fromType, Type toType) @@ -785,11 +826,14 @@ private FunctionHandle resolveFunctionInternal(Optional transacti return functionNamespaceManager.resolveFunction(transactionHandle, functionName, parameterTypes.stream().map(TypeSignatureProvider::getTypeSignature).collect(toImmutableList())); } - Collection candidates = functionNamespaceManager.getFunctions(transactionHandle, functionName); - - Optional match = functionSignatureMatcher.match(candidates, parameterTypes, true); - if (match.isPresent()) { - return functionNamespaceManager.getFunctionHandle(transactionHandle, match.get()); + try { + return getMatchingFunctionHandle(functionName, transactionHandle, functionNamespaceManager, parameterTypes, true); + } + catch (PrestoException e) { + // Could still match to a magic literal function + if (e.getErrorCode().getCode() != StandardErrorCode.FUNCTION_NOT_FOUND.toErrorCode().getCode()) { + throw e; + } } if (functionName.getObjectName().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { @@ -802,10 +846,11 @@ private FunctionHandle resolveFunctionInternal(Optional transacti // verify we have one parameter of the proper type checkArgument(parameterTypes.size() == 1, "Expected one argument to literal function, but got %s", parameterTypes); - return new BuiltInFunctionHandle(getMagicLiteralFunctionSignature(type)); + return new BuiltInFunctionHandle(getMagicLiteralFunctionSignature(type), BuiltInType.NONE); } - throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, candidates)); + throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage( + functionName, parameterTypes, getFunctions(functionName, transactionHandle, functionNamespaceManager))); } private FunctionHandle resolveBuiltInFunction(QualifiedObjectName functionName, List parameterTypes) @@ -829,7 +874,7 @@ private FunctionHandle lookupCachedFunction(QualifiedObjectName functionName, Li } } - private Optional> getServingFunctionNamespaceManager(CatalogSchemaName functionNamespace) + public Optional> getServingFunctionNamespaceManager(CatalogSchemaName functionNamespace) { return Optional.ofNullable(functionNamespaceManagers.get(functionNamespace.getCatalogName())); } @@ -840,7 +885,6 @@ private Optional> getServingFunc } @Override - @SuppressWarnings("unchecked") public SpecializedFunctionKey getSpecializedFunctionKey(Signature signature) { QualifiedObjectName functionName = signature.getName(); @@ -849,8 +893,13 @@ public SpecializedFunctionKey getSpecializedFunctionKey(Signature signature) throw new PrestoException(FUNCTION_NOT_FOUND, format("Cannot find function namespace for signature '%s'", functionName)); } - Collection candidates = (Collection) functionNamespaceManager.get().getFunctions(Optional.empty(), functionName); + Collection candidates = functionNamespaceManager.get().getFunctions(Optional.empty(), functionName); + return getSpecializedFunctionKey(signature, candidates); + } + + public SpecializedFunctionKey getSpecializedFunctionKey(Signature signature, Collection candidates) + { // search for exact match Type returnType = getType(signature.getReturnType()); List argumentTypeSignatureProviders = fromTypeSignatures(signature.getArgumentTypes()); @@ -914,6 +963,80 @@ private Map getServingTypeManagerParametricTypes() .collect(toImmutableMap(ParametricType::getName, parametricType -> parametricType)); } + private Collection getFunctions( + QualifiedObjectName functionName, + Optional transactionHandle, + FunctionNamespaceManager functionNamespaceManager) + { + return ImmutableList.builder() + .addAll(functionNamespaceManager.getFunctions(transactionHandle, functionName)) + .addAll(builtInNativeFunctionNamespaceManager.getFunctions(functionName)) + .addAll(builtInPluginFunctionNamespaceManager.getFunctions(functionName)) + .build(); + } + + /** + * Gets the function handle of the function if there is a match. We enforce explicit naming for dynamic function namespaces. + * All unqualified function names will only be resolved against the built-in default function namespace. We get all the candidates + * from the current default namespace and additionally all the candidates from builtInNativeFunctionNamespaceManager. + * + * @throws PrestoException if there are no matches or multiple matches + */ + private FunctionHandle getMatchingFunctionHandle( + QualifiedObjectName functionName, + Optional transactionHandle, + FunctionNamespaceManager functionNamespaceManager, + List parameterTypes, + boolean coercionAllowed) + { + Optional matchingDefaultFunctionSignature = + getMatchingFunction(functionNamespaceManager.getFunctions(transactionHandle, functionName), parameterTypes, coercionAllowed); + Optional matchingPluginFunctionSignature = + getMatchingFunction(builtInPluginFunctionNamespaceManager.getFunctions(functionName), parameterTypes, coercionAllowed); + Optional matchingNativeFunctionSignature = + getMatchingFunction(builtInNativeFunctionNamespaceManager.getFunctions(functionName), parameterTypes, coercionAllowed); + + if (matchingDefaultFunctionSignature.isPresent() && matchingPluginFunctionSignature.isPresent()) { + throw new PrestoException(AMBIGUOUS_FUNCTION_CALL, format("Function '%s' has two matching signatures. Please specify parameter types. \n" + + "First match : '%s', Second match: '%s'", functionName, matchingDefaultFunctionSignature.get(), matchingPluginFunctionSignature.get())); + } + + if (matchingDefaultFunctionSignature.isPresent() && matchingNativeFunctionSignature.isPresent()) { + FunctionHandle defaultFunctionHandle = functionNamespaceManager.getFunctionHandle(transactionHandle, matchingDefaultFunctionSignature.get()); + FunctionHandle nativeFunctionHandle = builtInNativeFunctionNamespaceManager.getFunctionHandle(matchingNativeFunctionSignature.get()); + + if (functionNamespaceManager.getFunctionMetadata(defaultFunctionHandle).getImplementationType().equals(FunctionImplementationType.JAVA)) { + return defaultFunctionHandle; + } + if (functionNamespaceManager.getFunctionMetadata(defaultFunctionHandle).getImplementationType().equals(FunctionImplementationType.SQL)) { + return nativeFunctionHandle; + } + } + + if (matchingDefaultFunctionSignature.isPresent()) { + return functionNamespaceManager.getFunctionHandle(transactionHandle, matchingDefaultFunctionSignature.get()); + } + + if (matchingPluginFunctionSignature.isPresent()) { + return builtInPluginFunctionNamespaceManager.getFunctionHandle(matchingPluginFunctionSignature.get()); + } + + if (matchingNativeFunctionSignature.isPresent()) { + return builtInNativeFunctionNamespaceManager.getFunctionHandle(matchingNativeFunctionSignature.get()); + } + + throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, + getFunctions(functionName, transactionHandle, functionNamespaceManager))); + } + + private Optional getMatchingFunction( + Collection candidates, + List parameterTypes, + boolean coercionAllowed) + { + return functionSignatureMatcher.match(candidates, parameterTypes, coercionAllowed); + } + private static class FunctionResolutionCacheKey { private final QualifiedObjectName functionName; diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java index 9521027f30812..7dc7ee5194249 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.operator.scalar.annotations.CodegenScalarFromAnnotationsParser; import com.facebook.presto.operator.scalar.annotations.ScalarFromAnnotationsParser; import com.facebook.presto.operator.scalar.annotations.SqlInvokedScalarFromAnnotationsParser; @@ -28,6 +29,7 @@ import java.util.Collection; import java.util.List; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -44,6 +46,11 @@ public static List extractFunctions(Collection> classes) } public static List extractFunctions(Class clazz) + { + return extractFunctions(clazz, JAVA_BUILTIN_NAMESPACE); + } + + public static List extractFunctions(Class clazz, CatalogSchemaName defaultNamespace) { if (WindowFunction.class.isAssignableFrom(clazz)) { @SuppressWarnings("unchecked") @@ -61,12 +68,12 @@ public static List extractFunctions(Class clazz) } if (clazz.isAnnotationPresent(SqlInvokedScalarFunction.class)) { - return SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinition(clazz); + return SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinition(clazz, defaultNamespace); } List scalarFunctions = ImmutableList.builder() .addAll(ScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)) - .addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)) + .addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, defaultNamespace)) .addAll(CodegenScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)) .build(); checkArgument(!scalarFunctions.isEmpty(), "Class [%s] does not define any scalar functions", clazz.getName()); diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java index 13a90da951278..f3dab976902a9 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.List; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static java.util.Objects.requireNonNull; public class FunctionListBuilder @@ -62,13 +63,13 @@ public FunctionListBuilder scalars(Class clazz) public FunctionListBuilder sqlInvokedScalar(Class clazz) { - functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinition(clazz)); + functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinition(clazz, JAVA_BUILTIN_NAMESPACE)); return this; } public FunctionListBuilder sqlInvokedScalars(Class clazz) { - functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)); + functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, JAVA_BUILTIN_NAMESPACE)); return this; } diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionMap.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionMap.java new file mode 100644 index 0000000000000..e0cdf7c6269e4 --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionMap.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.function.SqlFunction; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; +import static com.google.common.base.Preconditions.checkState; + +public class FunctionMap +{ + private final Multimap functions; + + public FunctionMap() + { + functions = ImmutableListMultimap.of(); + } + + public FunctionMap(FunctionMap map, Iterable functions) + { + this.functions = ImmutableListMultimap.builder() + .putAll(map.functions) + .putAll(Multimaps.index(functions, function -> function.getSignature().getName())) + .build(); + + // Make sure all functions with the same name are aggregations or none of them are + for (Map.Entry> entry : this.functions.asMap().entrySet()) { + Collection values = entry.getValue(); + long aggregations = values.stream() + .map(function -> function.getSignature().getKind()) + .filter(kind -> kind == AGGREGATE) + .count(); + checkState(aggregations == 0 || aggregations == values.size(), "'%s' is both an aggregation and a scalar function", entry.getKey()); + } + } + + public List list() + { + return ImmutableList.copyOf(functions.values()); + } + + public Collection get(QualifiedObjectName name) + { + return functions.get(name); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java index b6a3a644a7f50..ac83dcd2d26ae 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -243,7 +243,7 @@ public static MetadataManager createTestMetadataManager(TransactionManager trans { BlockEncodingManager blockEncodingManager = new BlockEncodingManager(); return new MetadataManager( - new FunctionAndTypeManager(transactionManager, blockEncodingManager, featuresConfig, functionsConfig, new HandleResolver(), ImmutableSet.of()), + new FunctionAndTypeManager(transactionManager, blockEncodingManager, featuresConfig, functionsConfig, new HandleResolver(), ImmutableSet.of(), () -> null), blockEncodingManager, createTestingSessionPropertyManager(), new SchemaPropertyManager(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/NativeFunctionRegistryTool.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/NativeFunctionRegistryTool.java new file mode 100644 index 0000000000000..b0b243bea165f --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/NativeFunctionRegistryTool.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.metadata; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.JsonResponseHandler; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; +import com.facebook.presto.spi.Node; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + +import java.net.URI; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_ARGUMENTS; +import static java.util.Objects.requireNonNull; + +public class NativeFunctionRegistryTool +{ + private static final int MAX_RETRIES = 8; + private static final long RETRY_DELAY_MS = Duration.ofMinutes(1).toMillis(); + private static final Logger log = Logger.get(NativeFunctionRegistryTool.class); + private final JsonCodec>> nativeFunctionSignatureMapJsonCodec; + private final NodeManager nodeManager; + private final HttpClient httpClient; + private static final String FUNCTION_SIGNATURES_ENDPOINT = "/v1/functions"; + + @Inject + public NativeFunctionRegistryTool( + @ForNativeFunctionRegistryInfo HttpClient httpClient, + JsonCodec>> nativeFunctionSignatureMapJsonCodec, + NodeManager nodeManager) + { + this.nativeFunctionSignatureMapJsonCodec = + requireNonNull(nativeFunctionSignatureMapJsonCodec, "nativeFunctionSignatureMapJsonCodec is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.httpClient = requireNonNull(httpClient, "typeManager is null"); + } + + public UdfFunctionSignatureMap getNativeFunctionSignatureMap() + { + try { + Request request = Request.Builder.prepareGet().setUri(getSidecarLocationOnStartup()).build(); + Map> nativeFunctionSignatureMap = httpClient.execute(request, JsonResponseHandler.createJsonResponseHandler(nativeFunctionSignatureMapJsonCodec)); + return new UdfFunctionSignatureMap(ImmutableMap.copyOf(nativeFunctionSignatureMap)); + } + catch (Exception e) { + throw new PrestoException(INVALID_ARGUMENTS, "Failed to get functions from sidecar.", e); + } + } + + private URI getSidecarLocationOnStartup() + { + Node sidecarNode = null; + int retryCount = 0; + while (sidecarNode == null && retryCount < MAX_RETRIES) { + try { + sidecarNode = nodeManager.getSidecarNode(); + break; + } + catch (Exception e) { + retryCount++; + log.error("Attempt Error getting sidecar node (attempt " + retryCount + "): " + e.getMessage()); + if (retryCount < MAX_RETRIES) { + try { + Thread.sleep(RETRY_DELAY_MS); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Retry interrupted", ie); + } + } + else { + throw new RuntimeException("Failed to get sidecar node", e); + } + } + } + + return HttpUriBuilder + .uriBuilderFrom(sidecarNode.getHttpUri()) + .appendPath(FUNCTION_SIGNATURES_ENDPOINT) + .build(); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/NativeSidecarFunctionUtil.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/NativeSidecarFunctionUtil.java new file mode 100644 index 0000000000000..b3747fc305bad --- /dev/null +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/NativeSidecarFunctionUtil.java @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.metadata; + +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.NamedTypeSignature; +import com.facebook.presto.common.type.StandardTypes; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.common.type.TypeSignatureParameter; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.spi.function.AggregationFunctionMetadata; +import com.facebook.presto.spi.function.LongVariableConstraint; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.RoutineCharacteristics; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.function.TypeVariableConstraint; +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; + +public class NativeSidecarFunctionUtil +{ + private NativeSidecarFunctionUtil() {} + + public static synchronized SqlInvokedFunction createSqlInvokedFunction(String functionName, JsonBasedUdfFunctionMetadata jsonBasedUdfFunctionMetaData) + { + checkState(jsonBasedUdfFunctionMetaData.getRoutineCharacteristics().getLanguage().equals(RoutineCharacteristics.Language.CPP), "NativeFunctionNamespaceManager only supports CPP UDF"); + QualifiedObjectName qualifiedFunctionName = QualifiedObjectName.valueOf(new CatalogSchemaName("presto", jsonBasedUdfFunctionMetaData.getSchema()), functionName); + List parameterNameList = jsonBasedUdfFunctionMetaData.getParamNames(); + List parameterTypeList = convertApplicableTypeToVariable(jsonBasedUdfFunctionMetaData.getParamTypes()); + List typeVariableConstraintsList = jsonBasedUdfFunctionMetaData.getTypeVariableConstraints().isPresent() ? + jsonBasedUdfFunctionMetaData.getTypeVariableConstraints().get() : Collections.emptyList(); + List longVariableConstraintList = jsonBasedUdfFunctionMetaData.getLongVariableConstraints().isPresent() ? + jsonBasedUdfFunctionMetaData.getLongVariableConstraints().get() : Collections.emptyList(); + + TypeSignature outputType = convertApplicableTypeToVariable(jsonBasedUdfFunctionMetaData.getOutputType()); + ImmutableList.Builder parameterBuilder = ImmutableList.builder(); + for (int i = 0; i < parameterNameList.size(); i++) { + parameterBuilder.add(new Parameter(parameterNameList.get(i), parameterTypeList.get(i))); + } + + Optional aggregationFunctionMetadata = + jsonBasedUdfFunctionMetaData.getAggregateMetadata() + .map(metadata -> new AggregationFunctionMetadata( + convertApplicableTypeToVariable(metadata.getIntermediateType()), + metadata.isOrderSensitive())); + + return new SqlInvokedFunction( + qualifiedFunctionName, + parameterBuilder.build(), + typeVariableConstraintsList, + longVariableConstraintList, + outputType, + jsonBasedUdfFunctionMetaData.getDocString(), + jsonBasedUdfFunctionMetaData.getRoutineCharacteristics(), + "", + jsonBasedUdfFunctionMetaData.getVariableArity(), + notVersioned(), + jsonBasedUdfFunctionMetaData.getFunctionKind(), + aggregationFunctionMetadata); + } + + public static List convertApplicableTypeToVariable(List typeSignatures) + { + List newTypeSignaturesList = new ArrayList<>(); + for (TypeSignature typeSignature : typeSignatures) { + if (!typeSignature.getParameters().isEmpty()) { + TypeSignature newTypeSignature = + new TypeSignature( + typeSignature.getBase(), + getTypeSignatureParameters( + typeSignature, + typeSignature.getParameters())); + newTypeSignaturesList.add(newTypeSignature); + } + else { + newTypeSignaturesList.add(typeSignature); + } + } + return newTypeSignaturesList; + } + + public static TypeSignature convertApplicableTypeToVariable(TypeSignature typeSignature) + { + List typeSignaturesList = convertApplicableTypeToVariable(ImmutableList.of(typeSignature)); + checkArgument(!typeSignaturesList.isEmpty(), "Type signature list is empty for : " + typeSignature); + return typeSignaturesList.get(0); + } + + private static List getTypeSignatureParameters( + TypeSignature typeSignature, + List typeSignatureParameterList) + { + List newParameterTypeList = new ArrayList<>(); + for (TypeSignatureParameter parameter : typeSignatureParameterList) { + if (parameter.isLongLiteral()) { + newParameterTypeList.add(parameter); + continue; + } + + boolean isNamedTypeSignature = parameter.isNamedTypeSignature(); + TypeSignature parameterTypeSignature; + // If it's a named type signatures only in the case of row signature types. + if (isNamedTypeSignature) { + parameterTypeSignature = parameter.getNamedTypeSignature().getTypeSignature(); + } + else { + parameterTypeSignature = parameter.getTypeSignature(); + } + + if (parameterTypeSignature.getParameters().isEmpty()) { + boolean changeTypeToVariable = isDecimalTypeBase(typeSignature.getBase()); + if (changeTypeToVariable) { + newParameterTypeList.add( + TypeSignatureParameter.of(parameterTypeSignature.getBase())); + } + else { + if (isNamedTypeSignature) { + newParameterTypeList.add(TypeSignatureParameter.of(parameter.getNamedTypeSignature())); + } + else { + newParameterTypeList.add(TypeSignatureParameter.of(parameterTypeSignature)); + } + } + } + else { + TypeSignature newTypeSignature = + new TypeSignature( + parameterTypeSignature.getBase(), + getTypeSignatureParameters( + parameterTypeSignature.getStandardTypeSignature(), + parameterTypeSignature.getParameters())); + if (isNamedTypeSignature) { + newParameterTypeList.add( + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + newTypeSignature))); + } + else { + newParameterTypeList.add(TypeSignatureParameter.of(newTypeSignature)); + } + } + } + return newParameterTypeList; + } + + private static boolean isDecimalTypeBase(String typeBase) + { + return typeBase.equals(StandardTypes.DECIMAL); + } +} diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java index e34ede50a57f3..1c5f621cb8d05 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar.annotations; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.PrestoException; @@ -39,7 +40,6 @@ import java.util.stream.Stream; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.findPublicStaticMethods; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; @@ -60,7 +60,7 @@ public final class SqlInvokedScalarFromAnnotationsParser { private SqlInvokedScalarFromAnnotationsParser() {} - public static List parseFunctionDefinition(Class clazz) + public static List parseFunctionDefinition(Class clazz, CatalogSchemaName defaultNamespace) { checkArgument(clazz.isAnnotationPresent(SqlInvokedScalarFunction.class), "Class is not annotated with SqlInvokedScalarFunction: %s", clazz.getName()); @@ -68,15 +68,15 @@ public static List parseFunctionDefinition(Class clazz) Optional description = Optional.ofNullable(clazz.getAnnotation(Description.class)).map(Description::value); return findScalarsInFunctionDefinitionClass(clazz).stream() - .map(method -> createSqlInvokedFunctions(method, Optional.of(header), description)) + .map(method -> createSqlInvokedFunctions(method, Optional.of(header), description, defaultNamespace)) .flatMap(List::stream) .collect(toImmutableList()); } - public static List parseFunctionDefinitions(Class clazz) + public static List parseFunctionDefinitions(Class clazz, CatalogSchemaName defaultNamespace) { return findScalarsInFunctionSetClass(clazz).stream() - .map(method -> createSqlInvokedFunctions(method, Optional.empty(), Optional.empty())) + .map(method -> createSqlInvokedFunctions(method, Optional.empty(), Optional.empty(), defaultNamespace)) .flatMap(List::stream) .collect(toImmutableList()); } @@ -121,7 +121,7 @@ private static List findScalarsInFunctionSetClass(Class clazz) return ImmutableList.copyOf(methods); } - private static List createSqlInvokedFunctions(Method method, Optional header, Optional description) + private static List createSqlInvokedFunctions(Method method, Optional header, Optional description, CatalogSchemaName defaultNamespace) { SqlInvokedScalarFunction functionHeader = header.orElseGet(() -> method.getAnnotation(SqlInvokedScalarFunction.class)); String functionDescription = description.orElseGet(() -> method.isAnnotationPresent(Description.class) ? method.getAnnotation(Description.class).value() : ""); @@ -167,7 +167,7 @@ else if (method.isAnnotationPresent(SqlParameters.class)) { return Stream.concat(Stream.of(functionHeader.value()), stream(functionHeader.alias())) .map(name -> new SqlInvokedFunction( - QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name), + QualifiedObjectName.valueOf(defaultNamespace, name), parameters, typeVariableConstraints, emptyList(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/server/PluginManager.java b/presto-main-base/src/main/java/com/facebook/presto/server/PluginManager.java index 3dfe5202d05e6..d072762d14dbc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/server/PluginManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/server/PluginManager.java @@ -306,6 +306,12 @@ public void installPlugin(Plugin plugin) log.info("Registering client request filter factory"); clientRequestFilterManager.registerClientRequestFilterFactory(clientRequestFilterFactory); } + + for (Class functionClass : plugin.getSqlInvokedFunctions()) { + log.info("Registering functions from %s", functionClass.getName()); + metadata.getFunctionAndTypeManager().registerPluginFunctions( + extractFunctions(functionClass, metadata.getFunctionAndTypeManager().getDefaultNamespace())); + } } public void installCoordinatorPlugin(CoordinatorPlugin plugin) diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index a6749bf1a7e18..b94b06946aa0b 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -236,6 +236,7 @@ public class FeaturesConfig private boolean pushRemoteExchangeThroughGroupId; private boolean isOptimizeMultipleApproxPercentileOnSameFieldEnabled = true; private boolean nativeExecutionEnabled; + private boolean builtInSidecarFunctionsEnabled; private boolean disableTimeStampWithTimeZoneForNative; private boolean disableIPAddressForNative; private String nativeExecutionExecutablePath = "./presto_server"; @@ -2295,6 +2296,19 @@ public boolean isNativeExecutionEnabled() return this.nativeExecutionEnabled; } + @Config("built-in-sidecar-functions-enabled") + @ConfigDescription("Enable using CPP functions from sidecar over coordinator SQL implementations.") + public FeaturesConfig setBuiltInSidecarFunctionsEnabled(boolean builtInSidecarFunctionsEnabled) + { + this.builtInSidecarFunctionsEnabled = builtInSidecarFunctionsEnabled; + return this; + } + + public boolean isBuiltInSidecarFunctionsEnabled() + { + return this.builtInSidecarFunctionsEnabled; + } + @Config("disable-timestamp-with-timezone-for-native-execution") @ConfigDescription("Disable timestamp with timezone type on native engine") public FeaturesConfig setDisableTimeStampWithTimeZoneForNative(boolean disableTimeStampWithTimeZoneForNative) @@ -2970,6 +2984,7 @@ public boolean isInEqualityJoinPushdownEnabled() { return inEqualityJoinPushdownEnabled; } + public boolean isPrestoSparkExecutionEnvironment() { return prestoSparkExecutionEnvironment; diff --git a/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index d3c7044fc6df1..8d16a8176d6f3 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main-base/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -433,7 +433,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, featuresConfig.setIgnoreStatsCalculatorFailures(false); this.metadata = new MetadataManager( - new FunctionAndTypeManager(transactionManager, blockEncodingManager, featuresConfig, functionsConfig, new HandleResolver(), ImmutableSet.of()), + new FunctionAndTypeManager(transactionManager, blockEncodingManager, featuresConfig, functionsConfig, new HandleResolver(), ImmutableSet.of(), () -> null), blockEncodingManager, createTestingSessionPropertyManager( new SystemSessionProperties( diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestConvertApplicableTypeToVariable.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java similarity index 98% rename from presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestConvertApplicableTypeToVariable.java rename to presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java index 93f9ee75f2b82..5b153c666b0f9 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestConvertApplicableTypeToVariable.java +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestConvertApplicableTypeToVariable.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sidecar; +package com.facebook.presto.metadata; import com.facebook.presto.common.type.NamedTypeSignature; import com.facebook.presto.common.type.TypeSignature; @@ -22,7 +22,7 @@ import java.util.Optional; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManager.convertApplicableTypeToVariable; +import static com.facebook.presto.metadata.NativeSidecarFunctionUtil.convertApplicableTypeToVariable; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/TestFunctionAndTypeManager.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestFunctionAndTypeManager.java index ece53b2e9d8d7..acce39a173954 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/metadata/TestFunctionAndTypeManager.java +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/TestFunctionAndTypeManager.java @@ -20,6 +20,7 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.scalar.CustomFunctions; +import com.facebook.presto.spi.function.BuiltInType; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.Parameter; import com.facebook.presto.spi.function.RoutineCharacteristics; @@ -83,7 +84,7 @@ public void testIdentityCast() { FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager(); FunctionHandle exactOperator = functionAndTypeManager.lookupCast(CastType.CAST, HYPER_LOG_LOG, HYPER_LOG_LOG); - assertEquals(exactOperator, new BuiltInFunctionHandle(new Signature(CAST.getFunctionName(), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()))); + assertEquals(exactOperator, new BuiltInFunctionHandle(new Signature(CAST.getFunctionName(), SCALAR, HYPER_LOG_LOG.getTypeSignature(), HYPER_LOG_LOG.getTypeSignature()), BuiltInType.NONE)); } @Test @@ -444,7 +445,7 @@ public ResolveFunctionAssertion forParameters(String... parameters) public ResolveFunctionAssertion returns(SignatureBuilder functionSignature) { - FunctionHandle expectedFunction = new BuiltInFunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build()); + FunctionHandle expectedFunction = new BuiltInFunctionHandle(functionSignature.name(TEST_FUNCTION_NAME).build(), BuiltInType.NONE); FunctionHandle actualFunction = resolveFunctionHandle(); assertEquals(expectedFunction, actualFunction); return this; diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java b/presto-main-base/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java index 053635444a321..6ed8a5dacb20b 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java @@ -51,7 +51,7 @@ public void testParseFunctionDefinition() new ArrayType(BIGINT).getTypeSignature(), ImmutableList.of(INTEGER.getTypeSignature())); - List functions = SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(SingleImplementationSQLInvokedScalarFunction.class); + List functions = SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(SingleImplementationSQLInvokedScalarFunction.class, JAVA_BUILTIN_NAMESPACE); assertEquals(functions.size(), 1); SqlInvokedFunction f = functions.get(0); @@ -75,7 +75,7 @@ public void testParseFunctionDefinitionWithTypeParameter() ImmutableList.of(new TypeSignature("T")), false); - List functions = SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(SingleImplementationSQLInvokedScalarFunctionWithTypeParameter.class); + List functions = SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(SingleImplementationSQLInvokedScalarFunctionWithTypeParameter.class, JAVA_BUILTIN_NAMESPACE); assertEquals(functions.size(), 1); SqlInvokedFunction f = functions.get(0); diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestCustomFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestCustomFunctions.java index be335dd54c5c9..6d0b4f0f65052 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestCustomFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestCustomFunctions.java @@ -24,6 +24,7 @@ import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; public class TestCustomFunctions extends AbstractTestFunctions @@ -41,7 +42,7 @@ protected TestCustomFunctions(FeaturesConfig config) public void setupClass() { registerScalar(CustomFunctions.class); - List functions = SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(CustomFunctions.class); + List functions = SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(CustomFunctions.class, JAVA_BUILTIN_NAMESPACE); this.functionAssertions.addFunctions(functions); } diff --git a/presto-main-base/src/test/java/com/facebook/presto/type/AbstractTestType.java b/presto-main-base/src/test/java/com/facebook/presto/type/AbstractTestType.java index dfc25091c6ae0..fa1e22df05337 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/type/AbstractTestType.java +++ b/presto-main-base/src/test/java/com/facebook/presto/type/AbstractTestType.java @@ -71,7 +71,8 @@ public abstract class AbstractTestType new FeaturesConfig(), new FunctionsConfig(), new HandleResolver(), - ImmutableSet.of()); + ImmutableSet.of(), + () -> null); private final Class objectValueType; private final Block testBlock; diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 277a3126aab40..761267ec1b62d 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -171,10 +171,6 @@ com.facebook.airlift.drift drift-api - - com.facebook.airlift - http-client - joda-time joda-time @@ -231,6 +227,18 @@ com.facebook.airlift http-server + + + com.facebook.airlift + http-client + + + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + + com.facebook.airlift event diff --git a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java index 552b6da5d812e..6cccf7c3781a6 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/PrestoServer.java @@ -43,6 +43,7 @@ import com.facebook.presto.metadata.Catalog; import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.DiscoveryNodeManager; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.metadata.StaticCatalogStore; @@ -197,6 +198,7 @@ public void run() NodeInfo nodeInfo = injector.getInstance(NodeInfo.class); PluginNodeManager pluginNodeManager = new PluginNodeManager(nodeManager, nodeInfo.getEnvironment()); planCheckerProviderManager.loadPlanCheckerProviders(pluginNodeManager); + injector.getInstance(FunctionAndTypeManager.class).registerNativeFunctions(); injector.getInstance(ClientRequestFilterManager.class).loadClientRequestFilters(); injector.getInstance(ExpressionOptimizerManager.class).loadExpressionOptimizerFactories(); diff --git a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java index fa85aaeba1b2a..293b6a205f2e9 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java +++ b/presto-main/src/main/java/com/facebook/presto/server/ServerMainModule.java @@ -17,6 +17,8 @@ import com.facebook.airlift.configuration.AbstractConfigurationAwareModule; import com.facebook.airlift.discovery.client.ServiceAnnouncement; import com.facebook.airlift.http.server.TheServlet; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; import com.facebook.airlift.json.JsonObjectMapperProvider; import com.facebook.airlift.stats.GcMonitor; import com.facebook.airlift.stats.JmxGcMonitor; @@ -79,6 +81,7 @@ import com.facebook.presto.execution.scheduler.TableWriteInfo; import com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats; import com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; import com.facebook.presto.index.IndexManager; import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.memory.LocalMemoryManagerExporter; @@ -92,12 +95,14 @@ import com.facebook.presto.metadata.CatalogManager; import com.facebook.presto.metadata.ColumnPropertyManager; import com.facebook.presto.metadata.DiscoveryNodeManager; +import com.facebook.presto.metadata.ForNativeFunctionRegistryInfo; import com.facebook.presto.metadata.ForNodeManager; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.HandleJsonModule; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.metadata.NativeFunctionRegistryTool; import com.facebook.presto.metadata.SchemaPropertyManager; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.metadata.SessionPropertyProviderConfig; @@ -242,6 +247,7 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; +import com.google.inject.TypeLiteral; import com.google.inject.multibindings.MapBinder; import io.airlift.slice.Slice; import jakarta.annotation.PreDestroy; @@ -250,6 +256,7 @@ import jakarta.servlet.Servlet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; @@ -265,6 +272,7 @@ import static com.facebook.airlift.http.client.HttpClientBinder.httpClientBinder; import static com.facebook.airlift.jaxrs.JaxrsBinder.jaxrsBinder; import static com.facebook.airlift.json.JsonBinder.jsonBinder; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; import static com.facebook.airlift.json.JsonCodecBinder.jsonCodecBinder; import static com.facebook.airlift.json.smile.SmileCodecBinder.smileCodecBinder; import static com.facebook.airlift.units.DataSize.Unit.MEGABYTE; @@ -393,6 +401,11 @@ else if (serverConfig.isCoordinator()) { .withAddressSelector(((addressSelectorBinder, annotation, prefix) -> addressSelectorBinder.bind(AddressSelector.class).annotatedWith(annotation).to(FixedAddressSelector.class))); + binder.bind(NativeFunctionRegistryTool.class).in(Scopes.SINGLETON); + binder.bind(new TypeLiteral>>>() {}) + .toInstance(new JsonCodecFactory().mapJsonCodec(String.class, listJsonCodec(JsonBasedUdfFunctionMetadata.class))); + httpClientBinder(binder).bindHttpClient("native-function-registry", ForNativeFunctionRegistryInfo.class); + // node scheduler // TODO: remove from NodePartitioningManager and move to CoordinatorModule configBinder(binder).bindConfig(NodeSchedulerConfig.class); diff --git a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java index 1cb04591b15da..f3bbab96410c1 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java +++ b/presto-main/src/main/java/com/facebook/presto/server/testing/TestingPrestoServer.java @@ -51,6 +51,7 @@ import com.facebook.presto.memory.LocalMemoryManager; import com.facebook.presto.metadata.AllNodes; import com.facebook.presto.metadata.CatalogManager; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.InternalNode; import com.facebook.presto.metadata.InternalNodeManager; import com.facebook.presto.metadata.Metadata; @@ -147,6 +148,7 @@ public class TestingPrestoServer private final boolean preserveData; private final LifeCycleManager lifeCycleManager; private final PluginManager pluginManager; + private final FunctionAndTypeManager functionAndTypeManager; private final ConnectorManager connectorManager; private final TestingHttpServer server; private final CatalogManager catalogManager; @@ -367,6 +369,8 @@ public TestingPrestoServer( connectorManager = injector.getInstance(ConnectorManager.class); + functionAndTypeManager = injector.getInstance(FunctionAndTypeManager.class); + server = injector.getInstance(TestingHttpServer.class); catalogManager = injector.getInstance(CatalogManager.class); transactionManager = injector.getInstance(TransactionManager.class); @@ -501,6 +505,11 @@ private Map getServerProperties( return ImmutableMap.copyOf(serverProperties); } + public void registerNativeFunctions() + { + functionAndTypeManager.registerNativeFunctions(); + } + @Override public void close() throws IOException diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index ab6d81aba382b..d94941c6bf2c3 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -36,6 +36,16 @@ test + + org.jetbrains + annotations + + + + org.weakref + jmxutils + + io.airlift.tpch tpch @@ -232,6 +242,11 @@ test + + com.facebook.presto + presto-parser + + com.facebook.presto presto-spark-base diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index bc3049b63ff59..f507ab616ce77 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -123,6 +123,7 @@ public static class HiveQueryRunnerBuilder private String security; private boolean addStorageFormatToPath; private boolean coordinatorSidecarEnabled; + private boolean workerSidecarEnabled; private boolean enableRuntimeMetricsCollection; private boolean enableSsdCache; private boolean failOnNestedLoopJoin; @@ -218,6 +219,12 @@ public HiveQueryRunnerBuilder setCoordinatorSidecarEnabled(boolean coordinatorSi return this; } + public HiveQueryRunnerBuilder setWorkerSidecarEnabled(boolean workerSidecarEnabled) + { + this.workerSidecarEnabled = workerSidecarEnabled; + return this; + } + public HiveQueryRunnerBuilder setStorageFormat(String storageFormat) { this.storageFormat = storageFormat; @@ -261,7 +268,7 @@ public QueryRunner build() Optional> externalWorkerLauncher = Optional.empty(); if (this.useExternalWorkerLauncher) { externalWorkerLauncher = getExternalWorkerLauncher("hive", serverBinary, cacheMaxSize, remoteFunctionServerUds, - failOnNestedLoopJoin, coordinatorSidecarEnabled, enableRuntimeMetricsCollection, enableSsdCache); + failOnNestedLoopJoin, coordinatorSidecarEnabled, workerSidecarEnabled, enableRuntimeMetricsCollection, enableSsdCache); } return HiveQueryRunner.createQueryRunner( ImmutableList.of(), @@ -350,7 +357,7 @@ public QueryRunner build() Optional> externalWorkerLauncher = Optional.empty(); if (this.useExternalWorkerLauncher) { externalWorkerLauncher = getExternalWorkerLauncher("iceberg", serverBinary, cacheMaxSize, remoteFunctionServerUds, - false, false, false, false); + false, false, false, false, false); } return IcebergQueryRunner.builder() .setExtraProperties(extraProperties) @@ -446,6 +453,7 @@ public static Optional> getExternalWorkerLaunc Optional remoteFunctionServerUds, Boolean failOnNestedLoopJoin, boolean isCoordinatorSidecarEnabled, + boolean isWorkerSidecarEnabled, boolean enableRuntimeMetricsCollection, boolean enableSsdCache) { @@ -468,6 +476,10 @@ public static Optional> getExternalWorkerLaunc "native-sidecar=true%n" + "presto.default-namespace=native.default%n", configProperties); } + else if (isWorkerSidecarEnabled) { + configProperties = format("%s%n" + + "native-sidecar=true%n", configProperties); + } if (enableRuntimeMetricsCollection) { configProperties = format("%s%n" + diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestBuiltInNativeFunctions.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestBuiltInNativeFunctions.java new file mode 100644 index 0000000000000..bcf1fc1f8aa34 --- /dev/null +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/TestBuiltInNativeFunctions.java @@ -0,0 +1,178 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.nativeworker; + +import com.facebook.presto.cost.CostCalculator; +import com.facebook.presto.cost.CostCalculatorUsingExchanges; +import com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges; +import com.facebook.presto.cost.CostComparator; +import com.facebook.presto.cost.TaskCountEstimator; +import com.facebook.presto.execution.QueryManagerConfig; +import com.facebook.presto.execution.TaskManagerConfig; +import com.facebook.presto.metadata.InMemoryNodeManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.nodeManager.PluginNodeManager; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.sql.analyzer.FeaturesConfig; +import com.facebook.presto.sql.analyzer.QueryExplainer; +import com.facebook.presto.sql.expressions.ExpressionOptimizerManager; +import com.facebook.presto.sql.planner.PartitioningProviderManager; +import com.facebook.presto.sql.planner.PlanFragmenter; +import com.facebook.presto.sql.planner.PlanOptimizers; +import com.facebook.presto.sql.planner.optimizations.PlanOptimizer; +import com.facebook.presto.sql.planner.sanity.PlanChecker; +import com.facebook.presto.sql.tree.ExplainType; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableMap; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; +import org.weakref.jmx.MBeanExporter; +import org.weakref.jmx.testing.TestingMBeanServer; + +import java.util.List; +import java.util.regex.Pattern; + +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createLineitem; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createNation; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrders; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrdersEx; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createRegion; +import static com.facebook.presto.transaction.TransactionBuilder.transaction; +import static com.facebook.presto.util.AnalyzerUtil.createParsingOptions; +import static java.util.Collections.emptyList; +import static org.testng.Assert.fail; + +public class TestBuiltInNativeFunctions + extends AbstractTestQueryFramework +{ + @Override + protected void createTables() + { + QueryRunner queryRunner = (QueryRunner) getExpectedQueryRunner(); + createLineitem(queryRunner); + createNation(queryRunner); + createOrders(queryRunner); + createOrdersEx(queryRunner); + createRegion(queryRunner); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder() + .setExtraProperties(ImmutableMap.of("built-in-sidecar-functions-enabled", "true")) + .setAddStorageFormatToPath(true) + .setWorkerSidecarEnabled(true) + .build(); + + queryRunner.registerNativeFunctions(); + + return queryRunner; + } + + @Override + protected QueryRunner createExpectedQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.javaHiveQueryRunnerBuilder() + .setAddStorageFormatToPath(true) + .build(); + } + + private void assertJsonPlan(@Language("SQL") String query, boolean withBuiltInSidecarEnabled, @Language("RegExp") String jsonPlanRegex, boolean shouldContainRegex) + { + QueryRunner queryRunner; + if (withBuiltInSidecarEnabled) { + queryRunner = getQueryRunner(); + } + else { + queryRunner = (QueryRunner) getExpectedQueryRunner(); + } + + QueryExplainer explainer = getQueryExplainerFromProvidedQueryRunner(queryRunner); + transaction(queryRunner.getTransactionManager(), queryRunner.getAccessControl()) + .singleStatement() + .execute(queryRunner.getDefaultSession(), transactionSession -> { + String actualPlan = explainer.getJsonPlan(transactionSession, getSqlParser().createStatement(query, createParsingOptions(transactionSession)), ExplainType.Type.LOGICAL, emptyList(), WarningCollector.NOOP, query); + Pattern p = Pattern.compile(jsonPlanRegex, Pattern.MULTILINE); + if (shouldContainRegex) { + if (!p.matcher(actualPlan).find()) { + fail("Query plan text does not contain regex"); + } + } + else { + if (p.matcher(actualPlan).find()) { + fail("Query plan text contains bad pattern"); + } + } + + return null; + }); + } + + private QueryExplainer getQueryExplainerFromProvidedQueryRunner(QueryRunner queryRunner) + { + Metadata metadata = queryRunner.getMetadata(); + FeaturesConfig featuresConfig = createFeaturesConfig(); + boolean noExchange = queryRunner.getNodeCount() == 1; + TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount); + CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator); + List optimizers = new PlanOptimizers( + metadata, + getSqlParser(), + noExchange, + new MBeanExporter(new TestingMBeanServer()), + queryRunner.getSplitManager(), + queryRunner.getPlanOptimizerManager(), + queryRunner.getPageSourceManager(), + queryRunner.getStatsCalculator(), + costCalculator, + new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), + new CostComparator(featuresConfig), + taskCountEstimator, + new PartitioningProviderManager(), + featuresConfig, + new ExpressionOptimizerManager( + new PluginNodeManager(new InMemoryNodeManager()), + queryRunner.getMetadata().getFunctionAndTypeManager()), + new TaskManagerConfig()) + .getPlanningTimeOptimizers(); + return new QueryExplainer( + optimizers, + new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig(), featuresConfig, queryRunner.getPlanCheckerProviderManager()), + metadata, + queryRunner.getAccessControl(), + getSqlParser(), + queryRunner.getStatsCalculator(), + costCalculator, + ImmutableMap.of(), + new PlanChecker(featuresConfig, false, queryRunner.getPlanCheckerProviderManager())); + } + + @Test + public void testUdfQueries() + { + assertQuery("SELECT ARRAY['abc']"); + assertQuery("SELECT ARRAY[1, 2, 3]"); + assertQuery("SELECT map_remove_null_values( MAP( ARRAY['a', 'b', 'c'], ARRAY[1, NULL, 3] ) )"); + assertQuery("SELECT presto.default.map_remove_null_values( MAP( ARRAY['a', 'b', 'c'], ARRAY[1, NULL, 3] ) )"); + assertQueryFails("SELECT native.default.map_remove_null_values( MAP( ARRAY['a', 'b', 'c'], ARRAY[1, NULL, 3] ) )", ".*Function native.default.map_remove_null_values not registered.*"); + assertJsonPlan("SELECT map_remove_null_values( MAP( ARRAY['a', 'b', 'c'], ARRAY[1, NULL, 3] ) )", true, "lambda", false); + assertJsonPlan("SELECT map_remove_null_values( MAP( ARRAY['a', 'b', 'c'], ARRAY[1, NULL, 3] ) )", false, "lambda", true); + } +} diff --git a/presto-native-sidecar-plugin/pom.xml b/presto-native-sidecar-plugin/pom.xml index 153ad18295f8c..eb399fde9a608 100644 --- a/presto-native-sidecar-plugin/pom.xml +++ b/presto-native-sidecar-plugin/pom.xml @@ -203,14 +203,6 @@ com.facebook.presto presto-main-base - test - test-jar - - - - com.facebook.presto - presto-main-base - test @@ -234,7 +226,6 @@ com.facebook.presto presto-client - test diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java index 11ef917f5522d..208d510c92d4d 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java @@ -14,14 +14,10 @@ package com.facebook.presto.sidecar.functionNamespace; import com.facebook.airlift.log.Logger; -import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; -import com.facebook.presto.common.type.NamedTypeSignature; -import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.common.type.TypeSignature; -import com.facebook.presto.common.type.TypeSignatureParameter; import com.facebook.presto.common.type.UserDefinedType; import com.facebook.presto.functionNamespace.AbstractSqlInvokedFunctionNamespaceManager; import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; @@ -38,7 +34,6 @@ import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.FunctionMetadataManager; import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; -import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.Parameter; import com.facebook.presto.spi.function.ScalarFunctionImplementation; import com.facebook.presto.spi.function.Signature; @@ -48,20 +43,16 @@ import com.facebook.presto.spi.function.SqlFunctionSupplier; import com.facebook.presto.spi.function.SqlInvokedAggregationFunctionImplementation; import com.facebook.presto.spi.function.SqlInvokedFunction; -import com.facebook.presto.spi.function.TypeVariableConstraint; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Suppliers; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.UncheckedExecutionException; import jakarta.inject.Inject; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -69,14 +60,12 @@ import java.util.function.Supplier; import static com.facebook.presto.common.type.TypeSignatureUtils.resolveIntermediateType; +import static com.facebook.presto.metadata.NativeSidecarFunctionUtil.createSqlInvokedFunction; import static com.facebook.presto.spi.StandardErrorCode.DUPLICATE_FUNCTION_ERROR; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; -import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; -import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.CPP; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.MoreCollectors.onlyElement; import static java.lang.String.format; @@ -200,44 +189,6 @@ private AggregationFunctionImplementation processSqlFunctionHandle(SqlFunctionHa return aggregationImplementationByHandle.get(sqlFunctionHandle); } - protected synchronized SqlInvokedFunction createSqlInvokedFunction(String functionName, JsonBasedUdfFunctionMetadata jsonBasedUdfFunctionMetaData) - { - checkState(jsonBasedUdfFunctionMetaData.getRoutineCharacteristics().getLanguage().equals(CPP), "NativeFunctionNamespaceManager only supports CPP UDF"); - QualifiedObjectName qualifiedFunctionName = QualifiedObjectName.valueOf(new CatalogSchemaName(getCatalogName(), jsonBasedUdfFunctionMetaData.getSchema()), functionName); - List parameterNameList = jsonBasedUdfFunctionMetaData.getParamNames(); - List parameterTypeList = convertApplicableTypeToVariable(jsonBasedUdfFunctionMetaData.getParamTypes()); - List typeVariableConstraintsList = jsonBasedUdfFunctionMetaData.getTypeVariableConstraints().isPresent() ? - jsonBasedUdfFunctionMetaData.getTypeVariableConstraints().get() : Collections.emptyList(); - List longVariableConstraintList = jsonBasedUdfFunctionMetaData.getLongVariableConstraints().isPresent() ? - jsonBasedUdfFunctionMetaData.getLongVariableConstraints().get() : Collections.emptyList(); - - TypeSignature outputType = convertApplicableTypeToVariable(jsonBasedUdfFunctionMetaData.getOutputType()); - ImmutableList.Builder parameterBuilder = ImmutableList.builder(); - for (int i = 0; i < parameterNameList.size(); i++) { - parameterBuilder.add(new Parameter(parameterNameList.get(i), parameterTypeList.get(i))); - } - - Optional aggregationFunctionMetadata = - jsonBasedUdfFunctionMetaData.getAggregateMetadata() - .map(metadata -> new AggregationFunctionMetadata( - convertApplicableTypeToVariable(metadata.getIntermediateType()), - metadata.isOrderSensitive())); - - return new SqlInvokedFunction( - qualifiedFunctionName, - parameterBuilder.build(), - typeVariableConstraintsList, - longVariableConstraintList, - outputType, - jsonBasedUdfFunctionMetaData.getDocString(), - jsonBasedUdfFunctionMetaData.getRoutineCharacteristics(), - "", - jsonBasedUdfFunctionMetaData.getVariableArity(), - notVersioned(), - jsonBasedUdfFunctionMetaData.getFunctionKind(), - aggregationFunctionMetadata); - } - @Override protected Collection fetchFunctionsDirect(QualifiedObjectName functionName) { @@ -320,107 +271,12 @@ public final FunctionHandle getFunctionHandle(Optional typeSignaturesList = convertApplicableTypeToVariable(ImmutableList.of(typeSignature)); - checkArgument(!typeSignaturesList.isEmpty(), "Type signature list is empty for : " + typeSignature); - return typeSignaturesList.get(0); - } - - public static List convertApplicableTypeToVariable(List typeSignatures) - { - List newTypeSignaturesList = new ArrayList<>(); - for (TypeSignature typeSignature : typeSignatures) { - if (!typeSignature.getParameters().isEmpty()) { - TypeSignature newTypeSignature = - new TypeSignature( - typeSignature.getBase(), - getTypeSignatureParameters( - typeSignature, - typeSignature.getParameters())); - newTypeSignaturesList.add(newTypeSignature); - } - else { - newTypeSignaturesList.add(typeSignature); - } - } - return newTypeSignaturesList; - } - @VisibleForTesting public FunctionDefinitionProvider getFunctionDefinitionProvider() { return functionDefinitionProvider; } - private static List getTypeSignatureParameters( - TypeSignature typeSignature, - List typeSignatureParameterList) - { - List newParameterTypeList = new ArrayList<>(); - for (TypeSignatureParameter parameter : typeSignatureParameterList) { - if (parameter.isLongLiteral()) { - newParameterTypeList.add(parameter); - continue; - } - - boolean isNamedTypeSignature = parameter.isNamedTypeSignature(); - TypeSignature parameterTypeSignature; - // If it's a named type signatures only in the case of row signature types. - if (isNamedTypeSignature) { - parameterTypeSignature = parameter.getNamedTypeSignature().getTypeSignature(); - } - else { - parameterTypeSignature = parameter.getTypeSignature(); - } - - if (parameterTypeSignature.getParameters().isEmpty()) { - boolean changeTypeToVariable = isDecimalTypeBase(typeSignature.getBase()); - if (changeTypeToVariable) { - newParameterTypeList.add( - TypeSignatureParameter.of(parameterTypeSignature.getBase())); - } - else { - if (isNamedTypeSignature) { - newParameterTypeList.add(TypeSignatureParameter.of(parameter.getNamedTypeSignature())); - } - else { - newParameterTypeList.add(TypeSignatureParameter.of(parameterTypeSignature)); - } - } - } - else { - TypeSignature newTypeSignature = - new TypeSignature( - parameterTypeSignature.getBase(), - getTypeSignatureParameters( - parameterTypeSignature.getStandardTypeSignature(), - parameterTypeSignature.getParameters())); - if (isNamedTypeSignature) { - newParameterTypeList.add( - TypeSignatureParameter.of( - new NamedTypeSignature( - Optional.empty(), - newTypeSignature))); - } - else { - newParameterTypeList.add(TypeSignatureParameter.of(newTypeSignature)); - } - } - } - return newParameterTypeList; - } - - private static boolean isDecimalTypeBase(String typeBase) - { - return typeBase.equals(StandardTypes.DECIMAL); - } - // Hack ends here private synchronized void createFunction(SqlInvokedFunction function) diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java index 2da8ad1970eec..0164dd10c3af1 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java @@ -153,4 +153,9 @@ default Iterable getClientRequestFilterFactories() { return emptyList(); } + + default Set> getSqlInvokedFunctions() + { + return emptySet(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/BuiltInType.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/BuiltInType.java new file mode 100644 index 0000000000000..a9a0deb9212ec --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/BuiltInType.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +import com.facebook.drift.annotations.ThriftEnum; +import com.facebook.drift.annotations.ThriftEnumValue; + +@ThriftEnum +public enum BuiltInType +{ + NONE(0), + PLUGIN(1), + NATIVE(2); + + private final int value; + + BuiltInType(int value) + { + this.value = value; + } + + @ThriftEnumValue + public int getValue() + { + return value; + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java index f7e9bc46ff07a..01173194d270f 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java @@ -31,4 +31,10 @@ public interface FunctionHandle FunctionKind getKind(); List getArgumentTypes(); + + // todo: fix this hack + default BuiltInType getBuiltInType() + { + return BuiltInType.NONE; + } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java index 72999fbe97c16..12d1a4b712425 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestQueryFramework.java @@ -565,7 +565,7 @@ protected SubPlan subplan(String sql, Session session) } } - private QueryExplainer getQueryExplainer() + protected QueryExplainer getQueryExplainer() { Metadata metadata = queryRunner.getMetadata(); FeaturesConfig featuresConfig = createFeaturesConfig(); @@ -630,6 +630,12 @@ protected ExpectedQueryRunner getExpectedQueryRunner() return expectedQueryRunner; } + protected SqlParser getSqlParser() + { + checkState(sqlParser != null, "sqlParser not set"); + return sqlParser; + } + public interface QueryRunnerSupplier { QueryRunner get() diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index d9ce3439011ab..1f3706f97b3d5 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -1040,6 +1040,13 @@ public void loadPlanCheckerProviderManager(String planCheckerProviderName, Map