Skip to content

Commit bf6ec26

Browse files
pengpeng-lualecgrieser
pengpeng-lu
and
alecgrieser
authored
Support defining UDFs in RecordMetaData (#2995)
Co-authored-by: Alec Grieser <[email protected]>
1 parent 97f008e commit bf6ec26

File tree

43 files changed

+789
-443
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+789
-443
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@
3131
import com.apple.foundationdb.record.metadata.UnnestedRecordType;
3232
import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
3333
import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression;
34+
import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
3435
import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
36+
import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry;
3537
import com.apple.foundationdb.record.query.plan.synthetic.SyntheticRecordPlanner;
3638
import com.apple.foundationdb.record.util.MapUtils;
3739
import com.google.common.base.Verify;
@@ -83,6 +85,8 @@ public class RecordMetaData implements RecordMetaDataProvider {
8385
@Nonnull
8486
private final Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticTypeMap;
8587
@Nonnull
88+
private final Map<String, UserDefinedFunction> userDefinedFunctionMap;
89+
@Nonnull
8690
private final Map<String, Index> indexes;
8791
@Nonnull
8892
private final Map<String, Index> universalIndexes;
@@ -112,6 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) {
112116
Collections.unmodifiableMap(orig.indexes),
113117
Collections.unmodifiableMap(orig.universalIndexes),
114118
Collections.unmodifiableList(orig.formerIndexes),
119+
Collections.unmodifiableMap(orig.userDefinedFunctionMap),
115120
orig.splitLongRecords,
116121
orig.storeRecordVersions,
117122
orig.version,
@@ -131,6 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
131136
@Nonnull Map<String, Index> indexes,
132137
@Nonnull Map<String, Index> universalIndexes,
133138
@Nonnull List<FormerIndex> formerIndexes,
139+
@Nonnull Map<String, UserDefinedFunction> userDefinedFunctionMap,
134140
boolean splitLongRecords,
135141
boolean storeRecordVersions,
136142
int version,
@@ -147,6 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
147153
this.indexes = indexes;
148154
this.universalIndexes = universalIndexes;
149155
this.formerIndexes = formerIndexes;
156+
this.userDefinedFunctionMap = userDefinedFunctionMap;
150157
this.splitLongRecords = splitLongRecords;
151158
this.storeRecordVersions = storeRecordVersions;
152159
this.version = version;
@@ -692,7 +699,9 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor
692699
builder.addFormerIndexes(formerIndex.toProto());
693700
}
694701

695-
// Add in the final options.
702+
PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
703+
PlanHashable.CURRENT_FOR_CONTINUATION);
704+
builder.addAllUserDefinedFunctions(userDefinedFunctionMap.values().stream().map(func -> func.toProto(serializationContext)).collect(Collectors.toList()));
696705
builder.setSplitLongRecords(splitLongRecords);
697706
builder.setStoreRecordVersions(storeRecordVersions);
698707
builder.setVersion(version);

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@
4242
import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry;
4343
import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl;
4444
import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor;
45+
import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction;
46+
import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry;
47+
import com.apple.foundationdb.record.query.plan.serialization.PlanSerialization;
4548
import com.google.common.base.Verify;
4649
import com.google.common.collect.ImmutableMap;
4750
import com.google.common.collect.Maps;
@@ -110,6 +113,8 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
110113
@Nonnull
111114
private final Map<String, SyntheticRecordTypeBuilder<?>> syntheticRecordTypes;
112115
@Nonnull
116+
private final Map<String, UserDefinedFunction> userDefinedFunctionMap;
117+
@Nonnull
113118
private final Map<String, Index> indexes;
114119
@Nonnull
115120
private final Map<String, Index> universalIndexes;
@@ -144,6 +149,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
144149
indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance();
145150
evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance();
146151
syntheticRecordTypes = new HashMap<>();
152+
userDefinedFunctionMap = new HashMap<>();
147153
}
148154

149155
private void processSchemaOptions(boolean processExtensionOptions) {
@@ -222,6 +228,11 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa
222228
typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
223229
}
224230
}
231+
for (RecordMetaDataProto.PUserDefinedFunction function: metaDataProto.getUserDefinedFunctionsList()) {
232+
UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
233+
PlanHashable.CURRENT_FOR_CONTINUATION), function);
234+
userDefinedFunctionMap.put(func.getFunctionName(), func);
235+
}
225236
if (metaDataProto.hasSplitLongRecords()) {
226237
splitLongRecords = metaDataProto.getSplitLongRecords();
227238
}
@@ -1179,6 +1190,14 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) {
11791190
formerIndexes.add(formerIndex);
11801191
}
11811192

1193+
public void addUserDefinedFunction(@Nonnull UserDefinedFunction userDefinedFunction) {
1194+
userDefinedFunctionMap.put(userDefinedFunction.getFunctionName(), userDefinedFunction);
1195+
}
1196+
1197+
public void addUserDefinedFunctions(@Nonnull Iterable<? extends UserDefinedFunction> functions) {
1198+
functions.forEach(this::addUserDefinedFunction);
1199+
}
1200+
11821201
public boolean isSplitLongRecords() {
11831202
return splitLongRecords;
11841203
}
@@ -1420,7 +1439,7 @@ public RecordMetaData build(boolean validate) {
14201439
Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
14211440
RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields,
14221441
builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap,
1423-
indexes, universalIndexes, formerIndexes,
1442+
indexes, universalIndexes, formerIndexes, userDefinedFunctionMap,
14241443
splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
14251444
for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
14261445
KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
25-
import com.apple.foundationdb.record.RecordMetaDataProto;
25+
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
2626
import com.apple.foundationdb.record.logging.LogMessageKeys;
2727
import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression;
2828
import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
@@ -392,21 +392,21 @@ public static class Evaluated {
392392
* Values used in index keys in place of missing fields.
393393
*/
394394
public enum NullStandin {
395-
NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
396-
NULL_UNIQUE(RecordMetaDataProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
397-
NOT_NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
395+
NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
396+
NULL_UNIQUE(RecordKeyExpressionProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
397+
NOT_NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
398398

399-
private RecordMetaDataProto.Field.NullInterpretation proto;
399+
private RecordKeyExpressionProto.Field.NullInterpretation proto;
400400

401-
NullStandin(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) {
401+
NullStandin(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) {
402402
proto = nullInterpretation;
403403
}
404404

405-
public RecordMetaDataProto.Field.NullInterpretation toProto() {
405+
public RecordKeyExpressionProto.Field.NullInterpretation toProto() {
406406
return proto;
407407
}
408408

409-
public static NullStandin valueOf(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
409+
public static NullStandin valueOf(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
410410
switch (nullInterpretation) {
411411
case NOT_UNIQUE:
412412
return NULL;

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.record.ObjectPlanHash;
2525
import com.apple.foundationdb.record.PlanHashable;
26-
import com.apple.foundationdb.record.RecordMetaDataProto;
26+
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
2727
import com.apple.foundationdb.record.metadata.Key;
2828
import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
2929
import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -63,7 +63,7 @@ private DimensionsKeyExpression(@Nonnull final KeyExpression wholeKey,
6363
this.dimensionsSize = dimensionsSize;
6464
}
6565

66-
DimensionsKeyExpression(@Nonnull final RecordMetaDataProto.Dimensions dimensions) throws DeserializationException {
66+
DimensionsKeyExpression(@Nonnull final RecordKeyExpressionProto.Dimensions dimensions) throws DeserializationException {
6767
this(KeyExpression.fromProto(dimensions.getWholeKey()), dimensions.getPrefixSize(), dimensions.getDimensionsSize());
6868
}
6969

@@ -114,8 +114,8 @@ public boolean needsCopyingToPartialRecord() {
114114

115115
@Nonnull
116116
@Override
117-
public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
118-
final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder();
117+
public RecordKeyExpressionProto.Dimensions toProto() throws SerializationException {
118+
final RecordKeyExpressionProto.Dimensions.Builder builder = RecordKeyExpressionProto.Dimensions.newBuilder();
119119
builder.setWholeKey(getWholeKey().toKeyExpression());
120120
builder.setPrefixSize(prefixSize);
121121
builder.setDimensionsSize(dimensionsSize);
@@ -124,8 +124,8 @@ public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
124124

125125
@Nonnull
126126
@Override
127-
public RecordMetaDataProto.KeyExpression toKeyExpression() {
128-
return RecordMetaDataProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
127+
public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
128+
return RecordKeyExpressionProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
129129
}
130130

131131
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.record.ObjectPlanHash;
2525
import com.apple.foundationdb.record.PlanHashable;
26-
import com.apple.foundationdb.record.RecordMetaDataProto;
26+
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
2727
import com.apple.foundationdb.record.metadata.Key;
2828
import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
2929
import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -45,8 +45,8 @@ public class EmptyKeyExpression extends BaseKeyExpression implements KeyExpressi
4545
private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Empty-Key-Expression");
4646

4747
public static final EmptyKeyExpression EMPTY = new EmptyKeyExpression();
48-
public static final RecordMetaDataProto.KeyExpression EMPTY_PROTO =
49-
RecordMetaDataProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
48+
public static final RecordKeyExpressionProto.KeyExpression EMPTY_PROTO =
49+
RecordKeyExpressionProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
5050

5151
private EmptyKeyExpression() {
5252
// nothing to initialize
@@ -75,8 +75,8 @@ public int getColumnSize() {
7575

7676
@Nonnull
7777
@Override
78-
public RecordMetaDataProto.Empty toProto() throws SerializationException {
79-
return RecordMetaDataProto.Empty.getDefaultInstance();
78+
public RecordKeyExpressionProto.Empty toProto() throws SerializationException {
79+
return RecordKeyExpressionProto.Empty.getDefaultInstance();
8080
}
8181

8282
@Nonnull
@@ -87,7 +87,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
8787

8888
@Nonnull
8989
@Override
90-
public RecordMetaDataProto.KeyExpression toKeyExpression() {
90+
public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
9191
return EMPTY_PROTO;
9292
}
9393

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
import com.apple.foundationdb.record.ObjectPlanHash;
2525
import com.apple.foundationdb.record.PlanHashable;
2626
import com.apple.foundationdb.record.RecordCoreException;
27-
import com.apple.foundationdb.record.RecordMetaDataProto;
27+
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
2828
import com.apple.foundationdb.record.logging.LogMessageKeys;
2929
import com.apple.foundationdb.record.metadata.Key;
3030
import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -71,7 +71,7 @@ public FieldKeyExpression(@Nonnull String fieldName, @Nonnull FanType fanType, @
7171
this.nullStandin = nullStandin;
7272
}
7373

74-
public FieldKeyExpression(@Nonnull RecordMetaDataProto.Field field) throws DeserializationException {
74+
public FieldKeyExpression(@Nonnull RecordKeyExpressionProto.Field field) throws DeserializationException {
7575
if (!field.hasFieldName()) {
7676
throw new DeserializationException("Serialized Field is missing field name");
7777
}
@@ -193,8 +193,8 @@ public int getColumnSize() {
193193

194194
@Nonnull
195195
@Override
196-
public RecordMetaDataProto.Field toProto() throws SerializationException {
197-
return RecordMetaDataProto.Field.newBuilder()
196+
public RecordKeyExpressionProto.Field toProto() throws SerializationException {
197+
return RecordKeyExpressionProto.Field.newBuilder()
198198
.setFieldName(fieldName)
199199
.setFanType(fanType.toProto())
200200
.setNullInterpretation(nullStandin.toProto())
@@ -203,8 +203,8 @@ public RecordMetaDataProto.Field toProto() throws SerializationException {
203203

204204
@Nonnull
205205
@Override
206-
public RecordMetaDataProto.KeyExpression toKeyExpression() {
207-
return RecordMetaDataProto.KeyExpression.newBuilder().setField(toProto()).build();
206+
public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
207+
return RecordKeyExpressionProto.KeyExpression.newBuilder().setField(toProto()).build();
208208
}
209209

210210
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import com.apple.foundationdb.record.PlanHashable;
2626
import com.apple.foundationdb.record.RecordCoreArgumentException;
2727
import com.apple.foundationdb.record.RecordCoreException;
28-
import com.apple.foundationdb.record.RecordMetaDataProto;
28+
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
2929
import com.apple.foundationdb.record.logging.LogMessageKeys;
3030
import com.apple.foundationdb.record.metadata.Key;
3131
import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -232,7 +232,7 @@ public boolean equalsAtomic(AtomKeyExpression other) {
232232
* implementation, or the arguments provided are not suitable for the function.
233233
*/
234234
@Nonnull
235-
public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function function) throws DeserializationException {
235+
public static FunctionKeyExpression fromProto(RecordKeyExpressionProto.Function function) throws DeserializationException {
236236
try {
237237
return create(function.getName(), KeyExpression.fromProto(function.getArguments()));
238238
} catch (RecordCoreException e) {
@@ -242,17 +242,17 @@ public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function funct
242242

243243
@Nonnull
244244
@Override
245-
public final RecordMetaDataProto.Function toProto() throws SerializationException {
246-
RecordMetaDataProto.Function.Builder builder = RecordMetaDataProto.Function.newBuilder()
245+
public final RecordKeyExpressionProto.Function toProto() throws SerializationException {
246+
RecordKeyExpressionProto.Function.Builder builder = RecordKeyExpressionProto.Function.newBuilder()
247247
.setName(getName());
248248
builder.setArguments(getArguments().toKeyExpression());
249249
return builder.build();
250250
}
251251

252252
@Nonnull
253253
@Override
254-
public final RecordMetaDataProto.KeyExpression toKeyExpression() {
255-
return RecordMetaDataProto.KeyExpression.newBuilder().setFunction(toProto()).build();
254+
public final RecordKeyExpressionProto.KeyExpression toKeyExpression() {
255+
return RecordKeyExpressionProto.KeyExpression.newBuilder().setFunction(toProto()).build();
256256
}
257257

258258
@Nonnull

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import com.apple.foundationdb.annotation.API;
2424
import com.apple.foundationdb.record.ObjectPlanHash;
2525
import com.apple.foundationdb.record.PlanHashable;
26-
import com.apple.foundationdb.record.RecordMetaDataProto;
26+
import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
2727
import com.apple.foundationdb.record.metadata.Key;
2828
import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
2929
import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -53,7 +53,7 @@ public GroupingKeyExpression(@Nonnull KeyExpression wholeKey, int groupedCount)
5353
this.groupedCount = groupedCount;
5454
}
5555

56-
public GroupingKeyExpression(@Nonnull RecordMetaDataProto.Grouping grouping) throws DeserializationException {
56+
public GroupingKeyExpression(@Nonnull RecordKeyExpressionProto.Grouping grouping) throws DeserializationException {
5757
this(KeyExpression.fromProto(grouping.getWholeKey()), grouping.getGroupedCount());
5858
}
5959

@@ -100,17 +100,17 @@ public boolean needsCopyingToPartialRecord() {
100100

101101
@Nonnull
102102
@Override
103-
public RecordMetaDataProto.Grouping toProto() throws SerializationException {
104-
final RecordMetaDataProto.Grouping.Builder builder = RecordMetaDataProto.Grouping.newBuilder();
103+
public RecordKeyExpressionProto.Grouping toProto() throws SerializationException {
104+
final RecordKeyExpressionProto.Grouping.Builder builder = RecordKeyExpressionProto.Grouping.newBuilder();
105105
builder.setWholeKey(getWholeKey().toKeyExpression());
106106
builder.setGroupedCount(groupedCount);
107107
return builder.build();
108108
}
109109

110110
@Nonnull
111111
@Override
112-
public RecordMetaDataProto.KeyExpression toKeyExpression() {
113-
return RecordMetaDataProto.KeyExpression.newBuilder().setGrouping(toProto()).build();
112+
public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
113+
return RecordKeyExpressionProto.KeyExpression.newBuilder().setGrouping(toProto()).build();
114114
}
115115

116116
@Nonnull

0 commit comments

Comments
 (0)