diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java index df4ad626ec..5e1c8ef400 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java @@ -41,9 +41,10 @@ private IndexPrefetchRangeKeyValueCursor(@Nonnull final FDBRecordContext context @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, - int valuesLimit) { + int valuesLimit, + @Nonnull SerializationMode serializationMode) { - super(context, iterator, prefixLength, limitManager, valuesLimit); + super(context, iterator, prefixLength, limitManager, valuesLimit, serializationMode); } /** @@ -69,7 +70,7 @@ public IndexPrefetchRangeKeyValueCursor build() { AsyncIterator iterator = getTransaction() .getMappedRange(getBegin(), getEnd(), mapper, getLimit(), isReverse(), getStreamingMode()) .iterator(); - return new IndexPrefetchRangeKeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit()); + return new IndexPrefetchRangeKeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit(), serializationMode); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java index 6534ceb5a4..15b77d6c5f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java @@ -38,8 +38,9 @@ private KeyValueCursor(@Nonnull final FDBRecordContext context, @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, - int valuesLimit) { - super(context, iterator, prefixLength, limitManager, valuesLimit); + int valuesLimit, + @Nonnull SerializationMode serializationMode) { + super(context, iterator, prefixLength, limitManager, valuesLimit, serializationMode); } /** @@ -77,7 +78,7 @@ public KeyValueCursor build() { final AsyncIterator iterator = getTransaction() .getRange(getBegin(), getEnd(), getLimit(), isReverse(), getStreamingMode()) .iterator(); - return new KeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit()); + return new KeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit(), serializationMode); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index ec5f7b9d17..7868a02932 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.KeyRange; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorProto; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.TupleRange; @@ -42,11 +43,13 @@ import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.ZeroCopyByteString; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Arrays; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -61,22 +64,26 @@ public abstract class KeyValueCursorBase extends AsyncIterat private final int prefixLength; @Nonnull private final CursorLimitManager limitManager; - private int valuesLimit; + private final int valuesLimit; // the pointer may be mutated, but the actual array must never be mutated or continuations will break @Nullable private byte[] lastKey; + @Nonnull + private final SerializationMode serializationMode; protected KeyValueCursorBase(@Nonnull final FDBRecordContext context, @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, - int valuesLimit) { + int valuesLimit, + @Nonnull final SerializationMode serializationMode) { super(context.getExecutor(), iterator); this.context = context; this.prefixLength = prefixLength; this.limitManager = limitManager; this.valuesLimit = valuesLimit; + this.serializationMode = serializationMode; context.instrument(FDBStoreTimer.DetailEvents.GET_SCAN_RANGE_RAW_FIRST_CHUNK, iterator.onHasNext()); } @@ -131,21 +138,23 @@ public RecordCursorResult getNext() { @Nonnull private RecordCursorContinuation continuationHelper() { - return new Continuation(lastKey, prefixLength); + return new Continuation(lastKey, prefixLength, serializationMode); } - private static class Continuation implements RecordCursorContinuation { + public static class Continuation implements RecordCursorContinuation { @Nullable private final byte[] lastKey; private final int prefixLength; + private final SerializationMode serializationMode; - public Continuation(@Nullable final byte[] lastKey, final int prefixLength) { + public Continuation(@Nullable final byte[] lastKey, final int prefixLength, final SerializationMode serializationMode) { // Note that doing this without a full copy is dangerous if the array is ever mutated. // Currently, this never happens and the only thing that changes is which array lastKey points to. // However, if logic in KeyValueCursor or KeyValue changes, this could break continuations. // To resolve it, we could resort to doing a full copy here, although that's somewhat expensive. this.lastKey = lastKey; this.prefixLength = prefixLength; + this.serializationMode = serializationMode; } @Override @@ -156,21 +165,83 @@ public boolean isEnd() { @Nonnull @Override public ByteString toByteString() { - if (lastKey == null) { - return ByteString.EMPTY; + if (serializationMode == SerializationMode.TO_OLD) { + // lastKey = null when source iterator hit limit that we passed down. + if (lastKey == null) { + return ByteString.EMPTY; + } + ByteString base = ZeroCopyByteString.wrap(lastKey); + // when prefixLength == lastKey.length, toByteString() also returns ByteString.EMPTY + return base.substring(prefixLength, lastKey.length); + } else { + return toProto().toByteString(); } - ByteString base = ZeroCopyByteString.wrap(lastKey); - return base.substring(prefixLength, lastKey.length); } @Nullable @Override public byte[] toBytes() { + if (lastKey == null) { + return null; + } + ByteString byteString = toByteString(); + return byteString.isEmpty() ? new byte[0] : byteString.toByteArray(); + } + + @Nullable + public byte[] getInnerContinuationInBytes() { if (lastKey == null) { return null; } return Arrays.copyOfRange(lastKey, prefixLength, lastKey.length); } + + @Nonnull + public ByteString getInnerContinuationInByteString() { + if (lastKey == null) { + return ByteString.EMPTY; + } + ByteString base = ZeroCopyByteString.wrap(lastKey); + return base.substring(prefixLength, lastKey.length); + } + + public static byte[] fromRawBytes(@Nullable byte[] rawBytes, SerializationMode serializationMode) { + if (rawBytes == null) { + return null; + } + if (serializationMode == SerializationMode.TO_OLD) { + return rawBytes; + } + try { + RecordCursorProto.KeyValueCursorContinuation continuationProto = RecordCursorProto.KeyValueCursorContinuation.parseFrom(rawBytes); + if (continuationProto.hasPrefixLength()) { + return continuationProto.getContinuation().toByteArray(); + } else { + // parseFrom can parse an old serialization result as the new proto, wrong deserialization + return rawBytes; + } + } catch (InvalidProtocolBufferException ipbe) { + return rawBytes; + } + } + + @Nonnull + private RecordCursorProto.KeyValueCursorContinuation toProto() { + RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); + if (lastKey == null) { + // proto.hasContinuation() = false when lastKey = null + return builder.setPrefixLength(prefixLength).build(); + } else { + ByteString base = ZeroCopyByteString.wrap(Objects.requireNonNull(lastKey)); + // proto.hasContinuation() = ByteString.EMPTY when prefixLength = lastKey.length + return builder.setContinuation(base.substring(prefixLength, lastKey.length)).setPrefixLength(prefixLength).build(); + } + } + } + + public enum SerializationMode { + TO_OLD, + TO_NEW } /** @@ -208,9 +279,11 @@ public abstract static class Builder> { private StreamingMode streamingMode; private KeySelector begin; private KeySelector end; + protected SerializationMode serializationMode; protected Builder(@Nonnull Subspace subspace) { this.subspace = subspace; + this.serializationMode = SerializationMode.TO_OLD; } /** @@ -247,10 +320,12 @@ protected void prepare() { prefixLength = calculatePrefixLength(); reverse = scanProperties.isReverse(); + if (continuation != null) { - final byte[] continuationBytes = new byte[prefixLength + continuation.length]; + byte[] realContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, serializationMode); + final byte[] continuationBytes = new byte[prefixLength + realContinuation.length]; System.arraycopy(lowBytes, 0, continuationBytes, 0, prefixLength); - System.arraycopy(continuation, 0, continuationBytes, prefixLength, continuation.length); + System.arraycopy(realContinuation, 0, continuationBytes, prefixLength, realContinuation.length); if (reverse) { highBytes = continuationBytes; highEndpoint = EndpointType.CONTINUATION; @@ -334,6 +409,11 @@ public T setHigh(@Nonnull byte[] highBytes, @Nonnull EndpointType highEndpoint) return self(); } + public T setSerializationMode(@Nonnull final SerializationMode serializationMode) { + this.serializationMode = serializationMode; + return self(); + } + /** * Calculate the key prefix length for the returned values. This will be used to derive the primary key used in * the calculated continuation. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index bce54f1450..a0426115f7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -57,6 +57,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; import com.apple.foundationdb.record.provider.foundationdb.IndexScanRange; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.MultidimensionalIndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.UnsupportedRemoteFetchIndexException; import com.apple.foundationdb.record.query.plan.AvailableFields; @@ -161,6 +162,8 @@ public class RecordQueryIndexPlan implements RecordQueryPlanWithNoChildren, @Nonnull private final Supplier comparisonRangesSupplier; + @Nonnull + private final KeyValueCursorBase.SerializationMode serializationMode; public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexScanParameters scanParameters, final boolean reverse) { this(indexName, null, scanParameters, IndexFetchMethod.SCAN_AND_FETCH, FetchIndexRecords.PRIMARY_KEY, reverse, false); @@ -214,6 +217,21 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final Optional matchCandidateOptional, @Nonnull final Type resultType, @Nonnull final QueryPlanConstraint constraint) { + this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, matchCandidateOptional, resultType, constraint, KeyValueCursorBase.SerializationMode.TO_NEW); + } + + @VisibleForTesting + public RecordQueryIndexPlan(@Nonnull final String indexName, + @Nullable final KeyExpression commonPrimaryKey, + @Nonnull final IndexScanParameters scanParameters, + @Nonnull final IndexFetchMethod indexFetchMethod, + @Nonnull final FetchIndexRecords fetchIndexRecords, + final boolean reverse, + final boolean strictlySorted, + @Nonnull final Optional matchCandidateOptional, + @Nonnull final Type resultType, + @Nonnull final QueryPlanConstraint constraint, + @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { this.indexName = indexName; this.commonPrimaryKey = commonPrimaryKey; this.scanParameters = scanParameters; @@ -231,6 +249,7 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, } this.constraint = constraint; this.comparisonRangesSupplier = Suppliers.memoize(this::computeComparisonRanges); + this.serializationMode = serializationMode; } @Nonnull @@ -290,8 +309,9 @@ private RecordCursor executeUsingRemoteFetch(@N final RecordMetaData metaData = store.getRecordMetaData(); final Index index = metaData.getIndex(indexName); final IndexScanBounds scanBounds = scanParameters.bind(store, index, context); + byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, serializationMode); - return store.scanIndexRemoteFetch(index, scanBounds, continuation, executeProperties.asScanProperties(isReverse()), IndexOrphanBehavior.ERROR) + return store.scanIndexRemoteFetch(index, scanBounds, innerContinuation, executeProperties.asScanProperties(isReverse()), IndexOrphanBehavior.ERROR) .map(store::queriedRecord) .map(QueryResult::fromQueriedRecord); } @@ -303,13 +323,15 @@ public RecordCursor executeEntries(@Nonnull FDBR final RecordMetaData metaData = store.getRecordMetaData(); final Index index = metaData.getIndex(indexName); final IndexScanBounds scanBounds = scanParameters.bind(store, index, context); + byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, serializationMode); + if (!IndexScanType.BY_VALUE_OVER_SCAN.equals(getScanType())) { - return store.scanIndex(index, scanBounds, continuation, executeProperties.asScanProperties(reverse)); + return store.scanIndex(index, scanBounds, innerContinuation, executeProperties.asScanProperties(reverse)); } // Evaluate the scan bounds. Again, this optimization can only be done if we have a scan range if (!(scanBounds instanceof IndexScanRange)) { - return store.scanIndex(index, scanBounds, continuation, executeProperties.asScanProperties(reverse)); + return store.scanIndex(index, scanBounds, innerContinuation, executeProperties.asScanProperties(reverse)); } // Try to widen the scan range to include everything up @@ -318,7 +340,7 @@ public RecordCursor executeEntries(@Nonnull FDBR TupleRange widenedScanRange = widenRange(tupleScanRange); if (widenedScanRange == null) { // Unable to widen the range. Fall back to the original execution. - return store.scanIndex(index, scanBounds, continuation, executeProperties.asScanProperties(reverse)); + return store.scanIndex(index, scanBounds, innerContinuation, executeProperties.asScanProperties(reverse)); } return executeEntriesWithOverScan(tupleScanRange, widenedScanRange, store, index, continuation, executeProperties); @@ -328,7 +350,7 @@ private RecordCursor executeEntriesWithOverScan( @Nonnull FDBRecordStoreBase store, @Nonnull Index index, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties) { final byte[] prefixBytes = getRangePrefixBytes(tupleScanRange); - final IndexScanContinuationConvertor continuationConvertor = new IndexScanContinuationConvertor(prefixBytes); + final IndexScanContinuationConvertor continuationConvertor = new IndexScanContinuationConvertor(prefixBytes, serializationMode); // Scan a wider range, and then halt when either this scans outside the given range final IndexScanRange newScanRange = new IndexScanRange(IndexScanType.BY_VALUE, widenedScanRange); @@ -745,9 +767,12 @@ public static RecordQueryIndexPlan fromProto(@Nonnull final PlanSerializationCon private static class IndexScanContinuationConvertor implements RecordCursor.ContinuationConvertor { @Nonnull private final byte[] prefixBytes; + @Nonnull + private final KeyValueCursorBase.SerializationMode serializationMode; - public IndexScanContinuationConvertor(@Nonnull byte[] prefixBytes) { + public IndexScanContinuationConvertor(@Nonnull byte[] prefixBytes, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { this.prefixBytes = prefixBytes; + this.serializationMode = serializationMode; } @Nullable @@ -757,7 +782,8 @@ public byte[] unwrapContinuation(@Nullable final byte[] continuation) { return null; } // Add the prefix back to the inner continuation - return ByteArrayUtil.join(prefixBytes, continuation); + byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, serializationMode); + return ByteArrayUtil.join(prefixBytes, innerContinuation); } @Override @@ -765,7 +791,11 @@ public RecordCursorContinuation wrapContinuation(@Nonnull final RecordCursorCont if (continuation.isEnd()) { return continuation; } - final byte[] continuationBytes = continuation.toBytes(); + byte[] continuationBytes; + if (!(continuation instanceof KeyValueCursorBase.Continuation)) { + throw new RecordCoreException("can only wrap KeyValueCursorBase.Continuation class"); + } + continuationBytes = ((KeyValueCursorBase.Continuation) continuation).getInnerContinuationInBytes(); if (continuationBytes != null && ByteArrayUtil.startsWith(continuationBytes, prefixBytes)) { // Strip away the prefix. Note that ByteStrings re-use the underlying ByteArray, so this can // save a copy. @@ -796,7 +826,7 @@ public byte[] toBytes() { if (bytes == null) { synchronized (this) { if (bytes == null) { - byte[] baseContinuationBytes = baseContinuation.toBytes(); + byte[] baseContinuationBytes = baseContinuation instanceof KeyValueCursorBase.Continuation ? ((KeyValueCursorBase.Continuation) baseContinuation).getInnerContinuationInBytes() : baseContinuation.toBytes(); if (baseContinuationBytes == null) { return null; } @@ -810,7 +840,7 @@ public byte[] toBytes() { @Nonnull @Override public ByteString toByteString() { - return baseContinuation.toByteString().substring(prefixLength); + return (baseContinuation instanceof KeyValueCursorBase.Continuation) ? ((KeyValueCursorBase.Continuation) baseContinuation).getInnerContinuationInByteString().substring(prefixLength) : baseContinuation.toByteString().substring(prefixLength); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryScanPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryScanPlan.java index 81c51ba530..5944d7f5aa 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryScanPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryScanPlan.java @@ -36,6 +36,7 @@ import com.apple.foundationdb.record.provider.common.StoreTimer; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.AvailableFields; import com.apple.foundationdb.record.query.plan.ScanComparisons; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; @@ -94,6 +95,8 @@ public class RecordQueryScanPlan implements RecordQueryPlanWithNoChildren, Recor private final Optional matchCandidateOptional; @Nonnull private final Supplier comparisonRangesSupplier; + @Nonnull + private final KeyValueCursorBase.SerializationMode serializationMode; /** * Overloaded constructor. @@ -143,6 +146,16 @@ public RecordQueryScanPlan(@Nullable Set recordTypes, this(recordTypes, flowedType, commonPrimaryKey, comparisons, reverse, strictlySorted, Optional.of(matchCandidate)); } + public RecordQueryScanPlan(@Nullable Set recordTypes, + @Nonnull Type flowedType, + @Nullable KeyExpression commonPrimaryKey, + @Nonnull ScanComparisons comparisons, + boolean reverse, + boolean strictlySorted, + @Nonnull final Optional matchCandidateOptional) { + this(recordTypes, flowedType, commonPrimaryKey, comparisons, reverse, strictlySorted, matchCandidateOptional, KeyValueCursorBase.SerializationMode.TO_NEW); + } + /** * Overloaded constructor. @@ -157,12 +170,13 @@ public RecordQueryScanPlan(@Nullable Set recordTypes, */ @VisibleForTesting public RecordQueryScanPlan(@Nullable Set recordTypes, - @Nonnull Type flowedType, - @Nullable KeyExpression commonPrimaryKey, - @Nonnull ScanComparisons comparisons, - boolean reverse, - boolean strictlySorted, - @Nonnull final Optional matchCandidateOptional) { + @Nonnull Type flowedType, + @Nullable KeyExpression commonPrimaryKey, + @Nonnull ScanComparisons comparisons, + boolean reverse, + boolean strictlySorted, + @Nonnull final Optional matchCandidateOptional, + @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { this.recordTypes = recordTypes == null ? null : ImmutableSet.copyOf(recordTypes); this.flowedType = flowedType; this.commonPrimaryKey = commonPrimaryKey; @@ -171,6 +185,7 @@ public RecordQueryScanPlan(@Nullable Set recordTypes, this.strictlySorted = strictlySorted; this.matchCandidateOptional = matchCandidateOptional; this.comparisonRangesSupplier = Suppliers.memoize(this::computeComparisonRanges); + this.serializationMode = serializationMode; } @Nonnull @@ -180,8 +195,10 @@ public RecordCursor executePlan(@Nonnull final @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) { final TupleRange range = comparisons.toTupleRange(store, context); + byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, serializationMode); + return store.scanRecords( - range.getLow(), range.getHigh(), range.getLowEndpoint(), range.getHighEndpoint(), continuation, + range.getLow(), range.getHigh(), range.getLowEndpoint(), range.getHighEndpoint(), innerContinuation, executeProperties.asScanProperties(reverse)) .map(store::queriedRecord) .map(QueryResult::fromQueriedRecord); diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 04095bf779..a2b37b5b16 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -165,3 +165,8 @@ message OneOfTypedState { message RangeCursorContinuation { optional int64 nextPosition = 1; } + +message KeyValueCursorContinuation { + optional bytes continuation = 1; + optional int64 prefixLength = 2; +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java index d97e4b0ef9..0c2b306fab 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java @@ -44,6 +44,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.Arrays; import java.util.Collections; @@ -90,18 +92,24 @@ public void runBefore() { }); } - @Test - public void all() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void all(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { - KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) - .setContext(context) - .setRange(TupleRange.ALL) - .setContinuation(null) - .setScanProperties(ScanProperties.FORWARD_SCAN) - .build(); + byte[] continuation = null; + KeyValueCursor cursor = null; for (int i = 0; i < 5; i++) { for (int j = 0; j < 5; j++) { - KeyValue kv = cursor.getNext().get(); + cursor = KeyValueCursor.Builder.withSubspace(subspace) + .setContext(context) + .setRange(TupleRange.ALL) + .setContinuation(continuation) + .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) + .build(); + RecordCursorResult cursorResult = cursor.getNext(); + KeyValue kv = cursorResult.get(); + continuation = cursorResult.getContinuation().toBytes(); assertArrayEquals(subspace.pack(Tuple.from(i, j)), kv.getKey()); assertArrayEquals(Tuple.from(i, j).pack(), kv.getValue()); } @@ -113,6 +121,7 @@ public void all() { .setRange(TupleRange.ALL) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(10).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(10, (int)cursor.getCount().join()); cursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -120,6 +129,7 @@ public void all() { .setRange(TupleRange.ALL) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(15, (int)cursor.getCount().join()); @@ -127,14 +137,16 @@ public void all() { }); } - @Test - public void beginsWith() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void beginsWith(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); for (int j = 0; j < 5; j++) { KeyValue kv = cursor.getNext().get(); @@ -148,6 +160,7 @@ public void beginsWith() { .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(2, (int)cursor.getCount().join()); cursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -155,6 +168,7 @@ public void beginsWith() { .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(3).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(3, (int)cursor.getCount().join()); @@ -162,8 +176,9 @@ public void beginsWith() { }); } - @Test - public void inclusiveRange() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void inclusiveRange(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -171,6 +186,7 @@ public void inclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_INCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 3L), Tuple.from(3L, 4L), Tuple.from(4L, 0L), Tuple.from(4L, 1L), Tuple.from(4L, 2L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -181,6 +197,7 @@ public void inclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_INCLUSIVE) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 3L), Tuple.from(3L, 4L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -190,6 +207,7 @@ public void inclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_INCLUSIVE) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(4L, 0L), Tuple.from(4L, 1L), Tuple.from(4L, 2L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -198,8 +216,9 @@ public void inclusiveRange() { }); } - @Test - public void exclusiveRange() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void exclusiveRange(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -207,6 +226,7 @@ public void exclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 4L), Tuple.from(4L, 0L), Tuple.from(4L, 1L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -217,6 +237,7 @@ public void exclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 4L), Tuple.from(4L, 0L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -226,6 +247,7 @@ public void exclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Collections.singletonList(Tuple.from(4L, 1L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -234,8 +256,9 @@ public void exclusiveRange() { }); } - @Test - public void inclusiveNull() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void inclusiveNull(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursorIterator cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -243,6 +266,7 @@ public void inclusiveNull() { .setHigh((Tuple) null, EndpointType.RANGE_INCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build() .asIterator(); for (int j = 0; j < 5; j++) { @@ -256,8 +280,9 @@ public void inclusiveNull() { }); } - @Test - public void exclusiveNull() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void exclusiveNull(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursorIterator cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -265,6 +290,7 @@ public void exclusiveNull() { .setHigh((Tuple) null, EndpointType.RANGE_EXCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build() .asIterator(); assertThat(cursor.hasNext(), is(false)); @@ -273,13 +299,15 @@ public void exclusiveNull() { }); } - @Test - public void noNextReasons() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void noNextReasons(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(null) + .setSerializationMode(serializationMode) .setScanProperties(ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(3))) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 0L), Tuple.from(3L, 1L), Tuple.from(3L, 2L)), @@ -290,6 +318,7 @@ public void noNextReasons() { .setContext(context) .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(result.getContinuation().toBytes()) + .setSerializationMode(serializationMode) .setScanProperties(ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(3))) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 3L), Tuple.from(3L, 4L)), @@ -306,14 +335,16 @@ private ScanProperties forwardScanWithLimiter(RecordScanLimiter limiter) { return new ScanProperties(ExecuteProperties.SERIAL_EXECUTE.setState(new ExecuteState(limiter, null))); } - @Test - public void simpleScanLimit() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void simpleScanLimit(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(2); KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.ALL) .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode) .build(); assertEquals(2, (int) cursor.getCount().join()); RecordCursorResult result = cursor.getNext(); @@ -324,8 +355,9 @@ public void simpleScanLimit() { }); } - @Test - public void limitNotReached() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void limitNotReached(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(4); KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -333,6 +365,7 @@ public void limitNotReached() { .setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE) .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode) .build(); assertEquals(3, (int) cursor.getCount().join()); RecordCursorResult result = cursor.getNext(); @@ -347,15 +380,17 @@ private boolean hasNextAndAdvance(KeyValueCursor cursor) { return cursor.getNext().hasNext(); } - @Test - public void sharedLimiter() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void sharedLimiter(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(4); KeyValueCursor.Builder builder = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE) .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) - .setScanProperties(forwardScanWithLimiter(limiter)); + .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode); KeyValueCursor cursor1 = builder.build(); KeyValueCursor cursor2 = builder.build(); @@ -374,8 +409,9 @@ public void sharedLimiter() { }); } - @Test - public void limiterWithLookahead() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void limiterWithLookahead(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(1); KeyValueCursor kvCursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -383,6 +419,7 @@ public void limiterWithLookahead() { .setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE) .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode) .build(); RecordCursor cursor = kvCursor.skip(2); // should exhaust limit first RecordCursorResult result = cursor.getNext(); @@ -393,14 +430,16 @@ public void limiterWithLookahead() { }); } - @Test - public void emptyScan() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void emptyScan(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(9))) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); RecordCursorResult result = cursor.getNext(); assertFalse(result.hasNext()); @@ -411,14 +450,16 @@ public void emptyScan() { }); } - @Test - public void emptyScanSplit() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void emptyScanSplit(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursor kvCursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(9))) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); RecordCursor cursor = new SplitHelper.KeyValueUnsplitter(context, subspace, kvCursor, false, null, false, new CursorLimitManager(context, ScanProperties.FORWARD_SCAN)); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java new file mode 100644 index 0000000000..055919d222 --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java @@ -0,0 +1,156 @@ +/* + * JoinWithLimitTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query; + +import com.apple.foundationdb.relational.api.Continuation; +import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.api.exceptions.ContextualSQLException; +import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; +import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; +import com.apple.foundationdb.relational.recordlayer.RelationalStatementRule; +import com.apple.foundationdb.relational.recordlayer.UniqueIndexTests; +import com.apple.foundationdb.relational.recordlayer.Utils; +import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.sql.SQLException; +import java.util.stream.Stream; + +public class ForceContinuationQueryTests { + + @RegisterExtension + @Order(0) + public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); + + private static final String getTemplate_definition = + "create table t1(id bigint, col1 bigint, col2 bigint, primary key(id))\n" + + "create index mv1 as select count(*) from t1\n" + + "create index mv2 as select count(*) from t1 group by col2\n" + + "create index mv3 as select count(col1) from t1\n" + + "create index mv4 as select count(col1) from t1 group by col2\n" + + + "create table t2(id bigint, col1 bigint, col2 bigint, col3 bigint, primary key(id))\n" + + "create index mv5 as select col2 from t2\n" + + "create index mv7 as select min_ever(col3) from t2\n" + + + "create table t3(id bigint, col1 bigint, col2 bigint, primary key(id))\n" + + "create index t3_i1 as select count(*) from t3\n" + + "create index t3_i2 as select count(*) from t3 group by col1\n" + + "create index t3_i3 as select count(col2) from t3\n" + + "create index t3_i4 as select count(col2) from t3 group by col1\n" + + "create index t3_i5 as select sum(col1) from t3\n" + + "create index t3_i6 as select sum(col1) from t3 group by col2"; + + @RegisterExtension + @Order(1) + public final SimpleDatabaseRule db = new SimpleDatabaseRule(UniqueIndexTests.class, getTemplate_definition); + + @RegisterExtension + @Order(2) + public final RelationalConnectionRule connection = new RelationalConnectionRule(db::getConnectionUri) + .withOptions(Options.builder().withOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true).build()) + .withSchema(db.getSchemaName()); + + @RegisterExtension + @Order(3) + public final RelationalStatementRule statement = new RelationalStatementRule(connection); + + public ForceContinuationQueryTests() throws SQLException { + } + + @BeforeAll + public static void beforeAll() { + Utils.enableCascadesDebugger(); + } + + @BeforeEach + void setup() throws Exception { + statement.execute("INSERT INTO T1 VALUES (1, 10, 1), (2, null, 2), (3, null, 2), (4, 12, 2)"); + statement.execute("INSERT INTO T2 VALUES (1, 10, 1, 11), (2, null, 2, 20), (3, null, 2, 20), (4, 12, 2, 20)"); + statement.execute("insert into t3 values (1, 2, 3), (2, 2, 3), (3, 2, 3)"); + statement.execute("delete from t3"); + } + + @ParameterizedTest + @MethodSource("failedQueries") + void testOldSerializationFails(String sql, long result) throws Exception { + Continuation continuation; + statement.setMaxRows(1); + try (var resultSet = statement.executeQuery(sql)) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(result, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + // old kvCursorContinuation cause continuation at beginning exception + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + Assertions.assertThrows(ContextualSQLException.class, preparedStatement::executeQuery); + } + } + + // disabled until SerializationMode = TO_NEW + @Disabled + @ParameterizedTest + @MethodSource("failedQueries") + void testNewSerialization(String sql, long result) throws Exception { + Continuation continuation; + try (final var s = connection.createStatement()) { + s.setMaxRows(1); + try (var resultSet = s.executeQuery(sql)) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(result, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + // new serialization fixed the issue + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertFalse(resultSet.next()); + } + } + } + + private static Stream failedQueries() { + return Stream.of( + // aggregate-index-count.yamsql + Arguments.of("select count(*) from t1", 4L), + Arguments.of("select count(col1) from t1", 2L), + // aggregate-index-tests.yamsql + Arguments.of("select min_ever(col3) from t2", 11L), + // aggregate-empty-table.yamsql + Arguments.of("select count(*) from t3", 0L), + Arguments.of("select count(col2) from t3", 0L), + Arguments.of("select sum(col1) from t3", 0L) + ); + } +} diff --git a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql index a35beadf2e..c9b2aee0c3 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql @@ -85,8 +85,6 @@ test_block: - - query: select count(*) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count_star(*) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" - # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 - result: [{4}] - - query: select count(*) from t2 group by col2 @@ -95,8 +93,6 @@ test_block: - - query: select count(col1) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count(_._0.COL1) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" - # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 - result: [{2}] - - query: select count(col1) from t2 group by col2 diff --git a/yaml-tests/src/test/resources/aggregate-index-tests.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests.yamsql index 96551b98be..c480f5c2d7 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests.yamsql @@ -112,453 +112,8 @@ setup: test_block: name: agg-index-tests tests: - - - - query: select col1, sum(col2) from T1 group by col1; - - explain: "AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 15}, {!l 20, !l 76}] - - - - query: select col1 from T1 group by col1; - - explain: "ISCAN(VI1 <,>) | MAP (_ AS _0) | AGG () GROUP BY (_._0.COL1 AS _0) | MAP (_._0._0 AS COL1)" - - result: [{!l 10}, {!l 20}] - - - - query: select sum(col2) from T1 group by col1; - - explain: "AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" - - result: [{!l 15}, {!l 76}] - - - - query: select col1, sum(col2) from T1 group by col1 order by col1 asc; - - explain: "AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 15}, {!l 20, !l 76}] - - - - query: select col1, sum(col2) from T1 group by col1 order by col1 desc; - - explain: "AISCAN(MV1 <,> BY_GROUP REVERSE -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 20, !l 76}, {!l 10, !l 15}] - - - - query: select col1, sum(col2) + 1 from T1 group by col1; - - explain: "AISCAN(MV1 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 + 1 AS _1)" - - result: [{!l 10, !l 16}, {!l 20, !l 77}] - - - - query: select col1, max(col2) from T1 group by col1; - - explain: "AISCAN(MV8 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 5}, {!l 20, !l 13}] - - - - query: select col1, max(col2) from T1 group by col1 order by col1; - - explain: "AISCAN(MV8 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 5}, {!l 20, !l 13}] - - - - query: select col1, max(col2) from T1 group by col1 order by col1 desc; - - explain: "AISCAN(MV8 <,> BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 20, !l 13}, {!l 10, !l 5}] - - - - query: select col1, max(col2) from T1 where col1 = 10 group by col1; - - explain: "AISCAN(MV8 [EQUALS promote(@c13 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 5}] - - - # At some point, should be able to roll up values from the aggregate index. However, even - # controlling for that, it can still use the index - - query: select max(col2) from T1 use index (mv8); - - explain: "ISCAN(MV8 <,>) | MAP (_ AS _0) | AGG (max_l(_._0.COL2) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)" - - result: [{!l 13}] - - - # Min/max indexes need keep what amounts to a standard value index on their keys (in order to properly look up - # the min/max). That index should be usable for normal queries just like a value index. Note that the scan is - # NOT by group - - query: select col2 from T1 where col1 = 10; - - explain: "COVERING(MV8 [EQUALS promote(@c8 AS LONG)] -> [COL1: KEY[0], COL2: KEY[1], ID: KEY[3]]) | MAP (_.COL2 AS COL2)" - - result: [{!l 1}, {!l 2}, {!l 3}, {!l 4}, {!l 5}] - - - - query: select col2 from T1 where col1 = 10 order by col2 desc; - - explain: "COVERING(MV8 [EQUALS promote(@c8 AS LONG)] REVERSE -> [COL1: KEY[0], COL2: KEY[1], ID: KEY[3]]) | MAP (_.COL2 AS COL2)" - - result: [{!l 5}, {!l 4}, {!l 3}, {!l 2}, {!l 1}] - - - - query: select min(col3) from T2 group by col1, col2; - - explain: "ISCAN(MV2 <,>) | MAP (_ AS _0) | AGG (min_l(_._0.COL3) AS _0) GROUP BY (_._0.COL1 AS _0, _._0.COL2 AS _1) | MAP (_._1._0 AS _0)" - - result: [{!l 1}, {!l 2}, {!l 3}] - - - # this should use the aggregate index in the future, for now, it is using streaming aggregate - # over base table scan. - - query: select max(col2) from t2; - - explain: "ISCAN(MV3 <,>) | MAP (_ AS _0) | AGG (max_l(_._0.COL2) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)" - - result: [{!l 2}] - - - - query: select col1, sum(col2) from T1 USE INDEX (vi1) group by col1; - - explain: "ISCAN(VI1 <,>) | MAP (_ AS _0) | AGG (sum_l(_._0.COL2) AS _0) GROUP BY (_._0.COL1 AS _0) | MAP (_._0._0 AS COL1, _._1._0 AS _1)" - - - - query: select max(col2) from t2 group by col1, col3; - - explain: "AISCAN(MV9 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._2 AS _0)" - - result: [{!l 1}, {!l 1}, {!l 2}, {!l 2}, {!l 1}, {!l 1}, {!l 1}] - - - - query: select col1, col3, max(col2) from t2 group by col1, col3; - - explain: "AISCAN(MV9 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._0 AS COL1, _._1 AS COL3, _._2 AS _2)" - - result: [{COL1: 1, COL3: 1, !l 1}, {COL1: 1, COL3: 100, !l 1}, {COL1: 1, COL3: 2, !l 2}, {COL1: 1, COL3: 200, !l 2}, {COL1: 2, COL3: 3, !l 1}, {COL1: 2, COL3: 200, !l 1}, {COL1: 2, Col3: 400, !l 1}] - - - - query: select col3, max(col2) from t2 where col1 = 2 group by col1, col3; - - explain: "AISCAN(MV9 [EQUALS promote(@c13 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS _1)" - - result: [{COL3: 3, !l 1}, {COL3: 200, !l 1}, {COL3: 400, !l 1}] - - - - query: select col3, max(col2) from t2 where col1 = 1 group by col1, col3 order by max(col2) desc; - - explain: "AISCAN(MV9 [EQUALS promote(@c13 AS LONG)] BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS _1)" - - result: [{COL3: 200, !l 2}, {COL3: 2, !l 2}, {COL3: 100, !l 1}, {COL3: 1, !l 1}] - - - - query: select col3, max(col2) from t2 where col1 = 1 group by col1, col3 order by max(col2) desc; - - maxRows: 3 - - explain: "AISCAN(MV9 [EQUALS promote(@c13 AS LONG)] BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS _1)" - - result: [{COL3: 200, !l 2}, {COL3: 2, !l 2}, {COL3: 100, !l 1}] - - result: [{COL3: 1, !l 1}] - - - - query: select col3, max(col2) from t2 where col1 = 1 group by col1, col3 having max(col2) < 2 order by max(col2) desc; - - explain: "AISCAN(MV9 [EQUALS promote(@c13 AS LONG), [LESS_THAN promote(@c25 AS LONG)]] BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS _1)" - - result: [{COL3: 100, !l 1}, {COL3: 1, !l 1}] - - - - query: select col3, max(col2) from t2 where col1 = 1 group by col1, col3 having max(col2) < 2 and max(col2) >= 1 order by max(col2) desc; - - explain: "AISCAN(MV9 [EQUALS promote(@c13 AS LONG), [GREATER_THAN_OR_EQUALS promote(@c13 AS LONG) && LESS_THAN promote(@c25 AS LONG)]] BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS _1)" - - result: [{COL3: 100, !l 1}, {COL3: 1, !l 1}] - - - - query: select t.* from (select col3, max(col2) as m from t2 where col1 = 1 group by col1, col3) as t where m < 2 order by m desc; - - explain: "AISCAN(MV9 [EQUALS promote(@c21 AS LONG), [LESS_THAN promote(@c33 AS LONG)]] BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS M)" - - result: [{COL3: 100, !l 1}, {COL3: 1, !l 1}] - - - - query: select col3, m from (select col3, max(col2) as m from t2 where col1 = 1 group by col1, col3) as t order by m desc; - - explain: "AISCAN(MV9 [EQUALS promote(@c21 AS LONG)] BY_GROUP REVERSE -> [_0: KEY:[0], _1: KEY:[2], _2: KEY:[1]]) | MAP (_._1 AS COL3, _._2 AS M)" - - result: [{COL3: 200, M: 2}, {COL3: 2, M: 2}, {COL3: 100, M: 1}, {COL3: 1, M: 1}] - - - # Permuted max index can be used as a value index on the underlying columns - - query: select col3, col2 from t2 where col1 = 1 order by col3 asc; - - explain: "COVERING(MV9 [EQUALS promote(@c10 AS LONG)] -> [COL1: KEY[0], COL2: KEY[2], COL3: KEY[1], ID: KEY[4]]) | MAP (_.COL3 AS COL3, _.COL2 AS COL2)" - - result: [{COL3: 1, COL2: 1}, {COL3: 2, COL2: 2}, {COL3: 100, COL2: 1}, {COL3: 200, COL2: 2}] - - - - query: select col3, col2 from t2 where col1 = 1 order by col3 desc; - - explain: "COVERING(MV9 [EQUALS promote(@c10 AS LONG)] REVERSE -> [COL1: KEY[0], COL2: KEY[2], COL3: KEY[1], ID: KEY[4]]) | MAP (_.COL3 AS COL3, _.COL2 AS COL2)" - - result: [{COL3: 200, COL2: 2}, {COL3: 100, COL2: 1}, {COL3: 2, COL2: 2}, {COL3: 1, COL2: 1}] - - - # Permuted max index can also be used to evaluate other aggregate functions via aggregation and roll-up - - query: select col3, sum(col2) as s from t2 use index (mv9) where col1 = 1 group by col1, col3 order by col3 asc; - - explain: "ISCAN(MV9 [EQUALS promote(@c20 AS LONG)]) | MAP (_ AS _0) | AGG (sum_l(_._0.COL2) AS _0) GROUP BY (_._0.COL1 AS _0, _._0.COL3 AS _1) | MAP (_._0._1 AS COL3, _._1._0 AS S)" - - result: [{COL3: 1, S: 1}, {COL3: 2, S: 2}, {COL3: 100, S: 1}, {COL3: 200, S: 2}] - - - - query: select col3, sum(col2) as s from t2 use index (mv9) where col1 = 1 group by col1, col3 order by col3 desc; - - explain: "ISCAN(MV9 [EQUALS promote(@c20 AS LONG)] REVERSE) | MAP (_ AS _0) | AGG (sum_l(_._0.COL2) AS _0) GROUP BY (_._0.COL1 AS _0, _._0.COL3 AS _1) | MAP (_._0._1 AS COL3, _._1._0 AS S)" - - result: [{COL3: 200, S: 2}, {COL3: 100, S: 1}, {COL3: 2, S: 2}, {COL3: 1, S: 1}] -# - -# # grouping by constant is not yet supported. -# - query: select sum(col2) from t1 group by 3,2,1; -# - result: [{!l 220}] - - - # make sure we're not choosing the aggregate index (because we aggregate by col3 not col2) - - query: select max(col3) from t2 group by col1, col3; - - explain: "ISCAN(MV3 <,>) | MAP (_ AS _0) | AGG (max_l(_._0.COL3) AS _0) GROUP BY (_._0.COL1 AS _0, _._0.COL3 AS _1) | MAP (_._1._0 AS _0)" - - - - query: select min_ever(col3) from T2 group by col1, col2; - - result: [{!l 1}, {!l 2}, {!l 3}] - - - - query: select max_ever(col3) from T2 group by col1, col2; - - result: [{!l 100}, {!l 200}, {!l 400}] - - - - query: select min_ever(col3) from t2 - # Cannot enable FORCE_CONTINUATIONS with ungrouped aggregate scan because of: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 - - explain: "AISCAN(MV7 <,> BY_GROUP -> [_0: VALUE:[0]]) | MAP (_ AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)" - - result: [{!l 1}] - - - - query: select min_ever(col3) from t2 - - explain: "AISCAN(MV7 <,> BY_GROUP -> [_0: VALUE:[0]]) | MAP (_ AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)" - - - - query: select col1, max_ever(col2) from T1 group by col1; - - explain: "AISCAN(MV6 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 5}, {!l 20, !l 13}] - - - - query: select col1, min_ever(col2) from T1 group by col1; - - explain: "AISCAN(MV12 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 10, !l 1}, {!l 20, !l 6}] - - - # TODO, check how this aligns with COLLATION support - - query: select col2, max_ever(col1) from T4 group by col2; - - explain: "AISCAN(MV15 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL2, _._1 AS _1)" - - result: [{!l 10, 'value4'}, {!l 20, 'valueZ'}] - - - # TODO, check how this aligns with COLLATION support - - query: select col2, min_ever(col1) from T4 group by col2; - - explain: "AISCAN(MV14 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL2, _._1 AS _1)" - - result: [{!l 10, 'value1'}, {!l 20, 'valueA'}] - - - - query: select col1, sum(col2) from T1 where col1 > 15 group by col1; - - explain: "AISCAN(MV1 [[GREATER_THAN promote(@c13 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - - result: [{!l 20, !l 76}] - - query: select sum(col2) from T1 where col1 = 10 group by col1 - explain: "AISCAN(MV1 [EQUALS promote(@c11 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" - result: [{!l 15}] - - - - query: select sum(col2) from T1 where col1 <= 10 group by col1 having col1 > 0; - - explain: "AISCAN(MV1 [[GREATER_THAN promote(@c19 AS LONG) && LESS_THAN_OR_EQUALS promote(@c12 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" - - result: [{!l 15}] - - - - query: select max(b) as x1, a+3 as x2 from t3 group by a+3; - - explain: "AISCAN(MV10 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._1 AS X1, _._0 AS X2)" - - result: [{x1: 500, x2: 13}, {x1: 200, x2: 23}, {x1: 900, x2: 33}] - - - - query: select a+3 as x1, max(b) as x2 from t3 group by a+3; - - explain: "AISCAN(MV10 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._0 AS X1, _._1 AS X2)" - - result: [{x1: 13, x2: 500}, {x1: 23, x2: 200}, {x1: 33, x2: 900}] - - - - query: select a+b as x1, min(b) as x2 from t3 group by a+b, b+10; - - explain: "AISCAN(MV11 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2]]) | MAP (_._0 AS X1, _._2 AS X2)" - - result: [{x1: 13, x2: 3}, {x1: 22, x2: 2}, {x1: 36, x2: 6}, {220, 200}, {510, 500}, {930, 900}] - - - - query: select M as x1, min(b) as x2 from t3 group by a+b as M, b+10; - - explain: "AISCAN(MV11 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2]]) | MAP (_._0 AS X1, _._2 AS X2)" - - result: [{x1: 13, x2: 3}, {x1: 22, x2: 2}, {x1: 36, x2: 6}, {220, 200}, {510, 500}, {930, 900}] - - - - query: select M as x1, min(b) as x2 from t3 group by a+b as M, b+10 as N; - - explain: "AISCAN(MV11 <,> BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2]]) | MAP (_._0 AS X1, _._2 AS X2)" - - result: [{x1: 13, x2: 3}, {x1: 22, x2: 2}, {x1: 36, x2: 6}, {220, 200}, {510, 500}, {930, 900}] - - - - query: select max(b) as x1, a+3 as x2 from t3 where a + 3 < 10000 group by a+3; - - explain: "AISCAN(MV10 [[LESS_THAN promote(@c21 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1]]) | MAP (_._1 AS X1, _._0 AS X2)" - - maxRows: 1 - - result: [{x1: 500, x2: 13}] - - result: [{x1: 200, x2: 23}] - - result: [{x1: 900, x2: 33}] - - result: [] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e order by b, max(x), c, d, e - - explain: "AISCAN(MV16 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - initialVersionLessThan: 4.3.5.0 - - error: "0AF00" - - initialVersionAtLeast: 4.3.5.0 - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e order by b, max(x) - - explain: "AISCAN(MV16 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e order by b, min(x), c, d, e - - explain: "AISCAN(MV17 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - initialVersionLessThan: 4.3.5.0 - - error: "0AF00" - - initialVersionAtLeast: 4.3.5.0 - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e order by b, min(x) - - explain: "AISCAN(MV17 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e order by b, c, max(x), d, e - - explain: "AISCAN(MV18 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - initialVersionLessThan: 4.3.5.0 - - error: "0AF00" - - initialVersionAtLeast: 4.3.5.0 - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e order by b, c, max(x) - - explain: "AISCAN(MV18 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e order by b, c, min(x), d, e - - explain: "AISCAN(MV19 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - initialVersionLessThan: 4.3.5.0 - - error: "0AF00" - - initialVersionAtLeast: 4.3.5.0 - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e order by b, c, min(x) - - explain: "AISCAN(MV19 [EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by max(x), c - - explain: "[IN @c34] INUNION q0 -> { AISCAN(MV16 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.B, _.D, _.E)" - - result: [ - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by min(x), c - - explain: "[IN @c34] INUNION q0 -> { AISCAN(MV17 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.B, _.D, _.E)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') and d = 1 order by max(x), c - - explain: "[IN @c34] INUNION q0 -> { AISCAN(MV16 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | FILTER _._3 EQUALS promote(@c42 AS LONG) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.B, _.D, _.E)" - - result: [ - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') and d = 1 order by min(x), c - - explain: "[IN @c34] INUNION q0 -> { AISCAN(MV17 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | FILTER _._3 EQUALS promote(@c42 AS LONG) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.B, _.D, _.E)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') and d IN (1, 2) order by max(x), c - - explain: "[IN @c34] INUNION q0 -> { [IN promote(@c42 AS ARRAY(LONG))] INUNION q1 -> { AISCAN(MV16 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | FILTER _._3 EQUALS q1 | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.D, _.E) } COMPARE BY (_._4, _.C, _.B, _.E)" - - result: [ - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') and d IN (1, 2) order by min(x), c - - explain: "[IN @c34] INUNION q0 -> { [IN promote(@c42 AS ARRAY(LONG))] INUNION q1 -> { AISCAN(MV17 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | FILTER _._3 EQUALS q1 | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.D, _.E) } COMPARE BY (_._4, _.C, _.B, _.E)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') and d IN (1, 2) order by max(x), c, e - - explain: "[IN @c34] INUNION q0 -> { [IN promote(@c42 AS ARRAY(LONG))] INUNION q1 -> { AISCAN(MV16 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | FILTER _._3 EQUALS q1 | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.E, _.D) } COMPARE BY (_._4, _.C, _.E, _.B)" - - initialVersionLessThan: 4.3.5.0 - - error: "0AF00" - - initialVersionAtLeast: 4.3.5.0 - - result: [ - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 group by a, b, c, d, e having b IN ('foo', 'bar') and d IN (1, 2) order by min(x), c, e - - explain: "[IN @c34] INUNION q0 -> { [IN promote(@c42 AS ARRAY(LONG))] INUNION q1 -> { AISCAN(MV17 [EQUALS promote(@c19 AS LONG), EQUALS q0] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[3], _3: KEY:[4], _4: KEY:[5], _5: KEY:[2]]) | FILTER _._3 EQUALS q1 | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.C, _.E, _.D) } COMPARE BY (_._4, _.C, _.E, _.B)" - - initialVersionLessThan: 4.3.5.0 - - error: "0AF00" - - initialVersionAtLeast: 4.3.5.0 - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 3, d: 2, e: "e2", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - {b: "foo", c: 3, d: 1, e: "e1", _4: 3}, - ] - - - - query: select b, c, d, e, max(x) from t5 where a = 0 and c = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by max(x), d - - explain: "[IN @c38] INUNION q0 -> { AISCAN(MV18 [EQUALS promote(@c19 AS LONG), EQUALS q0, EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.D, _.B, _.C, _.E)" - - result: [ - {b: "foo", c: 0, d: 2, e: "e2", _4: 8}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 9}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 10}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 10}, - {b: "bar", c: 0, d: 1, e: "e1", _4: 11}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 11}, - ] - - - - query: select b, c, d, e, min(x) from t5 where a = 0 and c = 0 group by a, b, c, d, e having b IN ('foo', 'bar') order by min(x), d - - explain: "[IN @c38] INUNION q0 -> { AISCAN(MV19 [EQUALS promote(@c19 AS LONG), EQUALS q0, EQUALS promote(@c19 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: KEY:[2], _3: KEY:[4], _4: KEY:[5], _5: KEY:[3]]) | MAP (_._1 AS B, _._2 AS C, _._3 AS D, _._4 AS E, _._5 AS _4) } COMPARE BY (_._4, _.D, _.B, _.C, _.E)" - - result: [ - {b: "bar", c: 0, d: 1, e: "e1", _4: 0}, - {b: "bar", c: 0, d: 1, e: "e2", _4: 0}, - {b: "foo", c: 0, d: 1, e: "e1", _4: 1}, - {b: "foo", c: 0, d: 1, e: "e2", _4: 1}, - {b: "foo", c: 0, d: 2, e: "e1", _4: 2}, - {b: "foo", c: 0, d: 2, e: "e2", _4: 3}, - ] ... diff --git a/yaml-tests/src/test/resources/disabled-planner-rewrites/aggregate-index-tests.yamsql b/yaml-tests/src/test/resources/disabled-planner-rewrites/aggregate-index-tests.yamsql index 92b9dea375..6e15d6e586 100644 --- a/yaml-tests/src/test/resources/disabled-planner-rewrites/aggregate-index-tests.yamsql +++ b/yaml-tests/src/test/resources/disabled-planner-rewrites/aggregate-index-tests.yamsql @@ -343,10 +343,6 @@ test_block: - query: select col1, sum(col2) from T1 where col1 > 15 group by col1; - explain: "AISCAN(MV1 [[GREATER_THAN promote(@c13 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._0 AS COL1, _._1 AS _1)" - result: [{!l 20, !l 76}] - - - - query: select sum(col2) from T1 where col1 = 10 group by col1 - - explain: "AISCAN(MV1 [EQUALS promote(@c11 AS LONG)] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" - - result: [{!l 15}] - - query: select sum(col2) from T1 where col1 <= 10 group by col1 having col1 > 0; - explain: "AISCAN(MV1 [[GREATER_THAN promote(@c19 AS LONG) && LESS_THAN_OR_EQUALS promote(@c12 AS LONG)]] BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)"