Merge branch 'main' into distinct_from

Author: pengpeng-lu

docs/sphinx/source/ReleaseNotes.md

@@ -5,6 +5,43 @@ This document contains a log of changes to the FoundationDB Record Layer. It aim
 
 As the [versioning guide](Versioning.md) details, it cannot always be determined solely by looking at the version numbers whether one Record Layer version contains all changes included in another. In particular, bug fixes and backwards-compatible changes might be back-ported to or introduced as patches against older versions. To track when a patch version has been included in the main release train, some releases will say as a note that they contain all changes from a specific patch.
 
+## 4.5
+
+### 4.5.1.0
+
+<h4> Breaking Changes </h4>
+
+* Update to minor version 4.5 in recognition of incompatibility with older 4.3 versions - [PR #3508](https://github.com/FoundationDB/fdb-record-layer/pull/3508)
+* Remove QueryHashable - [PR #3392](https://github.com/FoundationDB/fdb-record-layer/pull/3392)
+<h4> Bug Fixes </h4>
+
+* Enable AggregateCursorContinuation serialize to the new implementation - [PR #3495](https://github.com/FoundationDB/fdb-record-layer/pull/3495)
+
+<details>
+<summary>
+
+<h4> Build/Test/Documentation/Style Improvements (click to expand) </h4>
+
+</summary>
+
+* Introduce option to setup transaction before executing queries - [PR #3471](https://github.com/FoundationDB/fdb-record-layer/pull/3471)
+* Omit portions of stack trace that are just junit in test summary - [PR #3499](https://github.com/FoundationDB/fdb-record-layer/pull/3499)
+
+</details>
+
+
+**[Full Changelog (4.4.15.0...4.5.1.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.4.15.0...4.5.1.0)**
+
+#### Mixed Mode Test Results
+
+Mixed mode testing run against the following previous versions:
+
+✅`4.4.3.0`, ✅`4.4.4.0`, ❌`4.4.7.0`, ❌`4.4.8.0`, ❌`4.4.9.0`, ❌`4.4.10.0`, ❌`4.4.11.0`, ✅`4.4.12.0`, ✅`4.4.14.0`, ✅`4.4.15.0`
+
+[See full test run](https://github.com/FoundationDB/fdb-record-layer/actions/runs/16597262218)
+
+
+
 ## 4.4
 
 ### 4.4.15.0

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RelOpValue.java

@@ -1138,7 +1138,10 @@ private enum UnaryPhysicalOperator {
         IS_NOT_NULL_EI(Comparisons.Type.NOT_NULL, Type.TypeCode.ENUM, Objects::nonNull),
 
         IS_NULL_ID(Comparisons.Type.IS_NULL, Type.TypeCode.UUID, Objects::isNull),
-        IS_NOT_NULL_ID(Comparisons.Type.NOT_NULL, Type.TypeCode.UUID, Objects::nonNull);
+        IS_NOT_NULL_ID(Comparisons.Type.NOT_NULL, Type.TypeCode.UUID, Objects::nonNull),
+
+        IS_NULL_NT(Comparisons.Type.IS_NULL, Type.TypeCode.NULL, Objects::isNull),
+        IS_NOT_NULL_NT(Comparisons.Type.NOT_NULL, Type.TypeCode.NULL, Objects::nonNull);
 
         @Nonnull
         private static final Supplier<BiMap<UnaryPhysicalOperator, PUnaryPhysicalOperator>> protoEnumBiMapSupplier =

fdb-record-layer-core/src/main/proto/record_query_plan.proto

@@ -1196,6 +1196,9 @@ message PUnaryRelOpValue {
 
       IS_NULL_ID = 19;
       IS_NOT_NULL_ID = 20;
+
+      IS_NULL_NT = 21;
+      IS_NOT_NULL_NT = 22;
   }
   optional PRelOpValue super = 1;
   optional PUnaryPhysicalOperator operator = 2;

fdb-relational-cli/bin/sqlline renamed to fdb-relational-cli/bin/fdb-relational-sqlline

@@ -108,7 +108,7 @@ generateGrammarSource {
 application {
     // https://docs.gradle.org/current/userguide/application_plugin.html
     // https://github.com/julianhyde/sqlline
-    mainClass = "com.apple.relational.cli.sqlline.RelationalSQLLine"
+    mainClass = "com.apple.foundationdb.relational.cli.sqlline.RelationalSQLLine"
     applicationName = "fdb-relational-sqlline"
     // JUL => log4j2 https://logging.apache.org/log4j/2.x/log4j-jul/index.html
     // TODO: Allow overriding default fdb.cluster file with FDB_CLUSTER_FILE.

fdb-relational-cli/fdb-relational-cli.gradle

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <!--
  log4j2.xml
 
@@ -18,7 +19,6 @@
  limitations under the License.
 -->
 
-<?xml version="1.0" encoding="UTF-8"?>
 <Configuration status="INFO">
     <Appenders>
         <Console name="Console" target="SYSTEM_OUT">

fdb-relational-cli/src/main/dist/conf/log4j2.xml

@@ -22,8 +22,6 @@
 
 import com.apple.foundationdb.relational.util.ExcludeFromJacocoGeneratedReport;
 
-import java.io.IOException;
-
 /**
  * Convenience class for launching sqlline so can run it under the debugger.
  * Be aware that the intellij terminal is dodgy. See
@@ -32,10 +30,9 @@
  */
 @ExcludeFromJacocoGeneratedReport // Test utility only.
 public class LaunchSQLLine {
-    public static void main(String[] args/*Ignored*/) throws IOException {
+    public static void main(String[] args/*Ignored*/) throws Exception {
         // Read [SQLLINE-80] to see why maxWidth must be set
-        sqlline.SqlLine.main(new String []{
-                "-ac", "com.apple.foundationdb.relational.cli.sqlline.Customize",
+        com.apple.foundationdb.relational.cli.sqlline.RelationalSQLLine.main(new String []{
                 "-u", "jdbc:embed:/__SYS",
                 "--maxWidth=160",
         });

fdb-relational-cli/src/test/java/com/apple/foundationdb/relational/cli/LaunchSQLLine.java

@@ -27,15 +27,16 @@
 import com.google.common.base.Suppliers;
 import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multiset;
 import com.google.common.collect.TreeMultiset;
 import com.google.protobuf.ByteString;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -56,12 +57,13 @@ public class Literals {
 
     private Literals(@Nonnull final List<OrderedLiteral> orderedLiterals) {
         this.orderedLiterals = ImmutableList.copyOf(orderedLiterals);
+        // Using unmodifiableMap because it allows null values, which are valid here
+        // and represent either null constants or null prepared parameters in queries.
         this.asMapSupplier = Suppliers.memoize(() ->
-                this.orderedLiterals
-                        .stream()
-                        .filter(orderedLiteral -> orderedLiteral.getLiteralObject() != null)
-                        .collect(ImmutableMap.toImmutableMap(OrderedLiteral::getConstantId,
-                                OrderedLiteral::getLiteralObject)));
+                Collections.unmodifiableMap(
+                        this.orderedLiterals
+                                .stream()
+                                .collect(LinkedHashMap::new, (m, v) -> m.put(v.getConstantId(), v.getLiteralObject()), LinkedHashMap::putAll)));
     }
 
     @Nonnull

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Literals.java

@@ -27,6 +27,8 @@
 import com.apple.foundationdb.record.query.expressions.Comparisons;
 import com.apple.foundationdb.record.query.plan.QueryPlanConstraint;
 import com.apple.foundationdb.record.query.plan.cascades.Quantifier;
+import com.apple.foundationdb.record.query.plan.cascades.predicates.AndPredicate;
+import com.apple.foundationdb.record.query.plan.cascades.predicates.OrPredicate;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate;
 import com.apple.foundationdb.record.query.plan.cascades.predicates.ValuePredicate;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
@@ -164,8 +166,24 @@ private void addEqualityConstraint(@Nonnull String leftTokenId, @Nonnull String
         }
         final var leftCov = ConstantObjectValue.of(Quantifier.constant(), leftTokenId, type);
         final var rightCov = ConstantObjectValue.of(Quantifier.constant(), rightTokenId, type);
+
+        // we can replace the relatively complex predicate below with a much simpler one: (leftCov isNotDistinctFrom rightCov)
+        // once https://github.com/FoundationDB/fdb-record-layer/issues/3504 is in, but this is ok for now.
+
+        // Term1: left != null AND right != null AND left = right
+        final var leftIsNotNull = new ValuePredicate(leftCov, new Comparisons.NullComparison(Comparisons.Type.NOT_NULL));
+        final var rightIsNotNull = new ValuePredicate(rightCov, new Comparisons.NullComparison(Comparisons.Type.NOT_NULL));
         final var equalityPredicate = new ValuePredicate(leftCov, new Comparisons.ValueComparison(Comparisons.Type.EQUALS, rightCov));
-        equalityConstraints.add(equalityPredicate);
+        final var notNullComparison = AndPredicate.and(ImmutableList.of(leftIsNotNull, rightIsNotNull, equalityPredicate));
+
+        // Term2: left = null AND right = null
+        final var leftIsNull = new ValuePredicate(leftCov, new Comparisons.NullComparison(Comparisons.Type.IS_NULL));
+        final var rightIsNull = new ValuePredicate(rightCov, new Comparisons.NullComparison(Comparisons.Type.IS_NULL));
+        final var bothAreNullComparison = AndPredicate.and(leftIsNull, rightIsNull);
+
+        // Term1 OR Term2
+        final var constraint = OrPredicate.or(notNullComparison, bothAreNullComparison);
+        equalityConstraints.add(constraint);
     }
 
 

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java

@@ -20,6 +20,7 @@
 
 package com.apple.foundationdb.relational.recordlayer;
 
+import com.apple.foundationdb.relational.util.Assert;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LogEvent;
@@ -119,10 +120,16 @@ public boolean lastMessageIsCacheHit() {
     }
 
     public boolean lastMessageIsCacheMiss() {
+        if (logAppender.getLogs().isEmpty()) {
+            Assert.failUnchecked("attempt to peak logger's last message although the logger is empty (did you forget to add 'options (log query)' maybe?)");
+        }
         return getLastLogEventMessage().contains("planCache=\"miss\"");
     }
 
     public boolean lastMessageIsCacheSkip() {
+        if (logAppender.getLogs().isEmpty()) {
+            Assert.failUnchecked("attempt to peak logger's last message although the logger is empty (did you forget to add 'options (log query)' maybe?)");
+        }
         return getLastLogEventMessage().contains("planCache=\"skip\"");
     }
 }