Merge branch 'master' into validate-secondary-index-with-object-storage

Author: feeblefakie

.github/workflows/ci.yaml

@@ -2187,7 +2187,7 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestObjectStorage

build.gradle

@@ -28,15 +28,15 @@ subprojects {
         slf4jVersion = '1.7.36'
         cassandraDriverVersion = '3.11.5'
         azureCosmosVersion = '4.75.0'
-        azureBlobStorageVersion = '12.31.3'
+        azureBlobStorageVersion = '12.32.0'
         jooqVersion = '3.14.16'
-        awssdkVersion = '2.37.3'
+        awssdkVersion = '2.38.2'
         commonsDbcp2Version = '2.13.0'
         mysqlDriverVersion = '8.4.0'
         postgresqlDriverVersion = '42.7.8'
         oracleDriverVersion = '23.26.0.0.0'
         sqlserverDriverVersion = '12.8.2.jre8'
-        sqliteDriverVersion = '3.50.3.0'
+        sqliteDriverVersion = '3.51.0.0'
         yugabyteDriverVersion = '42.7.3-yb-4'
         db2DriverVersion = '12.1.2.0'
         mariadDbDriverVersion = '3.5.6'
@@ -109,7 +109,8 @@ jreleaser {
             def stagingRepositories = ['core/build/staging-deploy',
                                        'integration-test/build/staging-deploy',
                                        'schema-loader/build/staging-deploy',
-                                       'data-loader/core/build/staging-deploy']
+                                       'data-loader/core/build/staging-deploy',
+                                       'data-loader/cli/build/staging-deploy']
             mavenCentral {
                 sonatype {
                     active = 'RELEASE'

core/build.gradle

@@ -188,7 +188,9 @@ dependencies {
         exclude group: 'org.slf4j', module: 'slf4j-api'
     }
     // The Java connector library used to establish simply a secure connection to AlloyDB on GCP
-    implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}")
+    implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}") {
+        exclude group: 'org.slf4j', module: 'slf4j-api'
+    }
     implementation "org.apache.commons:commons-text:${commonsTextVersion}"
     testImplementation platform("org.junit:junit-bom:${junitVersion}")
     testImplementation 'org.junit.jupiter:junit-jupiter'

core/src/main/java/com/scalar/db/transaction/consensuscommit/ConsensusCommitUtils.java

@@ -1,12 +1,22 @@
 package com.scalar.db.transaction.consensuscommit;
 
 import com.google.common.collect.ImmutableMap;
+import com.scalar.db.api.AndConditionSet;
 import com.scalar.db.api.ConditionBuilder;
+import com.scalar.db.api.ConditionSetBuilder;
+import com.scalar.db.api.ConditionalExpression;
+import com.scalar.db.api.Consistency;
+import com.scalar.db.api.Get;
+import com.scalar.db.api.GetBuilder;
 import com.scalar.db.api.Insert;
+import com.scalar.db.api.LikeExpression;
 import com.scalar.db.api.MutationCondition;
 import com.scalar.db.api.Operation;
 import com.scalar.db.api.Put;
 import com.scalar.db.api.PutBuilder;
+import com.scalar.db.api.Scan;
+import com.scalar.db.api.ScanBuilder;
+import com.scalar.db.api.Selection;
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.api.Update;
 import com.scalar.db.api.UpdateIf;
@@ -352,4 +362,149 @@ static TransactionTableMetadata getTransactionTableMetadata(
     }
     return metadata;
   }
+
+  static Get prepareGetForStorage(Get get, TableMetadata metadata) {
+    GetBuilder.BuildableGetOrGetWithIndexFromExisting builder =
+        Get.newBuilder(get).clearProjections().consistency(Consistency.LINEARIZABLE);
+
+    if (!get.getConjunctions().isEmpty()) {
+      // If there are conjunctions, we need to convert them to include conditions on the before
+      // image
+      Set<AndConditionSet> converted = convertConjunctions(get.getConjunctions(), metadata);
+      return builder.clearConditions().whereOr(converted).build();
+    }
+
+    return builder.build();
+  }
+
+  static Scan prepareScanForStorage(Scan scan, TableMetadata metadata) {
+    ScanBuilder.BuildableScanOrScanAllFromExisting builder =
+        Scan.newBuilder(scan).clearProjections().consistency(Consistency.LINEARIZABLE);
+
+    if (scan.getLimit() > 0) {
+      // Since the recovery process and the conjunction processing may exclude some records from
+      // the scan result, it is necessary to perform the scan without a limit.
+      builder.limit(0);
+    }
+
+    if (!scan.getConjunctions().isEmpty()) {
+      // If there are conjunctions, we need to convert them to include conditions on the before
+      // image
+      Set<AndConditionSet> converted = convertConjunctions(scan.getConjunctions(), metadata);
+      return builder.clearConditions().whereOr(converted).build();
+    }
+
+    return builder.build();
+  }
+
+  /**
+   * Converts the given conjunctions to include conditions on before images.
+   *
+   * <p>This is necessary because we might miss prepared records whose before images match the
+   * original conditions when reading from storage. For example, suppose we have the following
+   * records in storage:
+   *
+   * <pre>
+   *   | partition_key | clustering_key | column | status    | before_column | before_status  |
+   *   |---------------|----------------|--------|-----------|---------------|----------------|
+   *   | 0             | 0              | 1000   | COMMITTED |               |                |
+   *   | 0             | 1              | 200    | PREPARED  | 1000          | COMMITTED      |
+   * </pre>
+   *
+   * If we scan records with the condition "column = 1000" without converting the condition
+   * (conjunction), we only get the first record, not the second one, because the condition does not
+   * match. However, the second record has not been committed yet, so we should still retrieve it,
+   * considering the possibility that the record will be rolled back.
+   *
+   * <p>To handle such cases, we convert the conjunctions to include conditions on the before image.
+   * For example, if the original condition is:
+   *
+   * <pre>
+   *   column = 1000
+   * </pre>
+   *
+   * We convert it to:
+   *
+   * <pre>
+   *   column = 1000 OR before_column = 1000
+   * </pre>
+   *
+   * <p>Here are more examples:
+   *
+   * <p>Example 1:
+   *
+   * <pre>
+   *   {@code column >= 500 AND column < 1000}
+   * </pre>
+   *
+   * becomes:
+   *
+   * <pre>
+   *   {@code (column >= 500 AND column < 1000) OR (before_column >= 500 AND before_column < 1000)}
+   * </pre>
+   *
+   * <p>Example 2:
+   *
+   * <pre>
+   *   {@code column1 = 500 OR column2 != 1000}
+   * </pre>
+   *
+   * becomes:
+   *
+   * <pre>
+   *   {@code column1 = 500 OR column2 != 1000 OR before_column1 = 500 OR before_column2 != 1000}
+   * </pre>
+   *
+   * This way, we can ensure that prepared records whose before images satisfy the original scan
+   * conditions are not missed during the scan.
+   *
+   * @param conjunctions the conjunctions to convert
+   * @param metadata the table metadata of the target table
+   * @return the converted conjunctions
+   */
+  private static Set<AndConditionSet> convertConjunctions(
+      Set<Selection.Conjunction> conjunctions, TableMetadata metadata) {
+    Set<AndConditionSet> converted = new HashSet<>(conjunctions.size() * 2);
+
+    // Keep the original conjunctions
+    conjunctions.forEach(
+        c -> converted.add(ConditionSetBuilder.andConditionSet(c.getConditions()).build()));
+
+    // Add conditions on the before image
+    for (Selection.Conjunction conjunction : conjunctions) {
+      Set<ConditionalExpression> conditions = new HashSet<>(conjunction.getConditions().size());
+      for (ConditionalExpression condition : conjunction.getConditions()) {
+        String columnName = condition.getColumn().getName();
+
+        if (metadata.getPartitionKeyNames().contains(columnName)
+            || metadata.getClusteringKeyNames().contains(columnName)) {
+          // If the condition is on the primary key, we don't need to convert it
+          conditions.add(condition);
+          continue;
+        }
+
+        // Convert the condition to use the before image column
+        ConditionalExpression convertedCondition;
+        if (condition instanceof LikeExpression) {
+          LikeExpression likeExpression = (LikeExpression) condition;
+          convertedCondition =
+              ConditionBuilder.buildLikeExpression(
+                  likeExpression.getColumn().copyWith(Attribute.BEFORE_PREFIX + columnName),
+                  likeExpression.getOperator(),
+                  likeExpression.getEscape());
+        } else {
+          convertedCondition =
+              ConditionBuilder.buildConditionalExpression(
+                  condition.getColumn().copyWith(Attribute.BEFORE_PREFIX + columnName),
+                  condition.getOperator());
+        }
+
+        conditions.add(convertedCondition);
+      }
+
+      converted.add(ConditionSetBuilder.andConditionSet(conditions).build());
+    }
+
+    return converted;
+  }
 }