Merge branch 'master' into fix-secondary-index-behavior

Author: brfrn169

.github/workflows/ci.yaml

@@ -254,6 +254,136 @@ jobs:
           name: cassandra_3.11_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestCassandra
 
+  integration-test-for-cassandra-4-1:
+    name: Cassandra 4.1 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:4.1
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_4.1_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
+  integration-test-for-cassandra-5-0:
+    name: Cassandra 5.0 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:5.0
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_5.0_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
   integration-test-for-cosmos:
     name: Cosmos DB integration test (${{ matrix.mode.label }})
     runs-on: windows-latest
@@ -2187,7 +2317,7 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestObjectStorage

build.gradle

@@ -26,17 +26,17 @@ subprojects {
         guiceVersion = '5.1.0'
         guavaVersion = '32.1.3-jre'
         slf4jVersion = '1.7.36'
-        cassandraDriverVersion = '3.11.5'
+        cassandraDriverVersion = '3.12.1'
         azureCosmosVersion = '4.75.0'
-        azureBlobStorageVersion = '12.31.3'
+        azureBlobStorageVersion = '12.32.0'
         jooqVersion = '3.14.16'
-        awssdkVersion = '2.37.3'
+        awssdkVersion = '2.38.2'
         commonsDbcp2Version = '2.13.0'
         mysqlDriverVersion = '8.4.0'
         postgresqlDriverVersion = '42.7.8'
         oracleDriverVersion = '23.26.0.0.0'
         sqlserverDriverVersion = '12.8.2.jre8'
-        sqliteDriverVersion = '3.50.3.0'
+        sqliteDriverVersion = '3.51.0.0'
         yugabyteDriverVersion = '42.7.3-yb-4'
         db2DriverVersion = '12.1.2.0'
         mariadDbDriverVersion = '3.5.6'

core/build.gradle

@@ -167,7 +167,7 @@ dependencies {
     implementation "com.google.guava:guava:${guavaVersion}"
     implementation "com.google.inject:guice:${guiceVersion}"
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
-    implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
+    implementation "org.apache.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
     implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminTestUtils.java

@@ -30,25 +30,26 @@ public JdbcAdminTestUtils(Properties properties) {
   @Override
   public void dropNamespacesTable() throws Exception {
     execute(
-        "DROP TABLE " + rdbEngine.encloseFullTableName(metadataSchema, JdbcAdmin.NAMESPACES_TABLE));
+        "DROP TABLE "
+            + rdbEngine.encloseFullTableName(metadataSchema, NamespaceMetadataService.TABLE_NAME));
   }
 
   @Override
   public void dropMetadataTable() throws Exception {
-    dropTable(metadataSchema, JdbcAdmin.METADATA_TABLE);
+    dropTable(metadataSchema, TableMetadataService.TABLE_NAME);
   }
 
   @Override
   public void truncateNamespacesTable() throws Exception {
     String truncateTableStatement =
-        rdbEngine.truncateTableSql(metadataSchema, JdbcAdmin.NAMESPACES_TABLE);
+        rdbEngine.truncateTableSql(metadataSchema, NamespaceMetadataService.TABLE_NAME);
     execute(truncateTableStatement);
   }
 
   @Override
   public void truncateMetadataTable() throws Exception {
     String truncateTableStatement =
-        rdbEngine.truncateTableSql(metadataSchema, JdbcAdmin.METADATA_TABLE);
+        rdbEngine.truncateTableSql(metadataSchema, TableMetadataService.TABLE_NAME);
     execute(truncateTableStatement);
   }
 
@@ -57,7 +58,7 @@ public void truncateMetadataTable() throws Exception {
   public void corruptMetadata(String namespace, String table) throws Exception {
     String insertCorruptedMetadataStatement =
         "INSERT INTO "
-            + rdbEngine.encloseFullTableName(metadataSchema, JdbcAdmin.METADATA_TABLE)
+            + rdbEngine.encloseFullTableName(metadataSchema, TableMetadataService.TABLE_NAME)
             + " VALUES ('"
             + getFullTableName(namespace, table)
             + "','corrupted','corrupted','corrupted','corrupted','0','0')";
@@ -68,9 +69,9 @@ public void corruptMetadata(String namespace, String table) throws Exception {
   public void deleteMetadata(String namespace, String table) throws Exception {
     String deleteMetadataStatement =
         "DELETE FROM "
-            + rdbEngine.encloseFullTableName(metadataSchema, JdbcAdmin.METADATA_TABLE)
+            + rdbEngine.encloseFullTableName(metadataSchema, TableMetadataService.TABLE_NAME)
             + " WHERE "
-            + rdbEngine.enclose(JdbcAdmin.METADATA_COL_FULL_TABLE_NAME)
+            + rdbEngine.enclose(TableMetadataService.COL_FULL_TABLE_NAME)
             + " = ?";
     try (Connection connection = dataSource.getConnection();
         PreparedStatement preparedStatement =

core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageAdminTestUtils.java

@@ -9,12 +9,13 @@
 import com.scalar.db.storage.cassandra.CassandraAdmin;
 import com.scalar.db.storage.cassandra.CassandraConfig;
 import com.scalar.db.storage.cassandra.ClusterManager;
-import com.scalar.db.storage.jdbc.JdbcAdmin;
 import com.scalar.db.storage.jdbc.JdbcConfig;
 import com.scalar.db.storage.jdbc.JdbcTestUtils;
 import com.scalar.db.storage.jdbc.JdbcUtils;
+import com.scalar.db.storage.jdbc.NamespaceMetadataService;
 import com.scalar.db.storage.jdbc.RdbEngineFactory;
 import com.scalar.db.storage.jdbc.RdbEngineStrategy;
+import com.scalar.db.storage.jdbc.TableMetadataService;
 import com.scalar.db.util.AdminTestUtils;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import java.sql.Connection;
@@ -61,7 +62,8 @@ public void dropNamespacesTable() throws SQLException {
     // for JDBC
     execute(
         "DROP TABLE "
-            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.NAMESPACES_TABLE));
+            + rdbEngine.encloseFullTableName(
+                jdbcMetadataSchema, NamespaceMetadataService.TABLE_NAME));
   }
 
   @Override
@@ -71,7 +73,7 @@ public void dropMetadataTable() throws SQLException {
     // for JDBC
     execute(
         "DROP TABLE "
-            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE));
+            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME));
   }
 
   @Override
@@ -86,7 +88,7 @@ public void truncateNamespacesTable() throws SQLException {
 
     // for JDBC
     String truncateTableStatement =
-        rdbEngine.truncateTableSql(jdbcMetadataSchema, JdbcAdmin.NAMESPACES_TABLE);
+        rdbEngine.truncateTableSql(jdbcMetadataSchema, NamespaceMetadataService.TABLE_NAME);
     execute(truncateTableStatement);
   }
 
@@ -97,7 +99,7 @@ public void truncateMetadataTable() throws Exception {
     // for JDBC
     String truncateTableStatement =
         "TRUNCATE TABLE "
-            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE);
+            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME);
     execute(truncateTableStatement);
   }
 
@@ -109,7 +111,7 @@ public void corruptMetadata(String namespace, String table) throws Exception {
     // for JDBC
     String insertCorruptedMetadataStatement =
         "INSERT INTO "
-            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE)
+            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME)
             + " VALUES ('"
             + getFullTableName(namespace, table)
             + "','corrupted','corrupted','corrupted','corrupted','0','0')";
@@ -123,9 +125,9 @@ public void deleteMetadata(String namespace, String table) throws Exception {
     // for JDBC
     String deleteMetadataStatement =
         "DELETE FROM "
-            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, JdbcAdmin.METADATA_TABLE)
+            + rdbEngine.encloseFullTableName(jdbcMetadataSchema, TableMetadataService.TABLE_NAME)
             + " WHERE "
-            + rdbEngine.enclose(JdbcAdmin.METADATA_COL_FULL_TABLE_NAME)
+            + rdbEngine.enclose(TableMetadataService.COL_FULL_TABLE_NAME)
             + " = ?";
     try (Connection connection = dataSource.getConnection();
         PreparedStatement preparedStatement =

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java

@@ -1,5 +1,6 @@
 package com.scalar.db.storage.objectstorage;
 
+import com.scalar.db.api.TableMetadata;
 import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
 import com.scalar.db.util.AdminTestUtils;
 import java.util.Properties;
@@ -14,13 +15,17 @@ protected Properties getProps(String testName) {
   }
 
   @Override
-  protected AdminTestUtils getAdminTestUtils(String testName) {
-    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  protected TableMetadata getTableMetadata() {
+    return TableMetadata.newBuilder(TABLE_METADATA)
+        .removeSecondaryIndex(COL_NAME5)
+        .removeSecondaryIndex(COL_NAME6)
+        .build();
   }
 
   @Override
-  @Disabled("Temporarily disabled because it includes DML operations")
-  public void truncateTable_ShouldTruncateProperly() {}
+  protected AdminTestUtils getAdminTestUtils(String testName) {
+    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  }
 
   @Override
   @Disabled("Object Storage does not support index-related operations")

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairIntegrationTestWithObjectStorage.java

@@ -1,11 +1,39 @@
 package com.scalar.db.storage.objectstorage;
 
+import com.scalar.db.api.Scan;
+import com.scalar.db.api.TableMetadata;
+import com.scalar.db.io.DataType;
 import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairIntegrationTestBase;
 import java.util.Properties;
 
 public class ConsensusCommitAdminRepairIntegrationTestWithObjectStorage
     extends ConsensusCommitAdminRepairIntegrationTestBase {
 
+  @Override
+  protected TableMetadata getTableMetadata() {
+    return TableMetadata.newBuilder()
+        .addColumn(COL_NAME1, DataType.INT)
+        .addColumn(COL_NAME2, DataType.TEXT)
+        .addColumn(COL_NAME3, DataType.TEXT)
+        .addColumn(COL_NAME4, DataType.INT)
+        .addColumn(COL_NAME5, DataType.INT)
+        .addColumn(COL_NAME6, DataType.TEXT)
+        .addColumn(COL_NAME7, DataType.BIGINT)
+        .addColumn(COL_NAME8, DataType.FLOAT)
+        .addColumn(COL_NAME9, DataType.DOUBLE)
+        .addColumn(COL_NAME10, DataType.BOOLEAN)
+        .addColumn(COL_NAME11, DataType.BLOB)
+        .addColumn(COL_NAME12, DataType.DATE)
+        .addColumn(COL_NAME13, DataType.TIME)
+        .addColumn(COL_NAME14, DataType.TIMESTAMPTZ)
+        .addColumn(COL_NAME15, DataType.TIMESTAMP)
+        .addPartitionKey(COL_NAME2)
+        .addPartitionKey(COL_NAME1)
+        .addClusteringKey(COL_NAME4, Scan.Ordering.Order.ASC)
+        .addClusteringKey(COL_NAME3, Scan.Ordering.Order.DESC)
+        .build();
+  }
+
   @Override
   protected Properties getProps(String testName) {
     return ObjectStorageEnv.getProperties(testName);

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage.java

@@ -0,0 +1,23 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitCrossPartitionScanIntegrationTestBase;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage
+    extends ConsensusCommitCrossPartitionScanIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    Properties properties = ConsensusCommitObjectStorageEnv.getProperties(testName);
+    properties.setProperty(ConsensusCommitConfig.ISOLATION_LEVEL, "SERIALIZABLE");
+    return properties;
+  }
+
+  @Test
+  @Override
+  @Disabled("Cross-partition scan with ordering is not supported in Object Storage")
+  public void scan_CrossPartitionScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() {}
+}