Merge branch 'master' of https://github.com/scalar-labs/scalardb into fix/data-loader/handle-unkown-exception-when-importing

Author: thongdk8

.github/workflows/ci.yaml

@@ -254,6 +254,136 @@ jobs:
           name: cassandra_3.11_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestCassandra
 
+  integration-test-for-cassandra-4-1:
+    name: Cassandra 4.1 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:4.1
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_4.1_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
+  integration-test-for-cassandra-5-0:
+    name: Cassandra 5.0 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:5.0
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_5.0_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
   integration-test-for-cosmos:
     name: Cosmos DB integration test (${{ matrix.mode.label }})
     runs-on: windows-latest

.github/workflows/object-storage-adapter-check.yaml

@@ -0,0 +1,102 @@
+name: Test Object Storage Adapter
+
+on:
+  workflow_dispatch:
+    inputs:
+      INT_TEST_JAVA_RUNTIME_VERSION:
+        description: JDK version used to run the integration test
+        type: choice
+        required: false
+        default: '8'
+        options:
+          - '8'
+          - '11'
+          - '17'
+          - '21'
+      INT_TEST_JAVA_RUNTIME_VENDOR:
+        description: Vendor of the JDK used to run the integration test
+        type: choice
+        required: false
+        default: 'temurin'
+        options:
+          - 'corretto'
+          - 'microsoft'
+          - 'oracle'
+          - 'temurin'
+
+env:
+  TERM: dumb
+  JAVA_VERSION: '8'
+  JAVA_VENDOR: 'temurin'
+  INT_TEST_JAVA_RUNTIME_VERSION: "${{ github.event_name != 'workflow_dispatch' && '8' || inputs.INT_TEST_JAVA_RUNTIME_VERSION }}"
+  INT_TEST_JAVA_RUNTIME_VENDOR: "${{ github.event_name != 'workflow_dispatch' && 'temurin' || inputs.INT_TEST_JAVA_RUNTIME_VENDOR }}"
+  # Gradle will parse 'ORG_GRADLE_PROJECT_<project_property_name>' environment variables as project properties.
+  # The following variables configure the 'com.scalar.db.jdk-configuration' Gradle plugin.
+  ORG_GRADLE_PROJECT_javaVersion: '8'
+  ORG_GRADLE_PROJECT_javaVendor: 'temurin'
+  ORG_GRADLE_PROJECT_integrationTestJavaRuntimeVersion: "${{ github.event_name != 'workflow_dispatch' && '8' || inputs.INT_TEST_JAVA_RUNTIME_VERSION }}"
+  ORG_GRADLE_PROJECT_integrationTestJavaRuntimeVendor: "${{ github.event_name != 'workflow_dispatch' && 'temurin' || inputs.INT_TEST_JAVA_RUNTIME_VENDOR }}"
+  # This variable evaluates to: if {!(Temurin JDK 8) && !(Oracle JDK)} then {true} else {false}
+  # Oracle JDK that are linux compatible and publicly available through direct download exist for all LTS versions
+  SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK: "${{ (github.event_name == 'workflow_dispatch' && !(inputs.INT_TEST_JAVA_RUNTIME_VERSION == '8' && inputs.INT_TEST_JAVA_RUNTIME_VENDOR == 'temurin') && !(inputs.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle')) && 'true' || 'false' }}"
+  INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT: '"-Dscalardb.consensus_commit.coordinator.group_commit.enabled=true" "-Dscalardb.consensus_commit.coordinator.group_commit.old_group_abort_timeout_millis=15000" --tests "**.ConsensusCommit**"'
+  AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
+  S3_REGION: ap-northeast-1
+  S3_BUCKET_NAME: scalardb-test-bucket
+
+jobs:
+  integration-test-s3:
+    name: S3 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestObjectStorage' task
+        run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.storage=s3 -Dscalardb.object_storage.endpoint=${{ env.S3_REGION }}/${{ env.S3_BUCKET_NAME }} ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_3.0_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestObjectStorage

build.gradle

@@ -26,7 +26,7 @@ subprojects {
         guiceVersion = '5.1.0'
         guavaVersion = '32.1.3-jre'
         slf4jVersion = '1.7.36'
-        cassandraDriverVersion = '3.11.5'
+        cassandraDriverVersion = '3.12.1'
         azureCosmosVersion = '4.75.0'
         azureBlobStorageVersion = '12.32.0'
         jooqVersion = '3.14.16'
@@ -38,13 +38,13 @@ subprojects {
         sqlserverDriverVersion = '12.8.2.jre8'
         sqliteDriverVersion = '3.51.0.0'
         yugabyteDriverVersion = '42.7.3-yb-4'
-        db2DriverVersion = '12.1.2.0'
+        db2DriverVersion = '12.1.3.0'
         mariadDbDriverVersion = '3.5.6'
-        alloyDbJdbcConnectorVersion = '1.2.7'
+        alloyDbJdbcConnectorVersion = '1.2.8'
         picocliVersion = '4.7.7'
         commonsTextVersion = '1.14.0'
         junitVersion = '5.14.1'
-        commonsLangVersion = '3.19.0'
+        commonsLangVersion = '3.20.0'
         assertjVersion = '3.27.6'
         mockitoVersion = '4.11.0'
         spotbugsVersion = '4.8.6'

core/build.gradle

@@ -167,13 +167,15 @@ dependencies {
     implementation "com.google.guava:guava:${guavaVersion}"
     implementation "com.google.inject:guice:${guiceVersion}"
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
-    implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
+    implementation "org.apache.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
     implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"
     implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
     implementation 'software.amazon.awssdk:applicationautoscaling'
     implementation 'software.amazon.awssdk:dynamodb'
+    implementation 'software.amazon.awssdk:s3'
+    implementation 'software.amazon.awssdk:aws-crt-client'
     testImplementation 'software.amazon.awssdk:iam'
     testImplementation 'software.amazon.awssdk:iam-policy-builder'
     implementation "org.apache.commons:commons-dbcp2:${commonsDbcp2Version}"

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminTestUtils.java

@@ -90,7 +90,7 @@ private void execute(String sql) throws SQLException {
   @Override
   public boolean tableExists(String namespace, String table) throws Exception {
     String fullTableName = rdbEngine.encloseFullTableName(namespace, table);
-    String sql = rdbEngine.tableExistsInternalTableCheckSql(fullTableName);
+    String sql = rdbEngine.internalTableExistsCheckSql(fullTableName);
     try (Connection connection = dataSource.getConnection();
         Statement statement = connection.createStatement()) {
       statement.execute(sql);

core/src/integration-test/java/com/scalar/db/storage/multistorage/MultiStorageAdminTestUtils.java

@@ -227,7 +227,7 @@ private boolean tableExistsOnCassandra(String namespace, String table) {
 
   private boolean tableExistsOnJdbc(String namespace, String table) throws Exception {
     String fullTableName = rdbEngine.encloseFullTableName(namespace, table);
-    String sql = rdbEngine.tableExistsInternalTableCheckSql(fullTableName);
+    String sql = rdbEngine.internalTableExistsCheckSql(fullTableName);
     try (Connection connection = dataSource.getConnection();
         Statement statement = connection.createStatement()) {
       statement.execute(sql);

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java

@@ -27,10 +27,6 @@ protected AdminTestUtils getAdminTestUtils(String testName) {
     return new ObjectStorageAdminTestUtils(getProperties(testName));
   }
 
-  @Override
-  @Disabled("Temporarily disabled because it includes DML operations")
-  public void truncateTable_ShouldTruncateProperly() {}
-
   @Override
   @Disabled("Object Storage does not support index-related operations")
   public void createIndex_ForAllDataTypesWithExistingData_ShouldCreateIndexesCorrectly() {}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage.java

@@ -0,0 +1,23 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitCrossPartitionScanIntegrationTestBase;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class ConsensusCommitCrossPartitionScanIntegrationTestWithObjectStorage
+    extends ConsensusCommitCrossPartitionScanIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    Properties properties = ConsensusCommitObjectStorageEnv.getProperties(testName);
+    properties.setProperty(ConsensusCommitConfig.ISOLATION_LEVEL, "SERIALIZABLE");
+    return properties;
+  }
+
+  @Test
+  @Override
+  @Disabled("Cross-partition scan with ordering is not supported in Object Storage")
+  public void scan_CrossPartitionScanWithOrderingGivenForCommittedRecord_ShouldReturnRecords() {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitIntegrationTestWithObjectStorage.java

@@ -0,0 +1,51 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.api.TableMetadata;
+import com.scalar.db.io.DataType;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitIntegrationTestBase;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ConsensusCommitIntegrationTestWithObjectStorage
+    extends ConsensusCommitIntegrationTestBase {
+
+  @Override
+  protected TableMetadata getTableMetadata() {
+    return TableMetadata.newBuilder()
+        .addColumn(ACCOUNT_ID, DataType.INT)
+        .addColumn(ACCOUNT_TYPE, DataType.INT)
+        .addColumn(BALANCE, DataType.INT)
+        .addColumn(SOME_COLUMN, DataType.INT)
+        .addColumn(BOOLEAN_COL, DataType.BOOLEAN)
+        .addColumn(BIGINT_COL, DataType.BIGINT)
+        .addColumn(FLOAT_COL, DataType.FLOAT)
+        .addColumn(DOUBLE_COL, DataType.DOUBLE)
+        .addColumn(TEXT_COL, DataType.TEXT)
+        .addColumn(BLOB_COL, DataType.BLOB)
+        .addColumn(DATE_COL, DataType.DATE)
+        .addColumn(TIME_COL, DataType.TIME)
+        .addColumn(TIMESTAMPTZ_COL, DataType.TIMESTAMPTZ)
+        .addColumn(TIMESTAMP_COL, DataType.TIMESTAMP)
+        .addPartitionKey(ACCOUNT_ID)
+        .addClusteringKey(ACCOUNT_TYPE)
+        .build();
+  }
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ConsensusCommitObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void get_GetGivenForIndexColumn_ShouldReturnRecords() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void scanOrGetScanner_ScanGivenForIndexColumn_ShouldReturnRecords(ScanType scanType) {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void scanOrGetScanner_ScanGivenForIndexColumnWithConjunctions_ShouldReturnRecords(
+      ScanType scanType) {}
+}