Merge branch 'master' into support_cassy_4_5

Author: feeblefakie

.github/workflows/ci.yaml

@@ -2097,7 +2097,7 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tidb_6_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
@@ -2172,7 +2172,7 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tidb_7_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
@@ -2247,11 +2247,81 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tidb_8_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
 
+  integration-test-for-blob-storage:
+    name: Blob Storage integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    services:
+      postgres:
+        image: mcr.microsoft.com/azure-storage/azurite
+        env:
+          AZURITE_ACCOUNTS: "test:test"
+        ports:
+          - 10000:10000
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Create Blob Storage container
+        run: |
+          az storage container create \
+            --name test-container \
+            --connection-string "DefaultEndpointsProtocol=http;AccountName=test;AccountKey=test;BlobEndpoint=http://localhost:10000/test;"
+
+      - name: Execute Gradle 'integrationTestObjectStorage' task
+        run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.endpoint=http://localhost:10000/test/test-container -Dscalardb.object_storage.username=test -Dscalardb.object_storage.password=test ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestObjectStorage
+
   integration-test-for-multi-storage:
     name: Multi-storage integration test (${{ matrix.mode.label }})
     runs-on: ubuntu-latest

build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.diffplug.spotless" version "6.13.0"
-    id "org.jreleaser" version "1.20.0"
+    id "org.jreleaser" version "1.21.0"
 }
 
 ext {
@@ -28,21 +28,22 @@ subprojects {
         slf4jVersion = '1.7.36'
         cassandraDriverVersion = '3.12.1'
         azureCosmosVersion = '4.75.0'
+        azureBlobStorageVersion = '12.32.0'
         jooqVersion = '3.14.16'
-        awssdkVersion = '2.36.2'
+        awssdkVersion = '2.38.2'
         commonsDbcp2Version = '2.13.0'
         mysqlDriverVersion = '8.4.0'
         postgresqlDriverVersion = '42.7.8'
-        oracleDriverVersion = '23.9.0.25.07'
+        oracleDriverVersion = '23.26.0.0.0'
         sqlserverDriverVersion = '12.8.2.jre8'
-        sqliteDriverVersion = '3.50.3.0'
+        sqliteDriverVersion = '3.51.0.0'
         yugabyteDriverVersion = '42.7.3-yb-4'
         db2DriverVersion = '12.1.2.0'
         mariadDbDriverVersion = '3.5.6'
         alloyDbJdbcConnectorVersion = '1.2.7'
         picocliVersion = '4.7.7'
         commonsTextVersion = '1.14.0'
-        junitVersion = '5.14.0'
+        junitVersion = '5.14.1'
         commonsLangVersion = '3.19.0'
         assertjVersion = '3.27.6'
         mockitoVersion = '4.11.0'
@@ -108,7 +109,8 @@ jreleaser {
             def stagingRepositories = ['core/build/staging-deploy',
                                        'integration-test/build/staging-deploy',
                                        'schema-loader/build/staging-deploy',
-                                       'data-loader/core/build/staging-deploy']
+                                       'data-loader/core/build/staging-deploy',
+                                       'data-loader/cli/build/staging-deploy']
             mavenCentral {
                 sonatype {
                     active = 'RELEASE'

core/build.gradle

@@ -64,6 +64,16 @@ sourceSets {
         }
         resources.srcDir file('src/integration-test/resources')
     }
+    integrationTestObjectStorage {
+        java {
+            compileClasspath += main.output + test.output
+            runtimeClasspath += main.output + test.output
+            srcDir file('src/integration-test/java')
+            include '**/com/scalar/db/common/*.java'
+            include '**/com/scalar/db/storage/objectstorage/*.java'
+        }
+        resources.srcDir file('src/integration-test/resources')
+    }
     integrationTestMultiStorage {
         java {
             compileClasspath += main.output + test.output
@@ -136,6 +146,9 @@ configurations {
     integrationTestJdbcImplementation.extendsFrom testImplementation
     integrationTestJdbcRuntimeOnly.extendsFrom testRuntimeOnly
     integrationTestJdbcCompileOnly.extendsFrom testCompileOnly
+    integrationTestObjectStorageImplementation.extendsFrom testImplementation
+    integrationTestObjectStorageRuntimeOnly.extendsFrom testRuntimeOnly
+    integrationTestObjectStorageCompileOnly.extendsFrom testCompileOnly
     integrationTestMultiStorageImplementation.extendsFrom testImplementation
     integrationTestMultiStorageRuntimeOnly.extendsFrom testRuntimeOnly
     integrationTestMultiStorageCompileOnly.extendsFrom testCompileOnly
@@ -156,6 +169,7 @@ dependencies {
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
     implementation "org.apache.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
+    implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"
     implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
     implementation 'software.amazon.awssdk:applicationautoscaling'
@@ -174,7 +188,9 @@ dependencies {
         exclude group: 'org.slf4j', module: 'slf4j-api'
     }
     // The Java connector library used to establish simply a secure connection to AlloyDB on GCP
-    implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}")
+    implementation("com.google.cloud:alloydb-jdbc-connector:${alloyDbJdbcConnectorVersion}") {
+        exclude group: 'org.slf4j', module: 'slf4j-api'
+    }
     implementation "org.apache.commons:commons-text:${commonsTextVersion}"
     testImplementation platform("org.junit:junit-bom:${junitVersion}")
     testImplementation 'org.junit.jupiter:junit-jupiter'
@@ -251,6 +267,18 @@ task integrationTestJdbc(type: Test) {
     options {
         systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
     }
+    maxHeapSize = "4g"
+}
+
+task integrationTestObjectStorage(type: Test) {
+    description = 'Runs the integration tests for Object Storages.'
+    group = 'verification'
+    testClassesDirs = sourceSets.integrationTestObjectStorage.output.classesDirs
+    classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
+    outputs.upToDateWhen { false }  // ensures integration tests are run every time when called
+    options {
+        systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
+    }
 }
 
 task integrationTestMultiStorage(type: Test) {

core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraAdminPermissionIntegrationTest.java

@@ -126,16 +126,6 @@ protected void waitForTableDeletion() {
     }
   }
 
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in Cassandra")
-  public void getImportTableMetadata_WithSufficientPermission_ShouldSucceed() {}
-
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in Cassandra")
-  public void addRawColumnToTable_WithSufficientPermission_ShouldSucceed() {}
-
   @Test
   @Override
   @Disabled("Import-related functionality is not supported in Cassandra")

core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraAdminTestUtils.java

@@ -57,6 +57,11 @@ public void corruptMetadata(String namespace, String table) {
     // Do nothing
   }
 
+  @Override
+  public void deleteMetadata(String namespace, String table) throws Exception {
+    // Do nothing
+  }
+
   @Override
   public void dropNamespace(String namespace) {
     String dropKeyspaceQuery =

core/src/integration-test/java/com/scalar/db/storage/cosmos/CosmosAdminTestUtils.java

@@ -86,6 +86,15 @@ public void corruptMetadata(String namespace, String table) {
     container.upsertItem(corruptedMetadata);
   }
 
+  @Override
+  public void deleteMetadata(String namespace, String table) {
+    String fullTableName = getFullTableName(namespace, table);
+    CosmosContainer container =
+        client.getDatabase(metadataDatabase).getContainer(CosmosAdmin.TABLE_METADATA_CONTAINER);
+    container.deleteItem(
+        fullTableName, new PartitionKey(fullTableName), new CosmosItemRequestOptions());
+  }
+
   /**
    * Retrieve the stored procedure for the given table
    *

core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminCaseSensitivityIntegrationTest.java

@@ -29,6 +29,11 @@ protected AdminTestUtils getAdminTestUtils(String testName) {
     return new DynamoAdminTestUtils(getProperties(testName));
   }
 
+  @Override
+  @Disabled("DynamoDB does not have a concept of namespaces")
+  public void
+      dropNamespace_ForNamespaceWithNonScalarDBManagedTables_ShouldThrowIllegalArgumentException() {}
+
   @Override
   @Disabled("DynamoDB does not support dropping columns")
   public void dropColumnFromTable_DropColumnForEachExistingDataType_ShouldDropColumnsCorrectly() {}

core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminIntegrationTest.java

@@ -28,6 +28,11 @@ protected AdminTestUtils getAdminTestUtils(String testName) {
     return new DynamoAdminTestUtils(getProperties(testName));
   }
 
+  @Override
+  @Disabled("DynamoDB does not have a concept of namespaces")
+  public void
+      dropNamespace_ForNamespaceWithNonScalarDBManagedTables_ShouldThrowIllegalArgumentException() {}
+
   @Override
   @Disabled("DynamoDB does not support dropping columns")
   public void dropColumnFromTable_DropColumnForEachExistingDataType_ShouldDropColumnsCorrectly() {}

core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java

@@ -45,16 +45,6 @@ protected void sleepBetweenTests() {
     Uninterruptibles.sleepUninterruptibly(SLEEP_BETWEEN_TESTS_SECONDS, TimeUnit.SECONDS);
   }
 
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in DynamoDB")
-  public void getImportTableMetadata_WithSufficientPermission_ShouldSucceed() {}
-
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in DynamoDB")
-  public void addRawColumnToTable_WithSufficientPermission_ShouldSucceed() {}
-
   @Test
   @Override
   @Disabled("Import-related functionality is not supported in DynamoDB")

core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminTestUtils.java

@@ -171,6 +171,20 @@ public void corruptMetadata(String namespace, String table) {
             .build());
   }
 
+  @Override
+  public void deleteMetadata(String namespace, String table) {
+    String fullTableName =
+        getFullTableName(Namespace.of(namespacePrefix, namespace).prefixed(), table);
+    Map<String, AttributeValue> keyToDelete = new HashMap<>();
+    keyToDelete.put("table", AttributeValue.builder().s(fullTableName).build());
+
+    client.deleteItem(
+        DeleteItemRequest.builder()
+            .tableName(getFullTableName(metadataNamespace, DynamoAdmin.METADATA_TABLE))
+            .key(keyToDelete)
+            .build());
+  }
+
   @Override
   public void dropTable(String nonPrefixedNamespace, String table) {
     String namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace).prefixed();