Merge branch 'master' into unify-get-scan-preparation-logic-for-transaction-reads-and-serializable-validation

Author: feeblefakie

.github/workflows/ci.yaml

@@ -1967,7 +1967,7 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tidb_6_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
@@ -2042,7 +2042,7 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tidb_7_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
@@ -2117,11 +2117,81 @@ jobs:
 
       - name: Upload Gradle test reports
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: tidb_8_5_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestJdbc
 
+  integration-test-for-blob-storage:
+    name: Blob Storage integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    services:
+      postgres:
+        image: mcr.microsoft.com/azure-storage/azurite
+        env:
+          AZURITE_ACCOUNTS: "test:test"
+        ports:
+          - 10000:10000
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Create Blob Storage container
+        run: |
+          az storage container create \
+            --name test-container \
+            --connection-string "DefaultEndpointsProtocol=http;AccountName=test;AccountKey=test;BlobEndpoint=http://localhost:10000/test;"
+
+      - name: Execute Gradle 'integrationTestObjectStorage' task
+        run: ./gradlew integrationTestObjectStorage -Dscalardb.object_storage.endpoint=http://localhost:10000/test/test-container -Dscalardb.object_storage.username=test -Dscalardb.object_storage.password=test ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob_storage_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestObjectStorage
+
   integration-test-for-multi-storage:
     name: Multi-storage integration test (${{ matrix.mode.label }})
     runs-on: ubuntu-latest

build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.diffplug.spotless" version "6.13.0"
-    id "org.jreleaser" version "1.20.0"
+    id "org.jreleaser" version "1.21.0"
 }
 
 ext {
@@ -28,12 +28,13 @@ subprojects {
         slf4jVersion = '1.7.36'
         cassandraDriverVersion = '3.11.5'
         azureCosmosVersion = '4.75.0'
+        azureBlobStorageVersion = '12.31.3'
         jooqVersion = '3.14.16'
-        awssdkVersion = '2.36.2'
+        awssdkVersion = '2.37.3'
         commonsDbcp2Version = '2.13.0'
         mysqlDriverVersion = '8.4.0'
         postgresqlDriverVersion = '42.7.8'
-        oracleDriverVersion = '23.9.0.25.07'
+        oracleDriverVersion = '23.26.0.0.0'
         sqlserverDriverVersion = '12.8.2.jre8'
         sqliteDriverVersion = '3.50.3.0'
         yugabyteDriverVersion = '42.7.3-yb-4'
@@ -42,7 +43,7 @@ subprojects {
         alloyDbJdbcConnectorVersion = '1.2.7'
         picocliVersion = '4.7.7'
         commonsTextVersion = '1.14.0'
-        junitVersion = '5.14.0'
+        junitVersion = '5.14.1'
         commonsLangVersion = '3.19.0'
         assertjVersion = '3.27.6'
         mockitoVersion = '4.11.0'

core/build.gradle

@@ -64,6 +64,16 @@ sourceSets {
         }
         resources.srcDir file('src/integration-test/resources')
     }
+    integrationTestObjectStorage {
+        java {
+            compileClasspath += main.output + test.output
+            runtimeClasspath += main.output + test.output
+            srcDir file('src/integration-test/java')
+            include '**/com/scalar/db/common/*.java'
+            include '**/com/scalar/db/storage/objectstorage/*.java'
+        }
+        resources.srcDir file('src/integration-test/resources')
+    }
     integrationTestMultiStorage {
         java {
             compileClasspath += main.output + test.output
@@ -136,6 +146,9 @@ configurations {
     integrationTestJdbcImplementation.extendsFrom testImplementation
     integrationTestJdbcRuntimeOnly.extendsFrom testRuntimeOnly
     integrationTestJdbcCompileOnly.extendsFrom testCompileOnly
+    integrationTestObjectStorageImplementation.extendsFrom testImplementation
+    integrationTestObjectStorageRuntimeOnly.extendsFrom testRuntimeOnly
+    integrationTestObjectStorageCompileOnly.extendsFrom testCompileOnly
     integrationTestMultiStorageImplementation.extendsFrom testImplementation
     integrationTestMultiStorageRuntimeOnly.extendsFrom testRuntimeOnly
     integrationTestMultiStorageCompileOnly.extendsFrom testCompileOnly
@@ -156,6 +169,7 @@ dependencies {
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
     implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
+    implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"
     implementation platform("software.amazon.awssdk:bom:${awssdkVersion}")
     implementation 'software.amazon.awssdk:applicationautoscaling'
@@ -254,6 +268,17 @@ task integrationTestJdbc(type: Test) {
     maxHeapSize = "4g"
 }
 
+task integrationTestObjectStorage(type: Test) {
+    description = 'Runs the integration tests for Object Storages.'
+    group = 'verification'
+    testClassesDirs = sourceSets.integrationTestObjectStorage.output.classesDirs
+    classpath = sourceSets.integrationTestObjectStorage.runtimeClasspath
+    outputs.upToDateWhen { false }  // ensures integration tests are run every time when called
+    options {
+        systemProperties(System.getProperties().findAll{it.key.toString().startsWith("scalardb")})
+    }
+}
+
 task integrationTestMultiStorage(type: Test) {
     description = 'Runs the integration tests for multi-storage.'
     group = 'verification'

core/src/integration-test/java/com/scalar/db/storage/cassandra/CassandraAdminPermissionIntegrationTest.java

@@ -126,16 +126,6 @@ protected void waitForTableDeletion() {
     }
   }
 
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in Cassandra")
-  public void getImportTableMetadata_WithSufficientPermission_ShouldSucceed() {}
-
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in Cassandra")
-  public void addRawColumnToTable_WithSufficientPermission_ShouldSucceed() {}
-
   @Test
   @Override
   @Disabled("Import-related functionality is not supported in Cassandra")

core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java

@@ -45,16 +45,6 @@ protected void sleepBetweenTests() {
     Uninterruptibles.sleepUninterruptibly(SLEEP_BETWEEN_TESTS_SECONDS, TimeUnit.SECONDS);
   }
 
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in DynamoDB")
-  public void getImportTableMetadata_WithSufficientPermission_ShouldSucceed() {}
-
-  @Test
-  @Override
-  @Disabled("Import-related functionality is not supported in DynamoDB")
-  public void addRawColumnToTable_WithSufficientPermission_ShouldSucceed() {}
-
   @Test
   @Override
   @Disabled("Import-related functionality is not supported in DynamoDB")

core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcSchemaLoaderImportIntegrationTest.java

@@ -10,6 +10,7 @@
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.DisabledIf;
 import org.slf4j.Logger;
@@ -191,6 +192,7 @@ public void importTables_ImportableTablesAndNonRelatedSameNameTableGiven_ShouldI
     super.importTables_ImportableTablesAndNonRelatedSameNameTableGiven_ShouldImportProperly();
   }
 
+  @AfterAll
   @Override
   public void afterAll() {
     try {

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java

@@ -0,0 +1,138 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
+import com.scalar.db.util.AdminTestUtils;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ConsensusCommitAdminIntegrationTestWithObjectStorage
+    extends ConsensusCommitAdminIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  protected AdminTestUtils getAdminTestUtils(String testName) {
+    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  }
+
+  @Override
+  @Disabled("Temporarily disabled because it includes DML operations")
+  public void truncateTable_ShouldTruncateProperly() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForAllDataTypesWithExistingData_ShouldCreateIndexesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_ForAlreadyExistingIndex_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void createIndex_IfNotExists_ForAlreadyExistingIndex_ShouldNotThrowAnyException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_ForAllDataTypesWithExistingData_ShouldDropIndexCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_ForNonExistingIndex_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support index-related operations")
+  public void dropIndex_IfExists_ForNonExistingIndex_ShouldNotThrowAnyException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_DropColumnForEachExistingDataType_ShouldDropColumnsCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForPrimaryKeyColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support dropping columns")
+  public void dropColumnFromTable_ForIndexedColumn_ShouldDropColumnAndIndexCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ShouldRenameColumnCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForNonExistingColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForPrimaryKeyColumn_ShouldRenameColumnCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming columns")
+  public void renameColumn_ForIndexKeyColumn_ShouldRenameColumnAndIndexCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support altering column types")
+  public void
+      alterColumnType_AlterColumnTypeFromEachExistingDataTypeToText_ShouldAlterColumnTypesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support altering column types")
+  public void alterColumnType_WideningConversion_ShouldAlterColumnTypesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support altering column types")
+  public void alterColumnType_ForPrimaryKeyOrIndexKeyColumn_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_ForExistingTable_ShouldRenameTableCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_ForNonExistingTable_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_IfNewTableNameAlreadyExists_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesCorrectly() {}
+
+  @Override
+  @Disabled("Object Storage does not support renaming tables")
+  public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
+
+  @Override
+  @Disabled("The ScalarDB environment does not need to be upgraded with Object Storage")
+  public void
+      upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairIntegrationTestWithObjectStorage.java

@@ -0,0 +1,19 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairIntegrationTestBase;
+import java.util.Properties;
+
+public class ConsensusCommitAdminRepairIntegrationTestWithObjectStorage
+    extends ConsensusCommitAdminRepairIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  protected void initialize(String testName) throws Exception {
+    super.initialize(testName);
+    adminTestUtils = new ObjectStorageAdminTestUtils(getProperties(testName));
+  }
+}