Merge branch 'master' into feat/data-loader/changes-export

Author: inv-jishnu

.github/workflows/ci.yaml

@@ -254,6 +254,136 @@ jobs:
           name: cassandra_3.11_integration_test_reports_${{ matrix.mode.label }}
           path: core/build/reports/tests/integrationTestCassandra
 
+  integration-test-for-cassandra-4-1:
+    name: Cassandra 4.1 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:4.1
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_4.1_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
+  integration-test-for-cassandra-5-0:
+    name: Cassandra 5.0 integration test (${{ matrix.mode.label }})
+    runs-on: ubuntu-latest
+
+    services:
+      cassandra:
+        image: cassandra:5.0
+        env:
+          MAX_HEAP_SIZE: 2048m
+          HEAP_NEWSIZE: 512m
+        ports:
+          - 9042:9042
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - label: default
+            group_commit_enabled: false
+          - label: with_group_commit
+            group_commit_enabled: true
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }})
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: ${{ env.JAVA_VENDOR }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}) to run integration test
+        uses: actions/setup-java@v5
+        if: ${{ env.SET_UP_INT_TEST_RUNTIME_NON_ORACLE_JDK == 'true'}}
+        with:
+          java-version: ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}
+          distribution: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR }}
+
+      - name: Login to Oracle container registry
+        uses: docker/login-action@v3
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        with:
+          registry: container-registry.oracle.com
+          username: ${{ secrets.OCR_USERNAME }}
+          password: ${{ secrets.OCR_TOKEN }}
+
+      - name: Set up JDK ${{ env.INT_TEST_JAVA_RUNTIME_VERSION }} (oracle) to run the integration test
+        if: ${{ env.INT_TEST_JAVA_RUNTIME_VENDOR == 'oracle' }}
+        run: |
+          container_id=$(docker create "container-registry.oracle.com/java/jdk:${{ env.INT_TEST_JAVA_RUNTIME_VERSION }}")
+          docker cp -L "$container_id:/usr/java/default" /usr/lib/jvm/oracle-jdk && docker rm "$container_id"
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+
+      - name: Execute Gradle 'integrationTestCassandra' task
+        run: ./gradlew integrationTestCassandra ${{ matrix.mode.group_commit_enabled && env.INT_TEST_GRADLE_OPTIONS_FOR_GROUP_COMMIT || '' }}
+
+      - name: Upload Gradle test reports
+        if: always()
+        uses: actions/upload-artifact@v5
+        with:
+          name: cassandra_5.0_integration_test_reports_${{ matrix.mode.label }}
+          path: core/build/reports/tests/integrationTestCassandra
+
   integration-test-for-cosmos:
     name: Cosmos DB integration test (${{ matrix.mode.label }})
     runs-on: windows-latest

build.gradle

@@ -26,7 +26,7 @@ subprojects {
         guiceVersion = '5.1.0'
         guavaVersion = '32.1.3-jre'
         slf4jVersion = '1.7.36'
-        cassandraDriverVersion = '3.11.5'
+        cassandraDriverVersion = '3.12.1'
         azureCosmosVersion = '4.75.0'
         azureBlobStorageVersion = '12.32.0'
         jooqVersion = '3.14.16'

core/build.gradle

@@ -167,7 +167,7 @@ dependencies {
     implementation "com.google.guava:guava:${guavaVersion}"
     implementation "com.google.inject:guice:${guiceVersion}"
     implementation "org.slf4j:slf4j-api:${slf4jVersion}"
-    implementation "com.datastax.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
+    implementation "org.apache.cassandra:cassandra-driver-core:${cassandraDriverVersion}"
     implementation "com.azure:azure-cosmos:${azureCosmosVersion}"
     implementation "com.azure:azure-storage-blob:${azureBlobStorageVersion}"
     implementation "org.jooq:jooq:${jooqVersion}"

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java

@@ -62,7 +62,12 @@ public Integer call() throws Exception {
       FileUtils.validateFilePath(scalarDbPropertiesFilePath);
       validatePositiveValue(
           spec.commandLine(), dataChunkSize, DataLoaderError.INVALID_DATA_CHUNK_SIZE);
-      validatePositiveValue(spec.commandLine(), maxThreads, DataLoaderError.INVALID_MAX_THREADS);
+      // Only validate the argument when provided by the user, if not set a default
+      if (maxThreads != null) {
+        validatePositiveValue(spec.commandLine(), maxThreads, DataLoaderError.INVALID_MAX_THREADS);
+      } else {
+        maxThreads = Runtime.getRuntime().availableProcessors();
+      }
 
       StorageFactory storageFactory = StorageFactory.create(scalarDbPropertiesFilePath);
       TransactionFactory transactionFactory = TransactionFactory.create(scalarDbPropertiesFilePath);
@@ -129,6 +134,11 @@ private void validateDeprecatedOptions() {
         DEPRECATED_END_EXCLUSIVE_OPTION,
         END_INCLUSIVE_OPTION,
         END_INCLUSIVE_OPTION_SHORT);
+    validateDeprecatedOptionPair(
+        spec.commandLine(),
+        DEPRECATED_THREADS_OPTION,
+        MAX_THREADS_OPTION,
+        MAX_THREADS_OPTION_SHORT);
   }
 
   private String getScalarDbPropertiesFilePath() {

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java

@@ -16,6 +16,9 @@ public class ExportCommandOptions {
   public static final String END_INCLUSIVE_OPTION = "--end-inclusive";
   public static final String END_INCLUSIVE_OPTION_SHORT = "-ei";
   public static final String DEPRECATED_END_EXCLUSIVE_OPTION = "--end-exclusive";
+  public static final String MAX_THREADS_OPTION = "--max-threads";
+  public static final String MAX_THREADS_OPTION_SHORT = "-mt";
+  public static final String DEPRECATED_THREADS_OPTION = "--threads";
 
   @CommandLine.Option(
       names = {"--config", "-c"},
@@ -77,7 +80,18 @@ public class ExportCommandOptions {
       paramLabel = "<MAX_THREADS>",
       description =
           "Maximum number of threads to use for parallel processing (default: number of available processors)")
-  protected int maxThreads;
+  protected Integer maxThreads;
+
+  /**
+   * @deprecated As of release 3.6.2. Will be removed in release 4.0.0. Use --max-threads instead
+   */
+  @Deprecated
+  @CommandLine.Option(
+      names = {DEPRECATED_THREADS_OPTION},
+      paramLabel = "<THREADS>",
+      description = "Deprecated: Use --max-threads instead",
+      hidden = true)
+  protected Integer threadsDeprecated;
 
   @CommandLine.Option(
       names = {"--start-key", "-sk"},
@@ -184,5 +198,10 @@ public void applyDeprecatedOptions() {
     if (endExclusiveDeprecated != null) {
       scanEndInclusive = !endExclusiveDeprecated;
     }
+
+    // If the deprecated option is set, use its value
+    if (threadsDeprecated != null) {
+      maxThreads = threadsDeprecated;
+    }
   }
 }

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommand.java

@@ -61,7 +61,12 @@ public Integer call() throws Exception {
         spec.commandLine(), dataChunkSize, DataLoaderError.INVALID_DATA_CHUNK_SIZE);
     validatePositiveValue(
         spec.commandLine(), transactionSize, DataLoaderError.INVALID_TRANSACTION_SIZE);
-    validatePositiveValue(spec.commandLine(), maxThreads, DataLoaderError.INVALID_MAX_THREADS);
+    // Only validate the argument when provided by the user, if not set a default
+    if (maxThreads != null) {
+      validatePositiveValue(spec.commandLine(), maxThreads, DataLoaderError.INVALID_MAX_THREADS);
+    } else {
+      maxThreads = Runtime.getRuntime().availableProcessors();
+    }
     validatePositiveValue(
         spec.commandLine(), dataChunkQueueSize, DataLoaderError.INVALID_DATA_CHUNK_QUEUE_SIZE);
     ControlFile controlFile = parseControlFileFromPath(controlFilePath).orElse(null);
@@ -286,6 +291,11 @@ private void validateDeprecatedOptions() {
         DEPRECATED_THREADS_OPTION,
         MAX_THREADS_OPTION,
         MAX_THREADS_OPTION_SHORT);
+    validateDeprecatedOptionPair(
+        spec.commandLine(),
+        DEPRECATED_LOG_SUCCESS_RECORDS_OPTION,
+        ENABLE_LOG_SUCCESS_RECORDS_OPTION,
+        ENABLE_LOG_SUCCESS_RECORDS_OPTION_SHORT);
   }
 
   /**
@@ -303,7 +313,7 @@ private ImportOptions createImportOptions(ControlFile controlFile) {
             .controlFile(controlFile)
             .controlFileValidationLevel(controlFileValidation)
             .logRawRecord(logRawRecord)
-            .logSuccessRecords(logSuccessRecords)
+            .logSuccessRecords(enableLogSuccessRecords)
             .ignoreNullValues(ignoreNullValues)
             .namespace(namespace)
             .dataChunkSize(dataChunkSize)

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataimport/ImportCommandOptions.java

@@ -13,6 +13,10 @@ public class ImportCommandOptions {
   public static final String MAX_THREADS_OPTION_SHORT = "-mt";
   public static final String DEPRECATED_THREADS_OPTION = "--threads";
 
+  public static final String ENABLE_LOG_SUCCESS_RECORDS_OPTION = "--enable-log-success";
+  public static final String ENABLE_LOG_SUCCESS_RECORDS_OPTION_SHORT = "-ls";
+  public static final String DEPRECATED_LOG_SUCCESS_RECORDS_OPTION = "--log-success";
+
   @CommandLine.Option(
       names = {"--mode", "-m"},
       description = "ScalarDB mode (STORAGE, TRANSACTION) (default: STORAGE)",
@@ -38,17 +42,18 @@ public class ImportCommandOptions {
       names = {"--max-threads", "-mt"},
       paramLabel = "<MAX_THREADS>",
       description =
-          "Maximum number of threads to use for parallel processing (default: number of available processors)",
-      defaultValue = "16")
-  protected int maxThreads;
+          "Maximum number of threads to use for parallel processing (default: number of available processors)")
+  protected Integer maxThreads;
 
-  // Deprecated option - kept for backward compatibility
+  /**
+   * @deprecated As of release 3.6.2. Will be removed in release 4.0.0. Use --max-threads instead
+   */
+  @Deprecated
   @CommandLine.Option(
       names = {DEPRECATED_THREADS_OPTION},
       paramLabel = "<THREADS>",
       description = "Deprecated: Use --max-threads instead",
       hidden = true)
-  @Deprecated
   protected Integer threadsDeprecated;
 
   @CommandLine.Option(
@@ -69,11 +74,22 @@ public class ImportCommandOptions {
       description = "Path to the JSON control file for data mapping")
   protected String controlFilePath;
 
+  /**
+   * @deprecated As of release 3.6.2. Will be removed in release 4.0.0. Use --enable-log-success
+   *     instead
+   */
+  @Deprecated
+  @CommandLine.Option(
+      names = {"--log-success"},
+      description = "Deprecated: Use --enable-log-success",
+      hidden = true)
+  protected boolean logSuccessRecordsDeprecated;
+
   @CommandLine.Option(
-      names = {"--log-success", "-ls"},
+      names = {"--enable-log-success", "-ls"},
       description = "Enable logging of successfully processed records (default: false)",
       defaultValue = "false")
-  protected boolean logSuccessRecords;
+  protected boolean enableLogSuccessRecords;
 
   @CommandLine.Option(
       names = {"--log-dir", "-ld"},
@@ -145,29 +161,31 @@ public class ImportCommandOptions {
   @CommandLine.Option(
       names = {"--data-chunk-size", "-dcs"},
       paramLabel = "<DATA_CHUNK_SIZE>",
-      description = "Maximum number of records to be included in a single data chunk",
+      description =
+          "Maximum number of records to be included in a single data chunk (default: 500)",
       defaultValue = "500")
   protected int dataChunkSize;
 
   @CommandLine.Option(
       names = {"--transaction-size", "-ts"},
       paramLabel = "<TRANSACTION_SIZE>",
       description =
-          "Maximum number of put operations that are grouped together into one ScalarDB distributed transaction, only supported in ScalarDB transaction mode",
+          "Maximum number of put operations that are grouped together into one ScalarDB distributed transaction, only supported in ScalarDB transaction mode (default: 100)",
       defaultValue = "100")
   protected int transactionSize;
 
   @CommandLine.Option(
       names = {"--split-log-mode", "-slm"},
       paramLabel = "<SPLIT_LOG_MODE>",
-      description = "Split log file into multiple files based on data chunks",
+      description = "Split log file into multiple files based on data chunks (default: false)",
       defaultValue = "false")
   protected boolean splitLogMode;
 
   @CommandLine.Option(
       names = {"--data-chunk-queue-size", "-qs"},
       paramLabel = "<DATA_CHUNK_QUEUE_SIZE>",
-      description = "Maximum number of data chunks that can be kept at a time for processing",
+      description =
+          "Maximum number of data chunks that can be kept at a time for processing (default: 256)",
       defaultValue = "256")
   protected int dataChunkQueueSize;
 
@@ -183,5 +201,8 @@ public void applyDeprecatedOptions() {
     if (threadsDeprecated != null) {
       maxThreads = threadsDeprecated;
     }
+    if (logSuccessRecordsDeprecated) {
+      enableLogSuccessRecords = true;
+    }
   }
 }