Add import command for data loader CLI

.github/workflows/release-snapshot.yaml

@@ -1,6 +1,7 @@
 name: Release SNAPSHOT
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - master

.github/workflows/release.yaml

@@ -1,6 +1,7 @@
 name: Release
 
 on:
+  workflow_dispatch:
   push:
     tags:
       - "v[0-9]+.[0-9]+.[0-9]+"

README.md

@@ -23,7 +23,7 @@ You can install it in your application using your build tool such as Gradle and
 To add a dependency on ScalarDB using Gradle, use the following:
 ```gradle
 dependencies {
-    implementation 'com.scalar-labs:scalardb:3.15.1'
+    implementation 'com.scalar-labs:scalardb:3.15.2'
 }
 ```
 
@@ -32,7 +32,7 @@ To add a dependency using Maven:
 <dependency>
   <groupId>com.scalar-labs</groupId>
   <artifactId>scalardb</artifactId>
-  <version>3.15.1</version>
+  <version>3.15.2</version>
 </dependency>
 ```
 

build.gradle

@@ -25,17 +25,17 @@ subprojects {
         guavaVersion = '32.1.3-jre'
         slf4jVersion = '1.7.36'
         cassandraDriverVersion = '3.11.5'
-        azureCosmosVersion = '4.66.0'
+        azureCosmosVersion = '4.68.0'
         jooqVersion = '3.14.16'
-        awssdkVersion = '2.30.2'
+        awssdkVersion = '2.31.3'
         commonsDbcp2Version = '2.13.0'
         mysqlDriverVersion = '8.4.0'
         postgresqlDriverVersion = '42.7.5'
-        oracleDriverVersion = '23.6.0.24.10'
+        oracleDriverVersion = '23.7.0.25.01'
         sqlserverDriverVersion = '12.8.1.jre8'
-        sqliteDriverVersion = '3.48.0.0'
-        yugabyteDriverVersion = '42.7.3-yb-2'
-        mariadDbDriverVersion = '3.5.1'
+        sqliteDriverVersion = '3.49.1.0'
+        yugabyteDriverVersion = '42.7.3-yb-3'
+        mariadDbDriverVersion = '3.5.3'
         picocliVersion = '4.7.6'
         commonsTextVersion = '1.13.0'
         junitVersion = '5.11.4'

core/src/main/java/com/scalar/db/common/error/CoreError.java

@@ -804,24 +804,50 @@ public enum CoreError implements ScalarDbError {
       ""),
   DATA_LOADER_MISSING_CLUSTERING_KEY_COLUMN(
       Category.USER_ERROR,
-      "0175",
+      "0174",
       "Missing required field or column mapping for clustering key %s",
       "",
       ""),
   DATA_LOADER_MISSING_PARTITION_KEY_COLUMN(
       Category.USER_ERROR,
-      "0176",
+      "0175",
       "Missing required field or column mapping for partition key %s",
       "",
       ""),
   DATA_LOADER_MISSING_COLUMN(
-      Category.USER_ERROR, "0177", "Missing field or column mapping for %s", "", ""),
+      Category.USER_ERROR, "0176", "Missing field or column mapping for %s", "", ""),
+  DATA_LOADER_VALUE_TO_STRING_CONVERSION_FAILED(
+      Category.USER_ERROR,
+      "0177",
+      "Something went wrong while converting the ScalarDB values to strings. The table metadata and Value datatype probably do not match. Details: %s",
+      "",
+      ""),
+  DATA_LOADER_FILE_FORMAT_NOT_SUPPORTED(
+      Category.USER_ERROR, "0178", "The provided file format is not supported : %s", "", ""),
+  DATA_LOADER_COULD_NOT_FIND_PARTITION_KEY(
+      Category.USER_ERROR, "0179", "Could not find the partition key", "", ""),
+  DATA_LOADER_UPSERT_INSERT_MISSING_COLUMNS(
+      Category.USER_ERROR,
+      "0180",
+      "The source record needs to contain all fields if the UPSERT turns into an INSERT",
+      "",
+      ""),
+  DATA_LOADER_DATA_ALREADY_EXISTS(Category.USER_ERROR, "0181", "Record already exists", "", ""),
+  DATA_LOADER_DATA_NOT_FOUND(Category.USER_ERROR, "0182", "Record was not found", "", ""),
+  DATA_LOADER_COULD_NOT_FIND_CLUSTERING_KEY(
+      Category.USER_ERROR, "0183", "Could not find the clustering key", "", ""),
+  DATA_LOADER_TABLE_METADATA_MISSING(
+      Category.USER_ERROR, "0184", "No table metadata found", "", ""),
   DATA_LOADER_MISSING_SOURCE_FIELD(
       Category.USER_ERROR,
-      "0178",
+      "0185",
       "The data mapping source field '%s' for table '%s' is missing in the json data record",
       "",
       ""),
+  DATA_LOADER_CSV_DATA_MISMATCH(
+      Category.USER_ERROR, "0186", "The CSV row: %s does not match header: %s.", "", ""),
+  DATA_LOADER_JSON_CONTENT_START_ERROR(
+      Category.USER_ERROR, "0187", "Expected JSON file content to be an array", "", ""),
 
   //
   // Errors for the concurrency error category
@@ -1085,6 +1111,20 @@ public enum CoreError implements ScalarDbError {
       "Something went wrong while scanning. Are you sure you are running in the correct transaction mode? Details: %s",
       "",
       ""),
+  DATA_LOADER_CSV_FILE_READ_FAILED(
+      Category.INTERNAL_ERROR, "0049", "Failed to read CSV file. Details: %s.", "", ""),
+  DATA_LOADER_CSV_FILE_HEADER_READ_FAILED(
+      Category.INTERNAL_ERROR, "0050", "Failed to CSV read header line. Details: %s.", "", ""),
+  DATA_LOADER_DATA_CHUNK_PROCESS_FAILED(
+      Category.INTERNAL_ERROR,
+      "0051",
+      "Data chunk processing was interrupted. Details: %s",
+      "",
+      ""),
+  DATA_LOADER_JSON_FILE_READ_FAILED(
+      Category.INTERNAL_ERROR, "0052", "Failed to read JSON file. Details: %s.", "", ""),
+  DATA_LOADER_JSONLINES_FILE_READ_FAILED(
+      Category.INTERNAL_ERROR, "0053", "Failed to read JSON Lines file. Details: %s.", "", ""),
 
   //
   // Errors for the unknown transaction status error category

core/src/main/java/com/scalar/db/storage/jdbc/RdbEngineOracle.java

@@ -204,7 +204,7 @@ public boolean isDuplicateIndexError(SQLException e) {
   public String getDataTypeForEngine(DataType scalarDbDataType) {
     switch (scalarDbDataType) {
       case BIGINT:
-        return "NUMBER(19)";
+        return "NUMBER(16)";
       case BLOB:
         return "RAW(2000)";
       case BOOLEAN:

core/src/test/java/com/scalar/db/storage/jdbc/JdbcAdminTest.java

@@ -525,7 +525,7 @@ public void createTableInternal_ForSqlServer_ShouldCreateTableAndIndexes() throw
   public void createTableInternal_ForOracle_ShouldCreateTableAndIndexes() throws SQLException {
     createTableInternal_ForX_CreateTableAndIndexes(
         RdbEngine.ORACLE,
-        "CREATE TABLE \"my_ns\".\"foo_table\"(\"c3\" NUMBER(1),\"c1\" VARCHAR2(128),\"c4\" RAW(128),\"c2\" NUMBER(19),\"c5\" NUMBER(10),\"c6\" BINARY_DOUBLE,\"c7\" BINARY_FLOAT,\"c8\" DATE,\"c9\" TIMESTAMP(6),\"c10\" TIMESTAMP(3),\"c11\" TIMESTAMP(3) WITH TIME ZONE, PRIMARY KEY (\"c3\",\"c1\",\"c4\")) ROWDEPENDENCIES",
+        "CREATE TABLE \"my_ns\".\"foo_table\"(\"c3\" NUMBER(1),\"c1\" VARCHAR2(128),\"c4\" RAW(128),\"c2\" NUMBER(16),\"c5\" NUMBER(10),\"c6\" BINARY_DOUBLE,\"c7\" BINARY_FLOAT,\"c8\" DATE,\"c9\" TIMESTAMP(6),\"c10\" TIMESTAMP(3),\"c11\" TIMESTAMP(3) WITH TIME ZONE, PRIMARY KEY (\"c3\",\"c1\",\"c4\")) ROWDEPENDENCIES",
         "ALTER TABLE \"my_ns\".\"foo_table\" INITRANS 3 MAXTRANS 255",
         "CREATE UNIQUE INDEX \"my_ns.foo_table_clustering_order_idx\" ON \"my_ns\".\"foo_table\" (\"c3\" ASC,\"c1\" DESC,\"c4\" ASC)",
         "CREATE INDEX \"index_my_ns_foo_table_c4\" ON \"my_ns\".\"foo_table\" (\"c4\")",
@@ -539,7 +539,7 @@ public void createTableInternal_ForOracle_ShouldCreateTableAndIndexes() throws S
     when(config.getOracleVariableKeyColumnSize()).thenReturn(64);
     createTableInternal_ForX_CreateTableAndIndexes(
         new RdbEngineOracle(config),
-        "CREATE TABLE \"my_ns\".\"foo_table\"(\"c3\" NUMBER(1),\"c1\" VARCHAR2(64),\"c4\" RAW(64),\"c2\" NUMBER(19),\"c5\" NUMBER(10),\"c6\" BINARY_DOUBLE,\"c7\" BINARY_FLOAT,\"c8\" DATE,\"c9\" TIMESTAMP(6),\"c10\" TIMESTAMP(3),\"c11\" TIMESTAMP(3) WITH TIME ZONE, PRIMARY KEY (\"c3\",\"c1\",\"c4\")) ROWDEPENDENCIES",
+        "CREATE TABLE \"my_ns\".\"foo_table\"(\"c3\" NUMBER(1),\"c1\" VARCHAR2(64),\"c4\" RAW(64),\"c2\" NUMBER(16),\"c5\" NUMBER(10),\"c6\" BINARY_DOUBLE,\"c7\" BINARY_FLOAT,\"c8\" DATE,\"c9\" TIMESTAMP(6),\"c10\" TIMESTAMP(3),\"c11\" TIMESTAMP(3) WITH TIME ZONE, PRIMARY KEY (\"c3\",\"c1\",\"c4\")) ROWDEPENDENCIES",
         "ALTER TABLE \"my_ns\".\"foo_table\" INITRANS 3 MAXTRANS 255",
         "CREATE UNIQUE INDEX \"my_ns.foo_table_clustering_order_idx\" ON \"my_ns\".\"foo_table\" (\"c3\" ASC,\"c1\" DESC,\"c4\" ASC)",
         "CREATE INDEX \"index_my_ns_foo_table_c4\" ON \"my_ns\".\"foo_table\" (\"c4\")",
@@ -643,7 +643,7 @@ public void createTableInternal_IfNotExistsForOracle_ShouldCreateTableAndIndexes
       throws SQLException {
     createTableInternal_IfNotExistsForX_createTableAndIndexesIfNotExists(
         RdbEngine.ORACLE,
-        "CREATE TABLE \"my_ns\".\"foo_table\"(\"c3\" NUMBER(1),\"c1\" VARCHAR2(128),\"c4\" RAW(128),\"c2\" NUMBER(19),\"c5\" NUMBER(10),\"c6\" BINARY_DOUBLE,\"c7\" BINARY_FLOAT,\"c8\" DATE,\"c9\" TIMESTAMP(6),\"c10\" TIMESTAMP(3),\"c11\" TIMESTAMP(3) WITH TIME ZONE, PRIMARY KEY (\"c3\",\"c1\",\"c4\")) ROWDEPENDENCIES",
+        "CREATE TABLE \"my_ns\".\"foo_table\"(\"c3\" NUMBER(1),\"c1\" VARCHAR2(128),\"c4\" RAW(128),\"c2\" NUMBER(16),\"c5\" NUMBER(10),\"c6\" BINARY_DOUBLE,\"c7\" BINARY_FLOAT,\"c8\" DATE,\"c9\" TIMESTAMP(6),\"c10\" TIMESTAMP(3),\"c11\" TIMESTAMP(3) WITH TIME ZONE, PRIMARY KEY (\"c3\",\"c1\",\"c4\")) ROWDEPENDENCIES",
         "ALTER TABLE \"my_ns\".\"foo_table\" INITRANS 3 MAXTRANS 255",
         "CREATE UNIQUE INDEX \"my_ns.foo_table_clustering_order_idx\" ON \"my_ns\".\"foo_table\" (\"c3\" ASC,\"c1\" DESC,\"c4\" ASC)",
         "CREATE INDEX \"index_my_ns_foo_table_c4\" ON \"my_ns\".\"foo_table\" (\"c4\")",

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/ErrorMessage.java

@@ -0,0 +1,14 @@
+package com.scalar.db.dataloader.cli;
+
+public class ErrorMessage {
+  public static final String ERROR_IMPORT_TARGET_MISSING =
+      "Missing option: either '--namespace' and'--table' or '--control-file' options must be specified.";
+  public static final String ERROR_MISSING_FILE =
+      "File '%s' specified by argument '%s' does not exist.";
+  public static final String ERROR_LOG_DIRECTORY_WRITE_ACCESS =
+      "Not able to write to the log directory %s";
+  public static final String ERROR_CREATE_LOG_DIRECTORY_FAILED =
+      "Failed to create the log directory %s";
+  public static final String ERROR_CONTROL_FILE_INVALID_JSON =
+      "Not able to parse the %s control file";
+}