Add import log classes and utils

core/src/main/java/com/scalar/db/common/error/CoreError.java

@@ -802,6 +802,26 @@ public enum CoreError implements ScalarDbError {
       "Duplicated data mappings found for column '%s' in table '%s'",
       "",
       ""),
+  DATA_LOADER_MISSING_CLUSTERING_KEY_COLUMN(
+      Category.USER_ERROR,
+      "0175",
+      "Missing required field or column mapping for clustering key %s",
+      "",
+      ""),
+  DATA_LOADER_MISSING_PARTITION_KEY_COLUMN(
+      Category.USER_ERROR,
+      "0176",
+      "Missing required field or column mapping for partition key %s",
+      "",
+      ""),
+  DATA_LOADER_MISSING_COLUMN(
+      Category.USER_ERROR, "0177", "Missing field or column mapping for %s", "", ""),
+  DATA_LOADER_MISSING_SOURCE_FIELD(
+      Category.USER_ERROR,
+      "0178",
+      "The data mapping source field '%s' for table '%s' is missing in the json data record",
+      "",
+      ""),
 
   //
   // Errors for the concurrency error category

data-loader/build.gradle

@@ -17,6 +17,7 @@ subprojects {
         implementation("org.apache.commons:commons-lang3:${commonsLangVersion}")
         implementation("commons-io:commons-io:${commonsIoVersion}")
         implementation("org.slf4j:slf4j-simple:${slf4jVersion}")
+        implementation("software.amazon.awssdk:s3:2.25.31")
 
         // Mockito
         testImplementation "org.mockito:mockito-core:${mockitoVersion}"

data-loader/core/src/main/java/com/scalar/db/dataloader/core/DataLoaderObjectMapper.java

@@ -0,0 +1,14 @@
+package com.scalar.db.dataloader.core;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+
+public class DataLoaderObjectMapper extends ObjectMapper {
+
+  public DataLoaderObjectMapper() {
+    super();
+    this.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    this.registerModule(new JavaTimeModule());
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/ImportEventListener.java

@@ -0,0 +1,23 @@
+package com.scalar.db.dataloader.core.dataimport;
+
+import com.scalar.db.dataloader.core.dataimport.datachunk.ImportDataChunkStatus;
+import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult;
+import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchResult;
+import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchStatus;
+
+public interface ImportEventListener {
+
+  void onDataChunkStarted(ImportDataChunkStatus status);
+
+  void addOrUpdateDataChunkStatus(ImportDataChunkStatus status);
+
+  void onDataChunkCompleted(ImportDataChunkStatus status);
+
+  void onAllDataChunksCompleted();
+
+  void onTransactionBatchStarted(ImportTransactionBatchStatus batchStatus);
+
+  void onTransactionBatchCompleted(ImportTransactionBatchResult batchResult);
+
+  void onTaskComplete(ImportTaskResult taskResult);
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/ImportManager.java

@@ -0,0 +1,141 @@
+package com.scalar.db.dataloader.core.dataimport;
+
+import com.scalar.db.api.DistributedStorage;
+import com.scalar.db.api.DistributedTransactionManager;
+import com.scalar.db.api.TableMetadata;
+import com.scalar.db.dataloader.core.ScalarDBMode;
+import com.scalar.db.dataloader.core.dataimport.dao.ScalarDBDao;
+import com.scalar.db.dataloader.core.dataimport.datachunk.ImportDataChunkStatus;
+import com.scalar.db.dataloader.core.dataimport.processor.ImportProcessor;
+import com.scalar.db.dataloader.core.dataimport.processor.ImportProcessorFactory;
+import com.scalar.db.dataloader.core.dataimport.processor.ImportProcessorParams;
+import com.scalar.db.dataloader.core.dataimport.processor.TableColumnDataTypes;
+import com.scalar.db.dataloader.core.dataimport.task.result.ImportTaskResult;
+import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchResult;
+import com.scalar.db.dataloader.core.dataimport.transactionbatch.ImportTransactionBatchStatus;
+import java.io.BufferedReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import lombok.AllArgsConstructor;
+import lombok.NonNull;
+
+@AllArgsConstructor
+public class ImportManager implements ImportEventListener {
+
+  @NonNull private final Map<String, TableMetadata> tableMetadata;
+  @NonNull private final BufferedReader importFileReader;
+  @NonNull private final ImportOptions importOptions;
+  private final ImportProcessorFactory importProcessorFactory;
+  private final List<ImportEventListener> listeners = new ArrayList<>();
+  private final ScalarDBMode scalarDBMode;
+  private final DistributedStorage distributedStorage;
+  private final DistributedTransactionManager distributedTransactionManager;
+  private final List<ImportDataChunkStatus> importDataChunkStatusList = new ArrayList<>();
+
+  /**
+   * * Start the import process
+   *
+   * @return list of import data chunk status objects
+   */
+  public List<ImportDataChunkStatus> startImport() {
+    ImportProcessorParams params =
+        ImportProcessorParams.builder()
+            .scalarDBMode(scalarDBMode)
+            .importOptions(importOptions)
+            .tableMetadataByTableName(tableMetadata)
+            .dao(new ScalarDBDao())
+            .distributedTransactionManager(distributedTransactionManager)
+            .distributedStorage(distributedStorage)
+            .tableColumnDataTypes(getTableColumnDataTypes())
+            .build();
+    ImportProcessor processor = importProcessorFactory.createImportProcessor(params);
+    processor.addListener(this);
+    // If the data chunk size is 0, then process the entire file in a single data chunk
+    int dataChunkSize =
+        importOptions.getDataChunkSize() == 0
+            ? Integer.MAX_VALUE
+            : importOptions.getDataChunkSize();
+    return processor.process(
+        dataChunkSize, importOptions.getTransactionBatchSize(), importFileReader);
+  }
+
+  public void addListener(ImportEventListener listener) {
+    listeners.add(listener);
+  }
+
+  public void removeListener(ImportEventListener listener) {
+    listeners.remove(listener);
+  }
+
+  @Override
+  public void onDataChunkStarted(ImportDataChunkStatus status) {
+    for (ImportEventListener listener : listeners) {
+      listener.onDataChunkStarted(status);
+    }
+  }
+
+  @Override
+  public void addOrUpdateDataChunkStatus(ImportDataChunkStatus status) {
+    synchronized (importDataChunkStatusList) {
+      for (int i = 0; i < importDataChunkStatusList.size(); i++) {
+        if (importDataChunkStatusList.get(i).getDataChunkId() == status.getDataChunkId()) {
+          // Object found, replace it with the new one
+          importDataChunkStatusList.set(i, status);
+          return;
+        }
+      }
+      // If object is not found, add it to the list
+      importDataChunkStatusList.add(status);
+    }
+  }
+
+  @Override
+  public void onDataChunkCompleted(ImportDataChunkStatus status) {
+    for (ImportEventListener listener : listeners) {
+      listener.onDataChunkCompleted(status);
+    }
+  }
+
+  @Override
+  public void onTransactionBatchStarted(ImportTransactionBatchStatus status) {
+    for (ImportEventListener listener : listeners) {
+      listener.onTransactionBatchStarted(status);
+    }
+  }
+
+  @Override
+  public void onTransactionBatchCompleted(ImportTransactionBatchResult batchResult) {
+    for (ImportEventListener listener : listeners) {
+      listener.onTransactionBatchCompleted(batchResult);
+    }
+  }
+
+  @Override
+  public void onTaskComplete(ImportTaskResult taskResult) {
+    for (ImportEventListener listener : listeners) {
+      listener.onTaskComplete(taskResult);
+    }
+  }
+
+  @Override
+  public void onAllDataChunksCompleted() {
+    for (ImportEventListener listener : listeners) {
+      listener.onAllDataChunksCompleted();
+    }
+  }
+
+  public List<ImportDataChunkStatus> getImportDataChunkStatusList() {
+    return importDataChunkStatusList;
+  }
+
+  public TableColumnDataTypes getTableColumnDataTypes() {
+    TableColumnDataTypes tableColumnDataTypes = new TableColumnDataTypes();
+    tableMetadata.forEach(
+        (name, metadata) ->
+            metadata
+                .getColumnDataTypes()
+                .forEach((k, v) -> tableColumnDataTypes.addColumnDataType(name, k, v)));
+    return tableColumnDataTypes;
+  }
+}

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataimport/ImportOptions.java

@@ -0,0 +1,38 @@
+package com.scalar.db.dataloader.core.dataimport;
+
+import com.scalar.db.dataloader.core.FileFormat;
+import com.scalar.db.dataloader.core.dataimport.controlfile.ControlFile;
+import com.scalar.db.dataloader.core.dataimport.controlfile.ControlFileValidationLevel;
+import com.scalar.db.dataloader.core.dataimport.log.LogMode;
+import lombok.Builder;
+import lombok.Data;
+
+/** Import options to import data into one or more ScalarDB tables */
+@Builder
+@Data
+public class ImportOptions {
+
+  @Builder.Default private final ImportMode importMode = ImportMode.UPSERT;
+  @Builder.Default private final boolean requireAllColumns = false;
+  @Builder.Default private final FileFormat fileFormat = FileFormat.JSON;
+  @Builder.Default private final boolean prettyPrint = false;
+  @Builder.Default private final boolean ignoreNullValues = false;
+  @Builder.Default private final LogMode logMode = LogMode.SPLIT_BY_DATA_CHUNK;
+
+  @Builder.Default
+  private final ControlFileValidationLevel controlFileValidationLevel =
+      ControlFileValidationLevel.MAPPED;
+
+  @Builder.Default private final char delimiter = ',';
+
+  @Builder.Default private final boolean logSuccessRecords = false;
+  @Builder.Default private final boolean logRawRecord = false;
+
+  private final int dataChunkSize;
+  private final int transactionBatchSize;
+  private final ControlFile controlFile;
+  private final String namespace;
+  private final String tableName;
+  private final int maxThreads;
+  private final String customHeaderRow;
+}