Deprecate the include metadata export argument and depend on ScalarDB config

# Conflicts:
#	data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java
#	data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/CsvExportManagerTest.java

Author: ypeckstadt

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommand.java

@@ -162,7 +162,7 @@ private void validateOutputDirectory() throws DirectoryValidationException {
   private ExportManager createExportManager(
       TransactionFactory transactionFactory, ScalarDbDao scalarDbDao, FileFormat fileFormat) {
     ProducerTaskFactory taskFactory =
-        new ProducerTaskFactory(delimiter, includeTransactionMetadata, prettyPrintJson);
+        new ProducerTaskFactory(delimiter, prettyPrintJson);
     DistributedTransactionManager manager = transactionFactory.getTransactionManager();
     switch (fileFormat) {
       case JSON:
@@ -181,7 +181,6 @@ private ExportOptions buildExportOptions(Key partitionKey, ScanRange scanRange)
         ExportOptions.builder(namespace, table, partitionKey, outputFormat)
             .sortOrders(sortOrders)
             .excludeHeaderRow(excludeHeader)
-            .includeTransactionMetadata(includeTransactionMetadata)
             .delimiter(delimiter)
             .limit(limit)
             .maxThreadCount(maxThreads)

data-loader/cli/src/main/java/com/scalar/db/dataloader/cli/command/dataexport/ExportCommandOptions.java

@@ -69,10 +69,16 @@ public class ExportCommandOptions {
       defaultValue = "json")
   protected FileFormat outputFormat;
 
+  /**
+   * @deprecated As of release 3.16.2 This option is no longer used and will be removed in release
+   *     4.0.0.
+   */
+  @Deprecated
   @CommandLine.Option(
       names = {"--include-metadata", "-m"},
-      description = "Include transaction metadata in the exported data (default: false)",
-      defaultValue = "false")
+      description = "Deprecated: This option is no longer used",
+      defaultValue = "false",
+      hidden = true)
   protected boolean includeTransactionMetadata;
 
   @CommandLine.Option(

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/CsvExportManager.java

@@ -5,7 +5,6 @@
 import com.scalar.db.dataloader.core.dataexport.producer.ProducerTaskFactory;
 import com.scalar.db.dataloader.core.dataimport.dao.ScalarDbDao;
 import com.scalar.db.dataloader.core.util.CsvUtil;
-import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
 import java.io.IOException;
 import java.io.Writer;
 import java.util.Iterator;
@@ -77,8 +76,7 @@ private String createCsvHeaderRow(ExportOptions exportOptions, TableMetadata tab
     Iterator<String> iterator = tableMetadata.getColumnNames().iterator();
     while (iterator.hasNext()) {
       String columnName = iterator.next();
-      if (shouldIgnoreColumn(
-          exportOptions.isIncludeTransactionMetadata(), columnName, tableMetadata, projections)) {
+      if (shouldIgnoreColumn(columnName, projections)) {
         continue;
       }
       headerRow.append(columnName);
@@ -92,22 +90,13 @@ private String createCsvHeaderRow(ExportOptions exportOptions, TableMetadata tab
   }
 
   /**
-   * To ignore a column or not based on conditions such as if it is a metadata column or if it is
-   * not include in selected projections
+   * To ignore a column or not based on if it is not included in selected projections
    *
-   * @param isIncludeTransactionMetadata to include transaction metadata or not
    * @param columnName column name
-   * @param tableMetadata table metadata
    * @param projections selected columns for projection
-   * @return ignore the column or not
+   * @return true if the column should be ignored, false otherwise
    */
-  private boolean shouldIgnoreColumn(
-      boolean isIncludeTransactionMetadata,
-      String columnName,
-      TableMetadata tableMetadata,
-      List<String> projections) {
-    return (!isIncludeTransactionMetadata
-            && ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata))
-        || (!projections.isEmpty() && !projections.contains(columnName));
+  private boolean shouldIgnoreColumn(String columnName, List<String> projections) {
+    return !projections.isEmpty() && !projections.contains(columnName);
   }
 }

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportManager.java

@@ -205,8 +205,7 @@ private void validateExportOptions(ExportOptions exportOptions, TableMetadata ta
    * @param tableMetadata metadata of the table
    */
   private void handleTransactionMetadata(ExportOptions exportOptions, TableMetadata tableMetadata) {
-    if (exportOptions.isIncludeTransactionMetadata()
-        && !exportOptions.getProjectionColumns().isEmpty()) {
+    if (!exportOptions.getProjectionColumns().isEmpty()) {
       List<String> projectionMetadata =
           TableMetadataUtil.populateProjectionsWithMetadata(
               tableMetadata, exportOptions.getProjectionColumns());

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/ExportOptions.java

@@ -27,7 +27,6 @@ public class ExportOptions {
   @Builder.Default private final int maxThreadCount = Runtime.getRuntime().availableProcessors();
   @Builder.Default private final String delimiter = ";";
   @Builder.Default private final boolean excludeHeaderRow = false;
-  @Builder.Default private final boolean includeTransactionMetadata = false;
   @Builder.Default private List<String> projectionColumns = Collections.emptyList();
   private List<Scan.Ordering> sortOrders;
 

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java

@@ -6,7 +6,6 @@
 import com.scalar.db.dataloader.core.util.CsvUtil;
 import com.scalar.db.dataloader.core.util.DecimalUtil;
 import com.scalar.db.io.DataType;
-import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
 import java.nio.charset.Charset;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -33,19 +32,17 @@ public class CsvProducerTask extends ProducerTask {
   /**
    * Class constructor
    *
-   * @param includeMetadata Include metadata in the exported data
    * @param projectColumns list of columns that is required in export data
    * @param tableMetadata Metadata for a single ScalarDB table
    * @param columnDataTypes Map of data types for the all columns in a ScalarDB table
    * @param delimiter Delimiter used in csv content
    */
   public CsvProducerTask(
-      boolean includeMetadata,
       List<String> projectColumns,
       TableMetadata tableMetadata,
       Map<String, DataType> columnDataTypes,
       String delimiter) {
-    super(includeMetadata, projectColumns, tableMetadata, columnDataTypes);
+    super(projectColumns, tableMetadata, columnDataTypes);
     this.delimiter = delimiter;
   }
 
@@ -82,12 +79,10 @@ private String convertResultToCsv(Result result) {
       while (iterator.hasNext()) {
         String columnName = iterator.next();
 
-        // Skip the field if it can be ignored based on check
+        // Skip the field if it's not in the projection list (when projections are specified)
         boolean columnNotProjected =
             !projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
-        boolean isMetadataColumn =
-            ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
-        if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
+        if (columnNotProjected) {
           continue;
         }
 

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonLineProducerTask.java

@@ -6,7 +6,6 @@
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.dataloader.core.DataLoaderObjectMapper;
 import com.scalar.db.io.DataType;
-import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
 import java.nio.charset.Charset;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -28,17 +27,15 @@ public class JsonLineProducerTask extends ProducerTask {
   /**
    * Class constructor
    *
-   * @param includeMetadata Include metadata in the exported data
    * @param projectionColumns list of columns that is required in export data
    * @param tableMetadata Metadata for a single ScalarDB table
    * @param columnDataTypes Map of data types for the all columns in a ScalarDB table
    */
   public JsonLineProducerTask(
-      boolean includeMetadata,
       List<String> projectionColumns,
       TableMetadata tableMetadata,
       Map<String, DataType> columnDataTypes) {
-    super(includeMetadata, projectionColumns, tableMetadata, columnDataTypes);
+    super(projectionColumns, tableMetadata, columnDataTypes);
   }
 
   /**
@@ -72,12 +69,10 @@ private ObjectNode generateJsonForResult(Result result) {
 
     // Loop through all the columns and to the json object
     for (String columnName : tableColumns) {
-      // Skip the field if it can be ignored based on check
+      // Skip the field if it's not in the projection list (when projections are specified)
       boolean columnNotProjected =
           !projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
-      boolean isMetadataColumn =
-          ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
-      if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
+      if (columnNotProjected) {
         continue;
       }
 

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonProducerTask.java

@@ -7,7 +7,6 @@
 import com.scalar.db.api.TableMetadata;
 import com.scalar.db.dataloader.core.DataLoaderObjectMapper;
 import com.scalar.db.io.DataType;
-import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
 import java.nio.charset.Charset;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -30,19 +29,17 @@ public class JsonProducerTask extends ProducerTask {
   /**
    * Class constructor
    *
-   * @param includeMetadata Include metadata in the exported data
    * @param projectionColumns list of columns that is required in export data
    * @param tableMetadata Metadata for a single ScalarDB table
    * @param columnDataTypes Map of data types for the all columns in a ScalarDB table
    * @param prettyPrintJson Json data should be formatted or not
    */
   public JsonProducerTask(
-      boolean includeMetadata,
       List<String> projectionColumns,
       TableMetadata tableMetadata,
       Map<String, DataType> columnDataTypes,
       boolean prettyPrintJson) {
-    super(includeMetadata, projectionColumns, tableMetadata, columnDataTypes);
+    super(projectionColumns, tableMetadata, columnDataTypes);
     this.prettyPrintJson = prettyPrintJson;
   }
 
@@ -84,12 +81,10 @@ private ObjectNode generateJsonForResult(Result result) {
 
     // Loop through all the columns and to the json object
     for (String columnName : tableColumns) {
-      // Skip the field if it can be ignored based on check
+      // Skip the field if it's not in the projection list (when projections are specified)
       boolean columnNotProjected =
           !projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
-      boolean isMetadataColumn =
-          ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
-      if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
+      if (columnNotProjected) {
         continue;
       }
 

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/ProducerTask.java

@@ -32,12 +32,6 @@ public abstract class ProducerTask {
    */
   protected final Map<String, DataType> dataTypeByColumnName;
 
-  /**
-   * Flag indicating whether to include internal metadata columns (e.g., transaction metadata) in
-   * the output.
-   */
-  protected final boolean includeMetadata;
-
   /**
    * A set of column names to include in the exported output. If empty, all columns are included by
    * default.
@@ -47,17 +41,14 @@ public abstract class ProducerTask {
   /**
    * Class constructor
    *
-   * @param includeMetadata Include metadata in the exported data
    * @param projectionColumns List of column name for projection
    * @param tableMetadata Metadata of the ScalarDB table
    * @param columnDataTypes Map of data types for the all columns in a ScalarDB table
    */
   protected ProducerTask(
-      boolean includeMetadata,
       List<String> projectionColumns,
       TableMetadata tableMetadata,
       Map<String, DataType> columnDataTypes) {
-    this.includeMetadata = includeMetadata;
     this.projectedColumnsSet = new HashSet<>(projectionColumns);
     this.tableMetadata = tableMetadata;
     this.dataTypeByColumnName = columnDataTypes;

data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/ProducerTaskFactory.java

@@ -14,7 +14,6 @@
 public class ProducerTaskFactory {
 
   private final String delimiter;
-  private final boolean includeMetadata;
   private final boolean prettyPrintJson;
 
   /**
@@ -36,21 +35,15 @@ public ProducerTask createProducerTask(
       case JSON:
         producerTask =
             new JsonProducerTask(
-                includeMetadata,
-                projectionColumns,
-                tableMetadata,
-                dataTypeByColumnName,
-                prettyPrintJson);
+                projectionColumns, tableMetadata, dataTypeByColumnName, prettyPrintJson);
         break;
       case JSONL:
         producerTask =
-            new JsonLineProducerTask(
-                includeMetadata, projectionColumns, tableMetadata, dataTypeByColumnName);
+            new JsonLineProducerTask(projectionColumns, tableMetadata, dataTypeByColumnName);
         break;
       case CSV:
         producerTask =
-            new CsvProducerTask(
-                includeMetadata, projectionColumns, tableMetadata, dataTypeByColumnName, delimiter);
+            new CsvProducerTask(projectionColumns, tableMetadata, dataTypeByColumnName, delimiter);
         break;
       default:
         throw new IllegalArgumentException(