Skip to content

Commit 1664c56

Browse files
feeblefakieinv-jishnu
feeblefakie
and
inv-jishnu
authored
Backport to branch(3.15) : Fix for export projection and null data excluded bugs (#2622)
Co-authored-by: inv-jishnu <[email protected]>
1 parent e977b2a commit 1664c56

File tree

7 files changed

+252
-3
lines changed

7 files changed

+252
-3
lines changed

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ private String convertResultToCsv(Result result) {
8282
String columnName = iterator.next();
8383

8484
// Skip the field if it can be ignored based on check
85-
boolean columnNotProjected = !projectedColumnsSet.contains(columnName);
85+
boolean columnNotProjected =
86+
!projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
8687
boolean isMetadataColumn =
8788
ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
8889
if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonLineProducerTask.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.scalar.db.dataloader.core.dataexport.producer;
22

3+
import com.fasterxml.jackson.databind.node.NullNode;
34
import com.fasterxml.jackson.databind.node.ObjectNode;
45
import com.scalar.db.api.Result;
56
import com.scalar.db.api.TableMetadata;
@@ -70,7 +71,8 @@ private ObjectNode generateJsonForResult(Result result) {
7071
// Loop through all the columns and to the json object
7172
for (String columnName : tableColumns) {
7273
// Skip the field if it can be ignored based on check
73-
boolean columnNotProjected = !projectedColumnsSet.contains(columnName);
74+
boolean columnNotProjected =
75+
!projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
7476
boolean isMetadataColumn =
7577
ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
7678
if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
@@ -94,6 +96,7 @@ private void addToObjectNode(
9496
ObjectNode objectNode, Result result, String columnName, DataType dataType) {
9597

9698
if (result.isNull(columnName)) {
99+
objectNode.putIfAbsent(columnName, NullNode.getInstance());
97100
return;
98101
}
99102

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonProducerTask.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.scalar.db.dataloader.core.dataexport.producer;
22

33
import com.fasterxml.jackson.databind.node.ArrayNode;
4+
import com.fasterxml.jackson.databind.node.NullNode;
45
import com.fasterxml.jackson.databind.node.ObjectNode;
56
import com.scalar.db.api.Result;
67
import com.scalar.db.api.TableMetadata;
@@ -81,7 +82,8 @@ private ObjectNode generateJsonForResult(Result result) {
8182
// Loop through all the columns and to the json object
8283
for (String columnName : tableColumns) {
8384
// Skip the field if it can be ignored based on check
84-
boolean columnNotProjected = !projectedColumnsSet.contains(columnName);
85+
boolean columnNotProjected =
86+
!projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
8587
boolean isMetadataColumn =
8688
ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
8789
if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
@@ -105,6 +107,7 @@ private void addToObjectNode(
105107
ObjectNode objectNode, Result result, String columnName, DataType dataType) {
106108

107109
if (result.isNull(columnName)) {
110+
objectNode.putIfAbsent(columnName, NullNode.getInstance());
108111
return;
109112
}
110113

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/UnitTestUtils.java‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,42 @@ public static ObjectNode getOutputDataWithoutMetadata() {
154154
rootNode.put(TEST_COLUMN_9, TEST_VALUE_TIME.toString());
155155
rootNode.put(TEST_COLUMN_10, TEST_VALUE_DATE_TIME.toString());
156156
rootNode.put(TEST_COLUMN_11, TEST_VALUE_INSTANT.toString());
157+
return rootNode;
158+
}
157159

160+
public static ObjectNode getPartialOutputDataWithoutMetadata() {
161+
ObjectMapper mapper = new ObjectMapper();
162+
ObjectNode rootNode = mapper.createObjectNode();
163+
rootNode.put(TEST_COLUMN_1_PK, TEST_VALUE_LONG);
164+
rootNode.put(TEST_COLUMN_2_CK, TEST_VALUE_INT);
165+
rootNode.put(TEST_COLUMN_3_CK, TEST_VALUE_BOOLEAN);
166+
rootNode.put(TEST_COLUMN_4, TEST_VALUE_FLOAT);
167+
rootNode.put(TEST_COLUMN_5, TEST_VALUE_DOUBLE);
168+
return rootNode;
169+
}
170+
171+
public static ObjectNode getPartialOutputDataWithMetadata() {
172+
ObjectMapper mapper = new ObjectMapper();
173+
ObjectNode rootNode = mapper.createObjectNode();
174+
rootNode.put(TEST_COLUMN_1_PK, TEST_VALUE_LONG);
175+
rootNode.put(TEST_COLUMN_2_CK, TEST_VALUE_INT);
176+
rootNode.put(TEST_COLUMN_3_CK, TEST_VALUE_BOOLEAN);
177+
rootNode.put(TEST_COLUMN_4, TEST_VALUE_FLOAT);
178+
rootNode.put(TEST_COLUMN_5, TEST_VALUE_DOUBLE);
179+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_4, TEST_VALUE_FLOAT);
180+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_5, TEST_VALUE_DOUBLE);
181+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_6, TEST_VALUE_TEXT);
182+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_7, TEST_VALUE_BLOB);
183+
rootNode.put(Attribute.ID, TEST_VALUE_TX_ID);
184+
rootNode.put(Attribute.STATE, TEST_VALUE_INT);
185+
rootNode.put(Attribute.VERSION, TEST_VALUE_INT);
186+
rootNode.put(Attribute.PREPARED_AT, TEST_VALUE_LONG);
187+
rootNode.put(Attribute.COMMITTED_AT, TEST_VALUE_LONG);
188+
rootNode.put(Attribute.BEFORE_ID, TEST_VALUE_TEXT);
189+
rootNode.put(Attribute.BEFORE_STATE, TEST_VALUE_INT);
190+
rootNode.put(Attribute.BEFORE_VERSION, TEST_VALUE_INT);
191+
rootNode.put(Attribute.BEFORE_PREPARED_AT, TEST_VALUE_LONG);
192+
rootNode.put(Attribute.BEFORE_COMMITTED_AT, TEST_VALUE_LONG);
158193
return rootNode;
159194
}
160195

@@ -188,6 +223,40 @@ public static List<String> getColumnsListOfMetadata() {
188223
return projectedColumns;
189224
}
190225

226+
public static List<String> getPartialColumnsListWithoutMetadata() {
227+
List<String> projectedColumns = new ArrayList<>();
228+
projectedColumns.add(TEST_COLUMN_1_PK);
229+
projectedColumns.add(TEST_COLUMN_2_CK);
230+
projectedColumns.add(TEST_COLUMN_3_CK);
231+
projectedColumns.add(TEST_COLUMN_4);
232+
projectedColumns.add(TEST_COLUMN_5);
233+
return projectedColumns;
234+
}
235+
236+
public static List<String> getPartialColumnsListWithMetadata() {
237+
List<String> projectedColumns = new ArrayList<>();
238+
projectedColumns.add(TEST_COLUMN_1_PK);
239+
projectedColumns.add(TEST_COLUMN_2_CK);
240+
projectedColumns.add(TEST_COLUMN_3_CK);
241+
projectedColumns.add(TEST_COLUMN_4);
242+
projectedColumns.add(TEST_COLUMN_5);
243+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_4);
244+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_5);
245+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_6);
246+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_7);
247+
projectedColumns.add(Attribute.ID);
248+
projectedColumns.add(Attribute.STATE);
249+
projectedColumns.add(Attribute.VERSION);
250+
projectedColumns.add(Attribute.PREPARED_AT);
251+
projectedColumns.add(Attribute.COMMITTED_AT);
252+
projectedColumns.add(Attribute.BEFORE_ID);
253+
projectedColumns.add(Attribute.BEFORE_STATE);
254+
projectedColumns.add(Attribute.BEFORE_VERSION);
255+
projectedColumns.add(Attribute.BEFORE_PREPARED_AT);
256+
projectedColumns.add(Attribute.BEFORE_COMMITTED_AT);
257+
return projectedColumns;
258+
}
259+
191260
public static Map<String, DataType> getColumnData() {
192261
Map<String, DataType> columnData = new HashMap<>();
193262
columnData.put(TEST_COLUMN_1_PK, BIGINT);

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,63 @@ void process_withValidResultListWithMetadata_shouldReturnValidCsvString() {
6060
String output = csvProducerTask.process(resultList);
6161
Assertions.assertEquals(expectedOutput, output.trim());
6262
}
63+
64+
@Test
65+
void process_withValidResultList_withPartialProjections_shouldReturnValidCsvString() {
66+
projectedColumns = UnitTestUtils.getPartialColumnsListWithoutMetadata();
67+
csvProducerTask = new CsvProducerTask(false, projectedColumns, mockMetadata, columnData, ",");
68+
String expectedOutput =
69+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049";
70+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
71+
Result result = new ResultImpl(values, mockMetadata);
72+
List<Result> resultList = new ArrayList<>();
73+
resultList.add(result);
74+
String output = csvProducerTask.process(resultList);
75+
System.out.println(output);
76+
Assertions.assertEquals(expectedOutput, output.trim());
77+
}
78+
79+
@Test
80+
void process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnValidCsvString() {
81+
projectedColumns = UnitTestUtils.getPartialColumnsListWithMetadata();
82+
csvProducerTask = new CsvProducerTask(true, projectedColumns, mockMetadata, columnData, ",");
83+
String expectedOutput =
84+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
85+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
86+
Result result = new ResultImpl(values, mockMetadata);
87+
List<Result> resultList = new ArrayList<>();
88+
resultList.add(result);
89+
String output = csvProducerTask.process(resultList);
90+
Assertions.assertEquals(expectedOutput, output.trim());
91+
}
92+
93+
@Test
94+
void
95+
process_withValidResultListWithNoProjectionSpecifiedWithoutMetadata_shouldReturnValidCsvString() {
96+
csvProducerTask =
97+
new CsvProducerTask(false, Collections.emptyList(), mockMetadata, columnData, ",");
98+
String expectedOutput =
99+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z";
100+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
101+
Result result = new ResultImpl(values, mockMetadata);
102+
List<Result> resultList = new ArrayList<>();
103+
resultList.add(result);
104+
String output = csvProducerTask.process(resultList);
105+
Assertions.assertEquals(expectedOutput, output.trim());
106+
}
107+
108+
@Test
109+
void
110+
process_withValidResultListWithNoProjectionSpecifiedWithMetadata_shouldReturnValidCsvString() {
111+
csvProducerTask =
112+
new CsvProducerTask(true, Collections.emptyList(), mockMetadata, columnData, ",");
113+
String expectedOutput =
114+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
115+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
116+
Result result = new ResultImpl(values, mockMetadata);
117+
List<Result> resultList = new ArrayList<>();
118+
resultList.add(result);
119+
String output = csvProducerTask.process(resultList);
120+
Assertions.assertEquals(expectedOutput, output.trim());
121+
}
63122
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/JsonLineProducerTaskTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,61 @@ void process_withValidResultListWithMetadata_shouldReturnValidJsonLineString() {
6060
String output = jsonLineProducerTask.process(resultList);
6161
Assertions.assertEquals(rootNode.toString(), output.trim());
6262
}
63+
64+
@Test
65+
void process_withValidResultList_withPartialProjections_shouldReturnValidJsonLineString() {
66+
projectedColumns = UnitTestUtils.getPartialColumnsListWithoutMetadata();
67+
jsonLineProducerTask =
68+
new JsonLineProducerTask(false, projectedColumns, mockMetadata, columnData);
69+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithoutMetadata();
70+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
71+
Result result = new ResultImpl(values, mockMetadata);
72+
List<Result> resultList = new ArrayList<>();
73+
resultList.add(result);
74+
String output = jsonLineProducerTask.process(resultList);
75+
Assertions.assertEquals(rootNode.toString(), output.trim());
76+
}
77+
78+
@Test
79+
void
80+
process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnValidJsonLineString() {
81+
projectedColumns = UnitTestUtils.getPartialColumnsListWithMetadata();
82+
jsonLineProducerTask =
83+
new JsonLineProducerTask(true, projectedColumns, mockMetadata, columnData);
84+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithMetadata();
85+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
86+
Result result = new ResultImpl(values, mockMetadata);
87+
List<Result> resultList = new ArrayList<>();
88+
resultList.add(result);
89+
String output = jsonLineProducerTask.process(resultList);
90+
Assertions.assertEquals(rootNode.toString(), output.trim());
91+
}
92+
93+
@Test
94+
void
95+
process_withValidResultListWithNoProjectionSpecifiedWithoutMetadata_shouldReturnValidJsonLineString() {
96+
jsonLineProducerTask =
97+
new JsonLineProducerTask(false, Collections.emptyList(), mockMetadata, columnData);
98+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithoutMetadata();
99+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
100+
Result result = new ResultImpl(values, mockMetadata);
101+
List<Result> resultList = new ArrayList<>();
102+
resultList.add(result);
103+
String output = jsonLineProducerTask.process(resultList);
104+
Assertions.assertEquals(rootNode.toString(), output.trim());
105+
}
106+
107+
@Test
108+
void
109+
process_withValidResultListWithNoProjectionSpecifiedWithMetadata_shouldReturnValidJsonLineString() {
110+
jsonLineProducerTask =
111+
new JsonLineProducerTask(true, Collections.emptyList(), mockMetadata, columnData);
112+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithMetadata();
113+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
114+
Result result = new ResultImpl(values, mockMetadata);
115+
List<Result> resultList = new ArrayList<>();
116+
resultList.add(result);
117+
String output = jsonLineProducerTask.process(resultList);
118+
Assertions.assertEquals(rootNode.toString(), output.trim());
119+
}
63120
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/JsonProducerTaskTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,61 @@ void process_withValidResultListWithMetadata_shouldReturnValidJsonString() {
5959
String output = jsonProducerTask.process(resultList);
6060
Assertions.assertEquals(rootNode.toPrettyString(), output.trim());
6161
}
62+
63+
@Test
64+
void process_withValidResultList_withPartialProjections_shouldReturnValidJsonLineString() {
65+
projectedColumns = UnitTestUtils.getPartialColumnsListWithoutMetadata();
66+
jsonProducerTask =
67+
new JsonProducerTask(false, projectedColumns, mockMetadata, columnData, false);
68+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithoutMetadata();
69+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
70+
Result result = new ResultImpl(values, mockMetadata);
71+
List<Result> resultList = new ArrayList<>();
72+
resultList.add(result);
73+
String output = jsonProducerTask.process(resultList);
74+
Assertions.assertEquals(rootNode.toString(), output.trim());
75+
}
76+
77+
@Test
78+
void
79+
process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnValidJsonLineString() {
80+
projectedColumns = UnitTestUtils.getPartialColumnsListWithMetadata();
81+
jsonProducerTask =
82+
new JsonProducerTask(true, projectedColumns, mockMetadata, columnData, false);
83+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithMetadata();
84+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
85+
Result result = new ResultImpl(values, mockMetadata);
86+
List<Result> resultList = new ArrayList<>();
87+
resultList.add(result);
88+
String output = jsonProducerTask.process(resultList);
89+
Assertions.assertEquals(rootNode.toString(), output.trim());
90+
}
91+
92+
@Test
93+
void
94+
process_withValidResultListWithNoProjectionSpecifiedWithoutMetadata_shouldReturnValidJsonLineString() {
95+
jsonProducerTask =
96+
new JsonProducerTask(false, Collections.emptyList(), mockMetadata, columnData, true);
97+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithoutMetadata();
98+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
99+
Result result = new ResultImpl(values, mockMetadata);
100+
List<Result> resultList = new ArrayList<>();
101+
resultList.add(result);
102+
String output = jsonProducerTask.process(resultList);
103+
Assertions.assertEquals(rootNode.toPrettyString(), output.trim());
104+
}
105+
106+
@Test
107+
void
108+
process_withValidResultListWithNoProjectionSpecifiedWithMetadata_shouldReturnValidJsonLineString() {
109+
jsonProducerTask =
110+
new JsonProducerTask(true, Collections.emptyList(), mockMetadata, columnData, true);
111+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithMetadata();
112+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
113+
Result result = new ResultImpl(values, mockMetadata);
114+
List<Result> resultList = new ArrayList<>();
115+
resultList.add(result);
116+
String output = jsonProducerTask.process(resultList);
117+
Assertions.assertEquals(rootNode.toPrettyString(), output.trim());
118+
}
62119
}

0 commit comments

Comments
 (0)