Skip to content

Commit 8c0734b

Browse files
feeblefakieinv-jishnu
feeblefakie
and
inv-jishnu
authored
Backport to branch(3.14) : Fix for export projection and null data excluded bugs (#2623)
Co-authored-by: inv-jishnu <[email protected]>
1 parent 0d03dda commit 8c0734b

File tree

7 files changed

+253
-3
lines changed

7 files changed

+253
-3
lines changed

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ private String convertResultToCsv(Result result) {
7878
String columnName = iterator.next();
7979

8080
// Skip the field if it can be ignored based on check
81-
boolean columnNotProjected = !projectedColumnsSet.contains(columnName);
81+
boolean columnNotProjected =
82+
!projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
8283
boolean isMetadataColumn =
8384
ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
8485
if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonLineProducerTask.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.scalar.db.dataloader.core.dataexport.producer;
22

3+
import com.fasterxml.jackson.databind.node.NullNode;
34
import com.fasterxml.jackson.databind.node.ObjectNode;
45
import com.scalar.db.api.Result;
56
import com.scalar.db.api.TableMetadata;
@@ -66,7 +67,8 @@ private ObjectNode generateJsonForResult(Result result) {
6667
// Loop through all the columns and to the json object
6768
for (String columnName : tableColumns) {
6869
// Skip the field if it can be ignored based on check
69-
boolean columnNotProjected = !projectedColumnsSet.contains(columnName);
70+
boolean columnNotProjected =
71+
!projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
7072
boolean isMetadataColumn =
7173
ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
7274
if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
@@ -90,6 +92,7 @@ private void addToObjectNode(
9092
ObjectNode objectNode, Result result, String columnName, DataType dataType) {
9193

9294
if (result.isNull(columnName)) {
95+
objectNode.putIfAbsent(columnName, NullNode.getInstance());
9396
return;
9497
}
9598

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonProducerTask.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.scalar.db.dataloader.core.dataexport.producer;
22

33
import com.fasterxml.jackson.databind.node.ArrayNode;
4+
import com.fasterxml.jackson.databind.node.NullNode;
45
import com.fasterxml.jackson.databind.node.ObjectNode;
56
import com.scalar.db.api.Result;
67
import com.scalar.db.api.TableMetadata;
@@ -77,7 +78,8 @@ private ObjectNode generateJsonForResult(Result result) {
7778
// Loop through all the columns and to the json object
7879
for (String columnName : tableColumns) {
7980
// Skip the field if it can be ignored based on check
80-
boolean columnNotProjected = !projectedColumnsSet.contains(columnName);
81+
boolean columnNotProjected =
82+
!projectedColumnsSet.isEmpty() && !projectedColumnsSet.contains(columnName);
8183
boolean isMetadataColumn =
8284
ConsensusCommitUtils.isTransactionMetaColumn(columnName, tableMetadata);
8385
if (columnNotProjected || (!includeMetadata && isMetadataColumn)) {
@@ -101,6 +103,7 @@ private void addToObjectNode(
101103
ObjectNode objectNode, Result result, String columnName, DataType dataType) {
102104

103105
if (result.isNull(columnName)) {
106+
objectNode.putIfAbsent(columnName, NullNode.getInstance());
104107
return;
105108
}
106109

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/UnitTestUtils.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,42 @@ public static ObjectNode getOutputDataWithoutMetadata() {
129129
return rootNode;
130130
}
131131

132+
public static ObjectNode getPartialOutputDataWithoutMetadata() {
133+
ObjectMapper mapper = new ObjectMapper();
134+
ObjectNode rootNode = mapper.createObjectNode();
135+
rootNode.put(TEST_COLUMN_1_PK, TEST_VALUE_LONG);
136+
rootNode.put(TEST_COLUMN_2_CK, TEST_VALUE_INT);
137+
rootNode.put(TEST_COLUMN_3_CK, TEST_VALUE_BOOLEAN);
138+
rootNode.put(TEST_COLUMN_4, TEST_VALUE_FLOAT);
139+
rootNode.put(TEST_COLUMN_5, TEST_VALUE_DOUBLE);
140+
return rootNode;
141+
}
142+
143+
public static ObjectNode getPartialOutputDataWithMetadata() {
144+
ObjectMapper mapper = new ObjectMapper();
145+
ObjectNode rootNode = mapper.createObjectNode();
146+
rootNode.put(TEST_COLUMN_1_PK, TEST_VALUE_LONG);
147+
rootNode.put(TEST_COLUMN_2_CK, TEST_VALUE_INT);
148+
rootNode.put(TEST_COLUMN_3_CK, TEST_VALUE_BOOLEAN);
149+
rootNode.put(TEST_COLUMN_4, TEST_VALUE_FLOAT);
150+
rootNode.put(TEST_COLUMN_5, TEST_VALUE_DOUBLE);
151+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_4, TEST_VALUE_FLOAT);
152+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_5, TEST_VALUE_DOUBLE);
153+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_6, TEST_VALUE_TEXT);
154+
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_7, TEST_VALUE_BLOB);
155+
rootNode.put(Attribute.ID, TEST_VALUE_TX_ID);
156+
rootNode.put(Attribute.STATE, TEST_VALUE_INT);
157+
rootNode.put(Attribute.VERSION, TEST_VALUE_INT);
158+
rootNode.put(Attribute.PREPARED_AT, TEST_VALUE_LONG);
159+
rootNode.put(Attribute.COMMITTED_AT, TEST_VALUE_LONG);
160+
rootNode.put(Attribute.BEFORE_ID, TEST_VALUE_TEXT);
161+
rootNode.put(Attribute.BEFORE_STATE, TEST_VALUE_INT);
162+
rootNode.put(Attribute.BEFORE_VERSION, TEST_VALUE_INT);
163+
rootNode.put(Attribute.BEFORE_PREPARED_AT, TEST_VALUE_LONG);
164+
rootNode.put(Attribute.BEFORE_COMMITTED_AT, TEST_VALUE_LONG);
165+
return rootNode;
166+
}
167+
132168
public static List<String> getColumnsListOfMetadata() {
133169
List<String> projectedColumns = new ArrayList<>();
134170
projectedColumns.add(TEST_COLUMN_1_PK);
@@ -155,6 +191,40 @@ public static List<String> getColumnsListOfMetadata() {
155191
return projectedColumns;
156192
}
157193

194+
public static List<String> getPartialColumnsListWithoutMetadata() {
195+
List<String> projectedColumns = new ArrayList<>();
196+
projectedColumns.add(TEST_COLUMN_1_PK);
197+
projectedColumns.add(TEST_COLUMN_2_CK);
198+
projectedColumns.add(TEST_COLUMN_3_CK);
199+
projectedColumns.add(TEST_COLUMN_4);
200+
projectedColumns.add(TEST_COLUMN_5);
201+
return projectedColumns;
202+
}
203+
204+
public static List<String> getPartialColumnsListWithMetadata() {
205+
List<String> projectedColumns = new ArrayList<>();
206+
projectedColumns.add(TEST_COLUMN_1_PK);
207+
projectedColumns.add(TEST_COLUMN_2_CK);
208+
projectedColumns.add(TEST_COLUMN_3_CK);
209+
projectedColumns.add(TEST_COLUMN_4);
210+
projectedColumns.add(TEST_COLUMN_5);
211+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_4);
212+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_5);
213+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_6);
214+
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_7);
215+
projectedColumns.add(Attribute.ID);
216+
projectedColumns.add(Attribute.STATE);
217+
projectedColumns.add(Attribute.VERSION);
218+
projectedColumns.add(Attribute.PREPARED_AT);
219+
projectedColumns.add(Attribute.COMMITTED_AT);
220+
projectedColumns.add(Attribute.BEFORE_ID);
221+
projectedColumns.add(Attribute.BEFORE_STATE);
222+
projectedColumns.add(Attribute.BEFORE_VERSION);
223+
projectedColumns.add(Attribute.BEFORE_PREPARED_AT);
224+
projectedColumns.add(Attribute.BEFORE_COMMITTED_AT);
225+
return projectedColumns;
226+
}
227+
158228
public static Map<String, DataType> getColumnData() {
159229
Map<String, DataType> columnData = new HashMap<>();
160230
columnData.put(TEST_COLUMN_1_PK, BIGINT);

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,63 @@ void process_withValidResultListWithMetadata_shouldReturnValidCsvString() {
6060
String output = csvProducerTask.process(resultList);
6161
Assertions.assertEquals(expectedOutput, output.trim());
6262
}
63+
64+
@Test
65+
void process_withValidResultList_withPartialProjections_shouldReturnValidCsvString() {
66+
projectedColumns = UnitTestUtils.getPartialColumnsListWithoutMetadata();
67+
csvProducerTask = new CsvProducerTask(false, projectedColumns, mockMetadata, columnData, ",");
68+
String expectedOutput =
69+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049";
70+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
71+
Result result = new ResultImpl(values, mockMetadata);
72+
List<Result> resultList = new ArrayList<>();
73+
resultList.add(result);
74+
String output = csvProducerTask.process(resultList);
75+
System.out.println(output);
76+
Assertions.assertEquals(expectedOutput, output.trim());
77+
}
78+
79+
@Test
80+
void process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnValidCsvString() {
81+
projectedColumns = UnitTestUtils.getPartialColumnsListWithMetadata();
82+
csvProducerTask = new CsvProducerTask(true, projectedColumns, mockMetadata, columnData, ",");
83+
String expectedOutput =
84+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
85+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
86+
Result result = new ResultImpl(values, mockMetadata);
87+
List<Result> resultList = new ArrayList<>();
88+
resultList.add(result);
89+
String output = csvProducerTask.process(resultList);
90+
Assertions.assertEquals(expectedOutput, output.trim());
91+
}
92+
93+
@Test
94+
void
95+
process_withValidResultListWithNoProjectionSpecifiedWithoutMetadata_shouldReturnValidCsvString() {
96+
csvProducerTask =
97+
new CsvProducerTask(false, Collections.emptyList(), mockMetadata, columnData, ",");
98+
String expectedOutput =
99+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl";
100+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
101+
Result result = new ResultImpl(values, mockMetadata);
102+
List<Result> resultList = new ArrayList<>();
103+
resultList.add(result);
104+
String output = csvProducerTask.process(resultList);
105+
Assertions.assertEquals(expectedOutput, output.trim());
106+
}
107+
108+
@Test
109+
void
110+
process_withValidResultListWithNoProjectionSpecifiedWithMetadata_shouldReturnValidCsvString() {
111+
csvProducerTask =
112+
new CsvProducerTask(true, Collections.emptyList(), mockMetadata, columnData, ",");
113+
String expectedOutput =
114+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
115+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
116+
Result result = new ResultImpl(values, mockMetadata);
117+
List<Result> resultList = new ArrayList<>();
118+
resultList.add(result);
119+
String output = csvProducerTask.process(resultList);
120+
Assertions.assertEquals(expectedOutput, output.trim());
121+
}
63122
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/JsonLineProducerTaskTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,4 +60,61 @@ void process_withValidResultListWithMetadata_shouldReturnValidJsonLineString() {
6060
String output = jsonLineProducerTask.process(resultList);
6161
Assertions.assertEquals(rootNode.toString(), output.trim());
6262
}
63+
64+
@Test
65+
void process_withValidResultList_withPartialProjections_shouldReturnValidJsonLineString() {
66+
projectedColumns = UnitTestUtils.getPartialColumnsListWithoutMetadata();
67+
jsonLineProducerTask =
68+
new JsonLineProducerTask(false, projectedColumns, mockMetadata, columnData);
69+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithoutMetadata();
70+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
71+
Result result = new ResultImpl(values, mockMetadata);
72+
List<Result> resultList = new ArrayList<>();
73+
resultList.add(result);
74+
String output = jsonLineProducerTask.process(resultList);
75+
Assertions.assertEquals(rootNode.toString(), output.trim());
76+
}
77+
78+
@Test
79+
void
80+
process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnValidJsonLineString() {
81+
projectedColumns = UnitTestUtils.getPartialColumnsListWithMetadata();
82+
jsonLineProducerTask =
83+
new JsonLineProducerTask(true, projectedColumns, mockMetadata, columnData);
84+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithMetadata();
85+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
86+
Result result = new ResultImpl(values, mockMetadata);
87+
List<Result> resultList = new ArrayList<>();
88+
resultList.add(result);
89+
String output = jsonLineProducerTask.process(resultList);
90+
Assertions.assertEquals(rootNode.toString(), output.trim());
91+
}
92+
93+
@Test
94+
void
95+
process_withValidResultListWithNoProjectionSpecifiedWithoutMetadata_shouldReturnValidJsonLineString() {
96+
jsonLineProducerTask =
97+
new JsonLineProducerTask(false, Collections.emptyList(), mockMetadata, columnData);
98+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithoutMetadata();
99+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
100+
Result result = new ResultImpl(values, mockMetadata);
101+
List<Result> resultList = new ArrayList<>();
102+
resultList.add(result);
103+
String output = jsonLineProducerTask.process(resultList);
104+
Assertions.assertEquals(rootNode.toString(), output.trim());
105+
}
106+
107+
@Test
108+
void
109+
process_withValidResultListWithNoProjectionSpecifiedWithMetadata_shouldReturnValidJsonLineString() {
110+
jsonLineProducerTask =
111+
new JsonLineProducerTask(true, Collections.emptyList(), mockMetadata, columnData);
112+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithMetadata();
113+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
114+
Result result = new ResultImpl(values, mockMetadata);
115+
List<Result> resultList = new ArrayList<>();
116+
resultList.add(result);
117+
String output = jsonLineProducerTask.process(resultList);
118+
Assertions.assertEquals(rootNode.toString(), output.trim());
119+
}
63120
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/JsonProducerTaskTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,61 @@ void process_withValidResultListWithMetadata_shouldReturnValidJsonString() {
5959
String output = jsonProducerTask.process(resultList);
6060
Assertions.assertEquals(rootNode.toPrettyString(), output.trim());
6161
}
62+
63+
@Test
64+
void process_withValidResultList_withPartialProjections_shouldReturnValidJsonLineString() {
65+
projectedColumns = UnitTestUtils.getPartialColumnsListWithoutMetadata();
66+
jsonProducerTask =
67+
new JsonProducerTask(false, projectedColumns, mockMetadata, columnData, false);
68+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithoutMetadata();
69+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
70+
Result result = new ResultImpl(values, mockMetadata);
71+
List<Result> resultList = new ArrayList<>();
72+
resultList.add(result);
73+
String output = jsonProducerTask.process(resultList);
74+
Assertions.assertEquals(rootNode.toString(), output.trim());
75+
}
76+
77+
@Test
78+
void
79+
process_withValidResultList_withPartialProjectionsAndMetadata_shouldReturnValidJsonLineString() {
80+
projectedColumns = UnitTestUtils.getPartialColumnsListWithMetadata();
81+
jsonProducerTask =
82+
new JsonProducerTask(true, projectedColumns, mockMetadata, columnData, false);
83+
ObjectNode rootNode = UnitTestUtils.getPartialOutputDataWithMetadata();
84+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
85+
Result result = new ResultImpl(values, mockMetadata);
86+
List<Result> resultList = new ArrayList<>();
87+
resultList.add(result);
88+
String output = jsonProducerTask.process(resultList);
89+
Assertions.assertEquals(rootNode.toString(), output.trim());
90+
}
91+
92+
@Test
93+
void
94+
process_withValidResultListWithNoProjectionSpecifiedWithoutMetadata_shouldReturnValidJsonLineString() {
95+
jsonProducerTask =
96+
new JsonProducerTask(false, Collections.emptyList(), mockMetadata, columnData, true);
97+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithoutMetadata();
98+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
99+
Result result = new ResultImpl(values, mockMetadata);
100+
List<Result> resultList = new ArrayList<>();
101+
resultList.add(result);
102+
String output = jsonProducerTask.process(resultList);
103+
Assertions.assertEquals(rootNode.toPrettyString(), output.trim());
104+
}
105+
106+
@Test
107+
void
108+
process_withValidResultListWithNoProjectionSpecifiedWithMetadata_shouldReturnValidJsonLineString() {
109+
jsonProducerTask =
110+
new JsonProducerTask(true, Collections.emptyList(), mockMetadata, columnData, true);
111+
ObjectNode rootNode = UnitTestUtils.getOutputDataWithMetadata();
112+
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
113+
Result result = new ResultImpl(values, mockMetadata);
114+
List<Result> resultList = new ArrayList<>();
115+
resultList.add(result);
116+
String output = jsonProducerTask.process(resultList);
117+
Assertions.assertEquals(rootNode.toPrettyString(), output.trim());
118+
}
62119
}

0 commit comments

Comments
 (0)