Skip to content

Commit 628b487

Browse files
feeblefakieinv-jishnu
feeblefakie
and
inv-jishnu
authored
Backport to branch(3.15) : Add support for new date, time data types (#2559)
Co-authored-by: inv-jishnu <[email protected]>
1 parent 4b276e5 commit 628b487

File tree

7 files changed

+185
-3
lines changed

7 files changed

+185
-3
lines changed

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTask.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
import com.scalar.db.io.DataType;
99
import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
1010
import java.nio.charset.Charset;
11+
import java.time.Instant;
12+
import java.time.LocalDate;
13+
import java.time.LocalDateTime;
14+
import java.time.LocalTime;
1115
import java.util.Base64;
1216
import java.util.Iterator;
1317
import java.util.LinkedHashSet;
@@ -144,6 +148,26 @@ private String convertToString(Result result, String columnName, DataType dataTy
144148
case TEXT:
145149
value = result.getText(columnName);
146150
break;
151+
case DATE:
152+
LocalDate date = result.getDate(columnName);
153+
assert date != null;
154+
value = date.toString();
155+
break;
156+
case TIME:
157+
LocalTime time = result.getTime(columnName);
158+
assert time != null;
159+
value = time.toString();
160+
break;
161+
case TIMESTAMP:
162+
LocalDateTime localDateTime = result.getTimestamp(columnName);
163+
assert localDateTime != null;
164+
value = localDateTime.toString();
165+
break;
166+
case TIMESTAMPTZ:
167+
Instant instant = result.getTimestampTZ(columnName);
168+
assert instant != null;
169+
value = instant.toString();
170+
break;
147171
default:
148172
throw new AssertionError("Unknown data type:" + dataType);
149173
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonLineProducerTask.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
import com.scalar.db.io.DataType;
88
import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
99
import java.nio.charset.Charset;
10+
import java.time.Instant;
11+
import java.time.LocalDate;
12+
import java.time.LocalDateTime;
13+
import java.time.LocalTime;
1014
import java.util.Base64;
1115
import java.util.LinkedHashSet;
1216
import java.util.List;
@@ -117,6 +121,26 @@ private void addToObjectNode(
117121
byte[] encoded = Base64.getEncoder().encode(result.getBlobAsBytes(columnName));
118122
objectNode.put(columnName, new String(encoded, Charset.defaultCharset()));
119123
break;
124+
case DATE:
125+
LocalDate date = result.getDate(columnName);
126+
assert date != null;
127+
objectNode.put(columnName, date.toString());
128+
break;
129+
case TIME:
130+
LocalTime time = result.getTime(columnName);
131+
assert time != null;
132+
objectNode.put(columnName, time.toString());
133+
break;
134+
case TIMESTAMP:
135+
LocalDateTime localDateTime = result.getTimestamp(columnName);
136+
assert localDateTime != null;
137+
objectNode.put(columnName, localDateTime.toString());
138+
break;
139+
case TIMESTAMPTZ:
140+
Instant instant = result.getTimestampTZ(columnName);
141+
assert instant != null;
142+
objectNode.put(columnName, instant.toString());
143+
break;
120144
default:
121145
throw new AssertionError("Unknown data type:" + dataType);
122146
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/dataexport/producer/JsonProducerTask.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
import com.scalar.db.io.DataType;
99
import com.scalar.db.transaction.consensuscommit.ConsensusCommitUtils;
1010
import java.nio.charset.Charset;
11+
import java.time.Instant;
12+
import java.time.LocalDate;
13+
import java.time.LocalDateTime;
14+
import java.time.LocalTime;
1115
import java.util.Base64;
1216
import java.util.LinkedHashSet;
1317
import java.util.List;
@@ -128,6 +132,26 @@ private void addToObjectNode(
128132
byte[] encoded = Base64.getEncoder().encode(result.getBlobAsBytes(columnName));
129133
objectNode.put(columnName, new String(encoded, Charset.defaultCharset()));
130134
break;
135+
case DATE:
136+
LocalDate date = result.getDate(columnName);
137+
assert date != null;
138+
objectNode.put(columnName, date.toString());
139+
break;
140+
case TIME:
141+
LocalTime time = result.getTime(columnName);
142+
assert time != null;
143+
objectNode.put(columnName, time.toString());
144+
break;
145+
case TIMESTAMP:
146+
LocalDateTime localDateTime = result.getTimestamp(columnName);
147+
assert localDateTime != null;
148+
objectNode.put(columnName, localDateTime.toString());
149+
break;
150+
case TIMESTAMPTZ:
151+
Instant instant = result.getTimestampTZ(columnName);
152+
assert instant != null;
153+
objectNode.put(columnName, instant.toString());
154+
break;
131155
default:
132156
throw new AssertionError("Unknown data type:" + dataType);
133157
}

‎data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,18 @@
88
import com.scalar.db.io.BooleanColumn;
99
import com.scalar.db.io.Column;
1010
import com.scalar.db.io.DataType;
11+
import com.scalar.db.io.DateColumn;
1112
import com.scalar.db.io.DoubleColumn;
1213
import com.scalar.db.io.FloatColumn;
1314
import com.scalar.db.io.IntColumn;
1415
import com.scalar.db.io.TextColumn;
16+
import com.scalar.db.io.TimeColumn;
17+
import com.scalar.db.io.TimestampColumn;
18+
import com.scalar.db.io.TimestampTZColumn;
19+
import java.time.Instant;
20+
import java.time.LocalDate;
21+
import java.time.LocalDateTime;
22+
import java.time.LocalTime;
1523
import java.util.Base64;
1624
import javax.annotation.Nullable;
1725

@@ -73,6 +81,22 @@ public static Column<?> createColumnFromValue(
7381
return value != null
7482
? BlobColumn.of(columnName, Base64.getDecoder().decode(value))
7583
: BlobColumn.ofNull(columnName);
84+
case DATE:
85+
return value != null
86+
? DateColumn.of(columnName, LocalDate.parse(value))
87+
: DateColumn.ofNull(columnName);
88+
case TIME:
89+
return value != null
90+
? TimeColumn.of(columnName, LocalTime.parse(value))
91+
: TimeColumn.ofNull(columnName);
92+
case TIMESTAMP:
93+
return value != null
94+
? TimestampColumn.of(columnName, LocalDateTime.parse(value))
95+
: TimestampColumn.ofNull(columnName);
96+
case TIMESTAMPTZ:
97+
return value != null
98+
? TimestampTZColumn.of(columnName, Instant.parse(value))
99+
: TimestampTZColumn.ofNull(columnName);
76100
default:
77101
throw new AssertionError();
78102
}

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/UnitTestUtils.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,20 @@
1212
import com.scalar.db.io.BooleanColumn;
1313
import com.scalar.db.io.Column;
1414
import com.scalar.db.io.DataType;
15+
import com.scalar.db.io.DateColumn;
1516
import com.scalar.db.io.DoubleColumn;
1617
import com.scalar.db.io.FloatColumn;
1718
import com.scalar.db.io.IntColumn;
1819
import com.scalar.db.io.TextColumn;
20+
import com.scalar.db.io.TimeColumn;
21+
import com.scalar.db.io.TimestampColumn;
22+
import com.scalar.db.io.TimestampTZColumn;
1923
import com.scalar.db.transaction.consensuscommit.Attribute;
2024
import java.nio.charset.StandardCharsets;
25+
import java.time.Instant;
26+
import java.time.LocalDate;
27+
import java.time.LocalDateTime;
28+
import java.time.LocalTime;
2129
import java.util.*;
2230

2331
/** Utils for the service unit tests */
@@ -31,6 +39,10 @@ public class UnitTestUtils {
3139
public static final String TEST_COLUMN_5 = "col5";
3240
public static final String TEST_COLUMN_6 = "col6";
3341
public static final String TEST_COLUMN_7 = "col7";
42+
public static final String TEST_COLUMN_8 = "col8";
43+
public static final String TEST_COLUMN_9 = "col9";
44+
public static final String TEST_COLUMN_10 = "col10";
45+
public static final String TEST_COLUMN_11 = "col11";
3446

3547
public static final String TEST_VALUE_TEXT = "test value";
3648

@@ -44,6 +56,10 @@ public class UnitTestUtils {
4456
public static final Long TEST_VALUE_LONG = BigIntColumn.MAX_VALUE;
4557
public static final boolean TEST_VALUE_BOOLEAN = true;
4658
public static final double TEST_VALUE_DOUBLE = Double.MIN_VALUE;
59+
public static final LocalDate TEST_VALUE_DATE = LocalDate.of(2000, 1, 1);
60+
public static final LocalTime TEST_VALUE_TIME = LocalTime.of(1, 1, 1);
61+
public static final LocalDateTime TEST_VALUE_DATE_TIME = LocalDateTime.of(2000, 1, 1, 1, 1);
62+
public static final Instant TEST_VALUE_INSTANT = Instant.ofEpochMilli(1740041740);
4763
public static final String TEST_CSV_DELIMITER = ";";
4864

4965
public static TableMetadata createTestTableMetadata() {
@@ -55,6 +71,10 @@ public static TableMetadata createTestTableMetadata() {
5571
.addColumn(TEST_COLUMN_5, DataType.DOUBLE)
5672
.addColumn(TEST_COLUMN_6, DataType.TEXT)
5773
.addColumn(TEST_COLUMN_7, BLOB)
74+
.addColumn(TEST_COLUMN_8, DataType.DATE)
75+
.addColumn(TEST_COLUMN_9, DataType.TIME)
76+
.addColumn(TEST_COLUMN_10, DataType.TIMESTAMP)
77+
.addColumn(TEST_COLUMN_11, DataType.TIMESTAMPTZ)
5878
.addColumn(Attribute.BEFORE_PREFIX + TEST_COLUMN_4, DataType.FLOAT)
5979
.addColumn(Attribute.BEFORE_PREFIX + TEST_COLUMN_5, DataType.DOUBLE)
6080
.addColumn(Attribute.BEFORE_PREFIX + TEST_COLUMN_6, DataType.TEXT)
@@ -85,6 +105,10 @@ public static ObjectNode getOutputDataWithMetadata() {
85105
rootNode.put(TEST_COLUMN_5, TEST_VALUE_DOUBLE);
86106
rootNode.put(TEST_COLUMN_6, TEST_VALUE_TEXT);
87107
rootNode.put(TEST_COLUMN_7, TEST_VALUE_BLOB);
108+
rootNode.put(TEST_COLUMN_8, TEST_VALUE_DATE.toString());
109+
rootNode.put(TEST_COLUMN_9, TEST_VALUE_TIME.toString());
110+
rootNode.put(TEST_COLUMN_10, TEST_VALUE_DATE_TIME.toString());
111+
rootNode.put(TEST_COLUMN_11, TEST_VALUE_INSTANT.toString());
88112
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_4, TEST_VALUE_FLOAT);
89113
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_5, TEST_VALUE_DOUBLE);
90114
rootNode.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_6, TEST_VALUE_TEXT);
@@ -112,6 +136,11 @@ public static ObjectNode getOutputDataWithoutMetadata() {
112136
rootNode.put(TEST_COLUMN_5, TEST_VALUE_DOUBLE);
113137
rootNode.put(TEST_COLUMN_6, TEST_VALUE_TEXT);
114138
rootNode.put(TEST_COLUMN_7, TEST_VALUE_BLOB);
139+
rootNode.put(TEST_COLUMN_8, TEST_VALUE_DATE.toString());
140+
rootNode.put(TEST_COLUMN_9, TEST_VALUE_TIME.toString());
141+
rootNode.put(TEST_COLUMN_10, TEST_VALUE_DATE_TIME.toString());
142+
rootNode.put(TEST_COLUMN_11, TEST_VALUE_INSTANT.toString());
143+
115144
return rootNode;
116145
}
117146

@@ -124,6 +153,10 @@ public static List<String> getColumnsListOfMetadata() {
124153
projectedColumns.add(TEST_COLUMN_5);
125154
projectedColumns.add(TEST_COLUMN_6);
126155
projectedColumns.add(TEST_COLUMN_7);
156+
projectedColumns.add(TEST_COLUMN_8);
157+
projectedColumns.add(TEST_COLUMN_9);
158+
projectedColumns.add(TEST_COLUMN_10);
159+
projectedColumns.add(TEST_COLUMN_11);
127160
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_4);
128161
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_5);
129162
projectedColumns.add(Attribute.BEFORE_PREFIX + TEST_COLUMN_6);
@@ -150,6 +183,10 @@ public static Map<String, DataType> getColumnData() {
150183
columnData.put(TEST_COLUMN_5, DataType.DOUBLE);
151184
columnData.put(TEST_COLUMN_6, DataType.TEXT);
152185
columnData.put(TEST_COLUMN_7, BLOB);
186+
columnData.put(TEST_COLUMN_8, DataType.DATE);
187+
columnData.put(TEST_COLUMN_9, DataType.TIME);
188+
columnData.put(TEST_COLUMN_10, DataType.TIMESTAMP);
189+
columnData.put(TEST_COLUMN_11, DataType.TIMESTAMPTZ);
153190
columnData.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_4, DataType.FLOAT);
154191
columnData.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_5, DataType.DOUBLE);
155192
columnData.put(Attribute.BEFORE_PREFIX + TEST_COLUMN_6, DataType.TEXT);
@@ -176,6 +213,10 @@ public static Map<String, Column<?>> createTestValues() {
176213
values.put(TEST_COLUMN_5, DoubleColumn.of(TEST_COLUMN_5, TEST_VALUE_DOUBLE));
177214
values.put(TEST_COLUMN_6, TextColumn.of(TEST_COLUMN_6, TEST_VALUE_TEXT));
178215
values.put(TEST_COLUMN_7, BlobColumn.of(TEST_COLUMN_7, TEST_VALUE_BLOB));
216+
values.put(TEST_COLUMN_8, DateColumn.of(TEST_COLUMN_8, TEST_VALUE_DATE));
217+
values.put(TEST_COLUMN_9, TimeColumn.of(TEST_COLUMN_9, TEST_VALUE_TIME));
218+
values.put(TEST_COLUMN_10, TimestampColumn.of(TEST_COLUMN_10, TEST_VALUE_DATE_TIME));
219+
values.put(TEST_COLUMN_11, TimestampTZColumn.of(TEST_COLUMN_11, TEST_VALUE_INSTANT));
179220
values.put(
180221
Attribute.BEFORE_PREFIX + TEST_COLUMN_4,
181222
FloatColumn.of(Attribute.BEFORE_PREFIX + TEST_COLUMN_4, TEST_VALUE_FLOAT));
@@ -219,6 +260,14 @@ public static String getSourceTestValue(DataType dataType) {
219260
return TEST_VALUE_BLOB_BASE64;
220261
case BOOLEAN:
221262
return Boolean.toString(TEST_VALUE_BOOLEAN);
263+
case DATE:
264+
return TEST_VALUE_DATE.toString();
265+
case TIME:
266+
return TEST_VALUE_TIME.toString();
267+
case TIMESTAMP:
268+
return TEST_VALUE_DATE_TIME.toString();
269+
case TIMESTAMPTZ:
270+
return TEST_VALUE_INSTANT.toString();
222271
case TEXT:
223272
default:
224273
return TEST_VALUE_TEXT;

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/dataexport/producer/CsvProducerTaskTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ void process_withEmptyResultList_shouldReturnEmptyString() {
3939
@Test
4040
void process_withValidResultList_shouldReturnValidCsvString() {
4141
String expectedOutput =
42-
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl";
42+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z";
4343
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
4444
Result result = new ResultImpl(values, mockMetadata);
4545
List<Result> resultList = new ArrayList<>();
@@ -52,7 +52,7 @@ void process_withValidResultList_shouldReturnValidCsvString() {
5252
void process_withValidResultListWithMetadata_shouldReturnValidCsvString() {
5353
csvProducerTask = new CsvProducerTask(true, projectedColumns, mockMetadata, columnData, ",");
5454
String expectedOutput =
55-
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
55+
"9007199254740992,2147483647,true,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,2000-01-01,01:01:01,2000-01-01T01:01,1970-01-21T03:20:41.740Z,0.000000000000000000000000000000000000000000001401298464324817,0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000049,test value,YmxvYiB0ZXN0IHZhbHVl,txt value 464654654,2147483647,2147483647,9007199254740992,9007199254740992,test value,2147483647,2147483647,9007199254740992,9007199254740992";
5656
Map<String, Column<?>> values = UnitTestUtils.createTestValues();
5757
Result result = new ResultImpl(values, mockMetadata);
5858
List<Result> resultList = new ArrayList<>();

‎data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,19 @@
1010
import com.scalar.db.io.BooleanColumn;
1111
import com.scalar.db.io.Column;
1212
import com.scalar.db.io.DataType;
13+
import com.scalar.db.io.DateColumn;
1314
import com.scalar.db.io.DoubleColumn;
1415
import com.scalar.db.io.FloatColumn;
1516
import com.scalar.db.io.IntColumn;
1617
import com.scalar.db.io.TextColumn;
18+
import com.scalar.db.io.TimeColumn;
19+
import com.scalar.db.io.TimestampColumn;
20+
import com.scalar.db.io.TimestampTZColumn;
1721
import java.nio.charset.StandardCharsets;
22+
import java.time.Instant;
23+
import java.time.LocalDate;
24+
import java.time.LocalDateTime;
25+
import java.time.LocalTime;
1826
import java.util.Base64;
1927
import java.util.stream.Stream;
2028
import org.junit.jupiter.api.Test;
@@ -61,7 +69,36 @@ private static Stream<Arguments> provideColumnsForCreateColumnFromValue() {
6169
"blobColumn",
6270
Base64.getEncoder().encodeToString("binary".getBytes(StandardCharsets.UTF_8)),
6371
BlobColumn.of("blobColumn", "binary".getBytes(StandardCharsets.UTF_8))),
64-
Arguments.of(DataType.BLOB, "blobColumn", null, BlobColumn.ofNull("blobColumn")));
72+
Arguments.of(DataType.BLOB, "blobColumn", null, BlobColumn.ofNull("blobColumn")),
73+
Arguments.of(
74+
DataType.DATE,
75+
"dateColumn",
76+
LocalDate.of(2000, 1, 1).toString(),
77+
DateColumn.of("dateColumn", LocalDate.of(2000, 1, 1))),
78+
Arguments.of(DataType.DATE, "dateColumn", null, DateColumn.ofNull("dateColumn")),
79+
Arguments.of(
80+
DataType.TIME,
81+
"timeColumn",
82+
LocalTime.of(1, 1, 1).toString(),
83+
TimeColumn.of("timeColumn", LocalTime.of(1, 1, 1))),
84+
Arguments.of(DataType.TIME, "timeColumn", null, TimeColumn.ofNull("timeColumn")),
85+
Arguments.of(
86+
DataType.TIMESTAMP,
87+
"timestampColumn",
88+
LocalDateTime.of(2000, 1, 1, 1, 1).toString(),
89+
TimestampColumn.of("timestampColumn", LocalDateTime.of(2000, 1, 1, 1, 1))),
90+
Arguments.of(
91+
DataType.TIMESTAMP, "timestampColumn", null, TimestampColumn.ofNull("timestampColumn")),
92+
Arguments.of(
93+
DataType.TIMESTAMPTZ,
94+
"timestampTZColumn",
95+
Instant.ofEpochMilli(1940041740).toString(),
96+
TimestampTZColumn.of("timestampTZColumn", Instant.ofEpochMilli(1940041740))),
97+
Arguments.of(
98+
DataType.TIMESTAMPTZ,
99+
"timestampTZColumn",
100+
null,
101+
TimestampTZColumn.ofNull("timestampTZColumn")));
65102
}
66103

67104
@ParameterizedTest

0 commit comments

Comments
 (0)