Skip to content

Commit 3361cae

Browse files
cirnoooo123cirnoooo123
committed
Merge remote-tracking branch 'origin/main' into main
2 parents e453555 + 4f6f108 commit 3361cae

File tree

37 files changed

+604
-108
lines changed

37 files changed

+604
-108
lines changed

‎adapter/adapter-csv/src/main/java/com/hufudb/openhufu/owner/adapter/csv/CsvAdapter.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.hufudb.openhufu.owner.adapter.AdapterConfig;
66
import com.hufudb.openhufu.plan.LeafPlan;
77
import com.hufudb.openhufu.plan.Plan;
8-
import com.hufudb.openhufu.proto.OpenHuFuData.ColumnType;
98
import java.io.File;
109
import java.io.IOException;
1110
import java.nio.file.Files;
@@ -15,7 +14,6 @@
1514
import java.util.Map;
1615
import java.util.Map.Entry;
1716
import java.util.stream.Collectors;
18-
import java.util.stream.Stream;
1917
import com.hufudb.openhufu.data.schema.Schema;
2018
import com.hufudb.openhufu.data.schema.SchemaManager;
2119
import com.hufudb.openhufu.data.schema.TableSchema;
@@ -52,7 +50,7 @@ public CsvAdapter(AdapterConfig config) {
5250
void loadTables(String csvDir, String delimiter) throws IOException {
5351
File base = new File(csvDir);
5452
if (!base.exists()) {
55-
throw new OpenHuFuException(ErrorCode.CSV_URL_NOT_EXISTS, csvDir);
53+
throw new OpenHuFuException(ErrorCode.CSV_URL_NOT_FOUND, csvDir);
5654
}
5755
if (!base.isDirectory()) {
5856
throw new OpenHuFuException(ErrorCode.CSV_URL_IS_NOT_FOLDER, csvDir);

‎adapter/adapter-csv/src/main/java/com/hufudb/openhufu/owner/adapter/csv/CsvTable.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.hufudb.openhufu.owner.adapter.csv;
22

3+
import com.hufudb.openhufu.common.exception.ErrorCode;
4+
import com.hufudb.openhufu.common.exception.OpenHuFuException;
35
import java.io.IOException;
46
import java.nio.charset.StandardCharsets;
57
import java.nio.file.Path;
@@ -154,7 +156,7 @@ DataSet scanWithSchema(Schema outSchema, List<Integer> mapping) {
154156
mappers.add(row -> row.get(actualColumnIdx));
155157
break;
156158
default:
157-
throw new RuntimeException("Unsupport type for csv adapter");
159+
throw new OpenHuFuException(ErrorCode.DATA_TYPE_NOT_SUPPORT, outType);
158160
}
159161
}
160162
try {

‎adapter/adapter-csv/src/test/java/com/hufudb/openhufu/owner/adapter/csv/CsvAdapterTest.java‎

Lines changed: 16 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,7 @@
1010
import java.sql.Date;
1111
import java.sql.Time;
1212
import java.sql.Timestamp;
13-
import java.util.ArrayList;
1413
import java.util.Calendar;
15-
import java.util.Collections;
1614
import java.util.HashSet;
1715
import java.util.List;
1816
import java.util.Set;
@@ -42,7 +40,7 @@ public class CsvAdapterTest {
4240

4341
@Test
4442
public void testLoadSingleFile() {
45-
URL source = CsvAdapterTest.class.getClassLoader().getResource("data/test1.csv");
43+
URL source = CsvAdapterTest.class.getClassLoader().getResource("data");
4644
CsvAdapterFactory factory = new CsvAdapterFactory();
4745
AdapterConfig config = new AdapterConfig();
4846
config.url = source.getPath();
@@ -51,8 +49,14 @@ public void testLoadSingleFile() {
5149
Adapter adapter = factory.create(config);
5250
SchemaManager manager = adapter.getSchemaManager();
5351
List<TableSchema> schemas = manager.getAllLocalTable();
54-
assertEquals(1, schemas.size());
55-
assertEquals("test1", schemas.get(0).getName());
52+
assertEquals(4, schemas.size());
53+
assertEquals(new HashSet<>() {{
54+
add("test1");
55+
add("test2");
56+
add("test3");
57+
add("test4");
58+
}},
59+
schemas.stream().map(TableSchema::getName).collect(Collectors.toSet()));
5660
}
5761

5862
@Test
@@ -92,8 +96,8 @@ public void testQuery() {
9296
t4.setActualName("test4");
9397
t4.setPublishedName("traffic");
9498
t4.setPublishedColumns(ImmutableList.of(
95-
new PojoColumnDesc("id", ColumnTypeWrapper.INT, ModifierWrapper.PUBLIC, 0),
96-
new PojoColumnDesc("location", ColumnTypeWrapper.POINT, ModifierWrapper.PUBLIC, 1)));
99+
new PojoColumnDesc("id", ColumnTypeWrapper.INT, ModifierWrapper.PUBLIC, 0),
100+
new PojoColumnDesc("location", ColumnTypeWrapper.POINT, ModifierWrapper.PUBLIC, 1)));
97101
manager.addPublishedTable(t1);
98102
manager.addPublishedTable(t2);
99103
manager.addPublishedTable(t3);
@@ -198,7 +202,7 @@ public void testQuery() {
198202
date.set(Calendar.YEAR, 2018);
199203
date.set(Calendar.MONTH, 8); // Note: calendar month starts with 0
200204
date.set(Calendar.DAY_OF_MONTH, 1);
201-
Expression dateCmp = ExpressionFactory.createBinaryOperator(OperatorType.LT, ColumnType.BOOLEAN,
205+
Expression dateCmp = ExpressionFactory.createBinaryOperator(OperatorType.LT, ColumnType.BOOLEAN,
202206
ExpressionFactory.createInputRef(1, ColumnType.DATE, Modifier.PUBLIC),
203207
ExpressionFactory.createLiteral(ColumnType.DATE, date));
204208
plan.setWhereExps(ImmutableList.of(dateCmp));
@@ -218,7 +222,7 @@ public void testQuery() {
218222
time.set(Calendar.HOUR_OF_DAY, 10);
219223
time.set(Calendar.MINUTE, 14);
220224
time.set(Calendar.SECOND, 45);
221-
Expression timeCmp = ExpressionFactory.createBinaryOperator(OperatorType.LT, ColumnType.BOOLEAN,
225+
Expression timeCmp = ExpressionFactory.createBinaryOperator(OperatorType.LT, ColumnType.BOOLEAN,
222226
ExpressionFactory.createInputRef(2, ColumnType.TIME, Modifier.PUBLIC),
223227
ExpressionFactory.createLiteral(ColumnType.TIME, time));
224228
plan.setWhereExps(ImmutableList.of(timeCmp));
@@ -241,9 +245,9 @@ public void testQuery() {
241245
ts.set(Calendar.HOUR_OF_DAY, 9);
242246
ts.set(Calendar.MINUTE, 5);
243247
ts.set(Calendar.SECOND, 10);
244-
Expression tsCmp = ExpressionFactory.createBinaryOperator(OperatorType.GE, ColumnType.BOOLEAN,
245-
ExpressionFactory.createInputRef(3, ColumnType.TIMESTAMP, Modifier.PUBLIC),
246-
ExpressionFactory.createLiteral(ColumnType.TIMESTAMP, ts));
248+
Expression tsCmp = ExpressionFactory.createBinaryOperator(OperatorType.GE, ColumnType.BOOLEAN,
249+
ExpressionFactory.createInputRef(3, ColumnType.TIMESTAMP, Modifier.PUBLIC),
250+
ExpressionFactory.createLiteral(ColumnType.TIMESTAMP, ts));
247251
plan.setWhereExps(ImmutableList.of(tsCmp));
248252
result = adapter.query(plan);
249253
it = result.getIterator();
@@ -273,28 +277,4 @@ public void testQuery() {
273277
assertFalse(it.next());
274278
result.close();
275279
}
276-
277-
@Test
278-
public void testLoadDir() {
279-
Set<String> tableNames = new HashSet<>(){{
280-
add("customer");
281-
add("lineitem");
282-
add("nation");
283-
add("orders");
284-
add("part");
285-
add("partsupp");
286-
add("region");
287-
add("supplier");
288-
}};
289-
URL source = CsvAdapterTest.class.getClassLoader().getResource("tpc-h");
290-
CsvAdapterFactory factory = new CsvAdapterFactory();
291-
AdapterConfig config = new AdapterConfig();
292-
config.url = source.getPath();
293-
config.datasource = DataSourceType.CSV;
294-
config.delimiter = "|";
295-
Adapter adapter = factory.create(config);
296-
SchemaManager manager = adapter.getSchemaManager();
297-
List<TableSchema> schemas = manager.getAllLocalTable();
298-
assertEquals(tableNames, schemas.stream().map(TableSchema::getName).collect(Collectors.toSet()));
299-
}
300280
}

‎adapter/adapter-csv/src/test/java/com/hufudb/openhufu/owner/adapter/csv/CsvTableTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@
1919
public class CsvTableTest {
2020
@Test
2121
public void testScanWithSchema() throws IOException {
22-
URL source = CsvTableTest.class.getClassLoader().getResource("data/test2.csv");
23-
CsvTable table = new CsvTable("test2", null, Paths.get(source.getPath()), ",");
22+
URL data = CsvTableTest.class.getClassLoader().getResource("data/test2.csv");
23+
URL schema = CsvTableTest.class.getClassLoader().getResource("data/test2.scm");
24+
CsvTable table = new CsvTable("test2", Paths.get(schema.getPath()), Paths.get(data.getPath()), ",");
2425
Schema.Builder builder = Schema.newBuilder();
2526
builder.add("Department", ColumnType.STRING, Modifier.PUBLIC);
2627
builder.add("Name", ColumnType.STRING, Modifier.PUBLIC);

‎adapter/adapter-csv/src/test/resources/data/test1.scm‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
age, workclass, fnlwgt, education, education-num, marital-status, occupation, relationship, race, sex, capital-gain, capital-loss, hours-per-week, native-country, income
2+
INT, STRING, LONG, STRING, INT, STRING, STRING, STRING, STRING, STRING, INT, INT, INT, STRING, STRING

‎adapter/adapter-csv/src/test/resources/data/test2.scm‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
name, age, score, dept_name, weight
2+
STRING, INT, INT, STRING, DOUBLE

‎adapter/adapter-csv/src/test/resources/data/test3.scm‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
license, cur_date, cur_time, time_stamp
2+
LONG, DATE, TIME, TIMESTAMP

‎adapter/adapter-csv/src/test/resources/data/test4.scm‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
id,location
2+
INT, POINT

‎benchmark/src/main/resources/tables.json‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,38 @@
11
[
2+
{
3+
"tableName": "customer",
4+
"localTables": [
5+
{
6+
"endpoint": "localhost:12345",
7+
"localName": "customer"
8+
},
9+
{
10+
"endpoint": "localhost:12346",
11+
"localName": "customer"
12+
},
13+
{
14+
"endpoint": "localhost:12347",
15+
"localName": "customer"
16+
}
17+
]
18+
},
19+
{
20+
"tableName": "lineitem",
21+
"localTables": [
22+
{
23+
"endpoint": "localhost:12345",
24+
"localName": "lineitem"
25+
},
26+
{
27+
"endpoint": "localhost:12346",
28+
"localName": "lineitem"
29+
},
30+
{
31+
"endpoint": "localhost:12347",
32+
"localName": "lineitem"
33+
}
34+
]
35+
},
236
{
337
"tableName": "nation",
438
"localTables": [
@@ -15,5 +49,90 @@
1549
"localName": "nation"
1650
}
1751
]
52+
},
53+
{
54+
"tableName": "orders",
55+
"localTables": [
56+
{
57+
"endpoint": "localhost:12345",
58+
"localName": "orders"
59+
},
60+
{
61+
"endpoint": "localhost:12346",
62+
"localName": "orders"
63+
},
64+
{
65+
"endpoint": "localhost:12347",
66+
"localName": "orders"
67+
}
68+
]
69+
},
70+
{
71+
"tableName": "part",
72+
"localTables": [
73+
{
74+
"endpoint": "localhost:12345",
75+
"localName": "part"
76+
},
77+
{
78+
"endpoint": "localhost:12346",
79+
"localName": "part"
80+
},
81+
{
82+
"endpoint": "localhost:12347",
83+
"localName": "part"
84+
}
85+
]
86+
},
87+
{
88+
"tableName": "partsupp",
89+
"localTables": [
90+
{
91+
"endpoint": "localhost:12345",
92+
"localName": "partsupp"
93+
},
94+
{
95+
"endpoint": "localhost:12346",
96+
"localName": "partsupp"
97+
},
98+
{
99+
"endpoint": "localhost:12347",
100+
"localName": "partsupp"
101+
}
102+
]
103+
},
104+
{
105+
"tableName": "region",
106+
"localTables": [
107+
{
108+
"endpoint": "localhost:12345",
109+
"localName": "region"
110+
},
111+
{
112+
"endpoint": "localhost:12346",
113+
"localName": "region"
114+
},
115+
{
116+
"endpoint": "localhost:12347",
117+
"localName": "region"
118+
}
119+
]
120+
},
121+
{
122+
"tableName": "supplier",
123+
"localTables": [
124+
{
125+
"endpoint": "localhost:12345",
126+
"localName": "supplier"
127+
},
128+
{
129+
"endpoint": "localhost:12346",
130+
"localName": "supplier"
131+
},
132+
{
133+
"endpoint": "localhost:12347",
134+
"localName": "supplier"
135+
}
136+
]
18137
}
19138
]

‎common/src/main/java/com/hufudb/openhufu/common/exception/BaseErrorCode.java‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

0 commit comments

Comments
 (0)