Skip to content

Commit 431369a

Browse files
HTHouHTHou
committed
Add IT
1 parent 5210167 commit 431369a

File tree

14 files changed

+552
-183
lines changed

14 files changed

+552
-183
lines changed

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -651,6 +651,12 @@ public CommonConfig setAuditableOperationResult(String auditableOperationResult)
651651
return this;
652652
}
653653

654+
@Override
655+
public CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit) {
656+
setProperty("restrict_object_limit", String.valueOf(restrictObjectLimit));
657+
return this;
658+
}
659+
654660
// For part of the log directory
655661
public String getClusterConfigStr() {
656662
return fromConsensusFullNameToAbbr(properties.getProperty(CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS))

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -684,4 +684,11 @@ public CommonConfig setAuditableOperationResult(String auditableOperationResult)
684684
cnConfig.setAuditableOperationResult(auditableOperationResult);
685685
return this;
686686
}
687+
688+
@Override
689+
public CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit) {
690+
cnConfig.setRestrictObjectLimit(restrictObjectLimit);
691+
dnConfig.setRestrictObjectLimit(restrictObjectLimit);
692+
return this;
693+
}
687694
}

‎integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,10 @@ public String getSystemDir() {
169169
return getDataNodeDir() + File.separator + "system";
170170
}
171171

172+
public String getDataNodeObjectDir() {
173+
return getDataNodeDir() + File.separator + "data" + File.separator + "object";
174+
}
175+
172176
@Override
173177
protected MppJVMConfig initVMConfig() {
174178
return MppJVMConfig.builder()

‎integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,4 +477,9 @@ public CommonConfig setAuditableOperationLevel(String auditableOperationLevel) {
477477
public CommonConfig setAuditableOperationResult(String auditableOperationResult) {
478478
return this;
479479
}
480+
481+
@Override
482+
public CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit) {
483+
return this;
484+
}
480485
}

‎integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,4 +211,6 @@ default CommonConfig setDefaultDatabaseLevel(int defaultDatabaseLevel) {
211211
CommonConfig setAuditableOperationLevel(String auditableOperationLevel);
212212

213213
CommonConfig setAuditableOperationResult(String auditableOperationResult);
214+
215+
CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit);
214216
}

‎integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.iotdb.relational.it.session;
20+
21+
import org.apache.iotdb.isession.ITableSession;
22+
import org.apache.iotdb.isession.SessionDataSet;
23+
import org.apache.iotdb.it.env.EnvFactory;
24+
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
25+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
26+
import org.apache.iotdb.itbase.category.TableClusterIT;
27+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
28+
import org.apache.iotdb.rpc.IoTDBConnectionException;
29+
import org.apache.iotdb.rpc.StatementExecutionException;
30+
import org.apache.iotdb.rpc.TSStatusCode;
31+
32+
import com.google.common.io.BaseEncoding;
33+
import org.apache.tsfile.enums.ColumnCategory;
34+
import org.apache.tsfile.enums.TSDataType;
35+
import org.apache.tsfile.utils.Binary;
36+
import org.apache.tsfile.utils.BytesUtils;
37+
import org.apache.tsfile.write.record.Tablet;
38+
import org.junit.After;
39+
import org.junit.AfterClass;
40+
import org.junit.Assert;
41+
import org.junit.Before;
42+
import org.junit.BeforeClass;
43+
import org.junit.Test;
44+
import org.junit.experimental.categories.Category;
45+
import org.junit.runner.RunWith;
46+
47+
import java.io.File;
48+
import java.io.IOException;
49+
import java.nio.charset.StandardCharsets;
50+
import java.nio.file.Files;
51+
import java.nio.file.Paths;
52+
import java.util.ArrayList;
53+
import java.util.Arrays;
54+
import java.util.List;
55+
56+
import static org.junit.Assert.assertNull;
57+
58+
@RunWith(IoTDBTestRunner.class)
59+
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
60+
public class IoTDBObjectInsertIT {
61+
62+
@BeforeClass
63+
public static void classSetUp() throws Exception {
64+
EnvFactory.getEnv().initClusterEnvironment();
65+
}
66+
67+
@Before
68+
public void setUp() throws Exception {
69+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
70+
session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1");
71+
}
72+
}
73+
74+
@After
75+
public void tearDown() throws Exception {
76+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
77+
session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
78+
}
79+
}
80+
81+
@AfterClass
82+
public static void classTearDown() {
83+
EnvFactory.getEnv().cleanClusterEnvironment();
84+
}
85+
86+
@Test
87+
public void insertObjectTest()
88+
throws IoTDBConnectionException, StatementExecutionException, IOException {
89+
String testObject =
90+
System.getProperty("user.dir")
91+
+ File.separator
92+
+ "target"
93+
+ File.separator
94+
+ "test-classes"
95+
+ File.separator
96+
+ "ainode-example"
97+
+ File.separator
98+
+ "model.pt";
99+
File object = new File(testObject);
100+
101+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
102+
session.executeNonQueryStatement("USE \"db1\"");
103+
// insert table data by tablet
104+
List<String> columnNameList =
105+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
106+
List<TSDataType> dataTypeList =
107+
Arrays.asList(
108+
TSDataType.STRING,
109+
TSDataType.STRING,
110+
TSDataType.STRING,
111+
TSDataType.FLOAT,
112+
TSDataType.OBJECT);
113+
List<ColumnCategory> columnTypeList =
114+
new ArrayList<>(
115+
Arrays.asList(
116+
ColumnCategory.TAG,
117+
ColumnCategory.TAG,
118+
ColumnCategory.TAG,
119+
ColumnCategory.FIELD,
120+
ColumnCategory.FIELD));
121+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
122+
int rowIndex = tablet.getRowSize();
123+
tablet.addTimestamp(rowIndex, 1);
124+
tablet.addValue(rowIndex, 0, "1");
125+
tablet.addValue(rowIndex, 1, "5");
126+
tablet.addValue(rowIndex, 2, "3");
127+
tablet.addValue(rowIndex, 3, 37.6F);
128+
tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject)));
129+
session.insert(tablet);
130+
tablet.reset();
131+
132+
try (SessionDataSet dataSet =
133+
session.executeQueryStatement("select file from object_table where time = 1")) {
134+
SessionDataSet.DataIterator iterator = dataSet.iterator();
135+
while (iterator.next()) {
136+
Assert.assertEquals(
137+
BytesUtils.parseObjectByteArrayToString(BytesUtils.longToBytes(object.length())),
138+
iterator.getString(1));
139+
}
140+
}
141+
142+
try (SessionDataSet dataSet =
143+
session.executeQueryStatement(
144+
"select READ_OBJECT(file) from object_table where time = 1")) {
145+
SessionDataSet.DataIterator iterator = dataSet.iterator();
146+
while (iterator.next()) {
147+
Binary binary = iterator.getBlob(1);
148+
Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues());
149+
}
150+
}
151+
}
152+
// test object file path
153+
boolean success = false;
154+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
155+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
156+
File objectDir = new File(objectDirStr);
157+
if (objectDir.exists() && objectDir.isDirectory()) {
158+
File[] regionDirs = objectDir.listFiles();
159+
if (regionDirs != null) {
160+
for (File regionDir : regionDirs) {
161+
if (regionDir.isDirectory()) {
162+
File objectFile =
163+
new File(
164+
regionDir,
165+
convertPathString("object_table")
166+
+ File.separator
167+
+ convertPathString("1")
168+
+ File.separator
169+
+ convertPathString("5")
170+
+ File.separator
171+
+ convertPathString("3")
172+
+ File.separator
173+
+ convertPathString("file")
174+
+ File.separator
175+
+ "1.bin");
176+
if (objectFile.exists() && objectFile.isFile()) {
177+
success = true;
178+
}
179+
}
180+
}
181+
}
182+
}
183+
}
184+
Assert.assertTrue(success);
185+
}
186+
187+
@Test
188+
public void insertObjectSegmentsTest()
189+
throws IoTDBConnectionException, StatementExecutionException, IOException {
190+
String testObject =
191+
System.getProperty("user.dir")
192+
+ File.separator
193+
+ "target"
194+
+ File.separator
195+
+ "test-classes"
196+
+ File.separator
197+
+ "ainode-example"
198+
+ File.separator
199+
+ "model.pt";
200+
byte[] objectBytes = Files.readAllBytes(Paths.get(testObject));
201+
List<byte[]> objectSegments = new ArrayList<>();
202+
for (int i = 0; i < objectBytes.length; i += 512) {
203+
objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length)));
204+
}
205+
206+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
207+
session.executeNonQueryStatement("USE \"db1\"");
208+
// insert table data by tablet
209+
List<String> columnNameList =
210+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
211+
List<TSDataType> dataTypeList =
212+
Arrays.asList(
213+
TSDataType.STRING,
214+
TSDataType.STRING,
215+
TSDataType.STRING,
216+
TSDataType.FLOAT,
217+
TSDataType.OBJECT);
218+
List<ColumnCategory> columnTypeList =
219+
new ArrayList<>(
220+
Arrays.asList(
221+
ColumnCategory.TAG,
222+
ColumnCategory.TAG,
223+
ColumnCategory.TAG,
224+
ColumnCategory.FIELD,
225+
ColumnCategory.FIELD));
226+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
227+
for (int i = 0; i < objectSegments.size() - 1; i++) {
228+
int rowIndex = tablet.getRowSize();
229+
tablet.addTimestamp(rowIndex, 1);
230+
tablet.addValue(rowIndex, 0, "1");
231+
tablet.addValue(rowIndex, 1, "5");
232+
tablet.addValue(rowIndex, 2, "3");
233+
tablet.addValue(rowIndex, 3, 37.6F);
234+
tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i));
235+
session.insert(tablet);
236+
tablet.reset();
237+
}
238+
239+
try (SessionDataSet dataSet =
240+
session.executeQueryStatement("select file from object_table where time = 1")) {
241+
SessionDataSet.DataIterator iterator = dataSet.iterator();
242+
while (iterator.next()) {
243+
assertNull(iterator.getString(1));
244+
}
245+
}
246+
247+
// insert segment with wrong offset
248+
try {
249+
int rowIndex = tablet.getRowSize();
250+
tablet.addTimestamp(rowIndex, 1);
251+
tablet.addValue(rowIndex, 0, "1");
252+
tablet.addValue(rowIndex, 1, "5");
253+
tablet.addValue(rowIndex, 2, "3");
254+
tablet.addValue(rowIndex, 3, 37.6F);
255+
tablet.addValue(rowIndex, 4, false, 512L, objectSegments.get(1));
256+
session.insert(tablet);
257+
} catch (StatementExecutionException e) {
258+
Assert.assertEquals(TSStatusCode.OBJECT_INSERT_ERROR.getStatusCode(), e.getStatusCode());
259+
Assert.assertEquals(
260+
String.format(
261+
"741: The file length %d is not equal to the offset %d",
262+
((objectSegments.size() - 1) * 512), 512L),
263+
e.getMessage());
264+
} finally {
265+
tablet.reset();
266+
}
267+
268+
// last segment
269+
int rowIndex = tablet.getRowSize();
270+
tablet.addTimestamp(rowIndex, 1);
271+
tablet.addValue(rowIndex, 0, "1");
272+
tablet.addValue(rowIndex, 1, "5");
273+
tablet.addValue(rowIndex, 2, "3");
274+
tablet.addValue(rowIndex, 3, 37.6F);
275+
tablet.addValue(
276+
rowIndex,
277+
4,
278+
true,
279+
(objectSegments.size() - 1) * 512L,
280+
objectSegments.get(objectSegments.size() - 1));
281+
session.insert(tablet);
282+
tablet.reset();
283+
284+
try (SessionDataSet dataSet =
285+
session.executeQueryStatement(
286+
"select READ_OBJECT(file) from object_table where time = 1")) {
287+
SessionDataSet.DataIterator iterator = dataSet.iterator();
288+
while (iterator.next()) {
289+
Binary binary = iterator.getBlob(1);
290+
Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues());
291+
}
292+
}
293+
}
294+
295+
// test object file path
296+
boolean success = false;
297+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
298+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
299+
File objectDir = new File(objectDirStr);
300+
if (objectDir.exists() && objectDir.isDirectory()) {
301+
File[] regionDirs = objectDir.listFiles();
302+
if (regionDirs != null) {
303+
for (File regionDir : regionDirs) {
304+
if (regionDir.isDirectory()) {
305+
File objectFile =
306+
new File(
307+
regionDir,
308+
convertPathString("object_table")
309+
+ File.separator
310+
+ convertPathString("1")
311+
+ File.separator
312+
+ convertPathString("5")
313+
+ File.separator
314+
+ convertPathString("3")
315+
+ File.separator
316+
+ convertPathString("file")
317+
+ File.separator
318+
+ "1.bin");
319+
if (objectFile.exists() && objectFile.isFile()) {
320+
success = true;
321+
}
322+
}
323+
}
324+
}
325+
}
326+
}
327+
Assert.assertTrue(success);
328+
}
329+
330+
protected String convertPathString(String path) {
331+
return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8));
332+
}
333+
}

0 commit comments

Comments
 (0)