Skip to content

Commit b5a5549

Browse files
HTHouHTHou
committed
add some IT
1 parent 6bbf453 commit b5a5549

File tree

2 files changed

+277
-2
lines changed

2 files changed

+277
-2
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.relational.it.session;
21+
22+
import org.apache.iotdb.isession.ITableSession;
23+
import org.apache.iotdb.isession.SessionDataSet;
24+
import org.apache.iotdb.it.env.EnvFactory;
25+
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
26+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
27+
import org.apache.iotdb.itbase.category.TableClusterIT;
28+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
29+
import org.apache.iotdb.rpc.IoTDBConnectionException;
30+
import org.apache.iotdb.rpc.StatementExecutionException;
31+
32+
import com.google.common.io.BaseEncoding;
33+
import org.apache.tsfile.enums.ColumnCategory;
34+
import org.apache.tsfile.enums.TSDataType;
35+
import org.apache.tsfile.utils.Binary;
36+
import org.apache.tsfile.write.record.Tablet;
37+
import org.junit.After;
38+
import org.junit.AfterClass;
39+
import org.junit.Assert;
40+
import org.junit.Before;
41+
import org.junit.BeforeClass;
42+
import org.junit.Test;
43+
import org.junit.experimental.categories.Category;
44+
import org.junit.runner.RunWith;
45+
46+
import java.io.File;
47+
import java.io.IOException;
48+
import java.nio.charset.StandardCharsets;
49+
import java.nio.file.Files;
50+
import java.nio.file.Paths;
51+
import java.util.ArrayList;
52+
import java.util.Arrays;
53+
import java.util.List;
54+
55+
import static org.junit.Assert.assertNull;
56+
57+
@RunWith(IoTDBTestRunner.class)
58+
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
59+
public class IoTDBObjectDeleteIT {
60+
61+
@BeforeClass
62+
public static void classSetUp() throws Exception {
63+
EnvFactory.getEnv().initClusterEnvironment();
64+
}
65+
66+
@Before
67+
public void setUp() throws Exception {
68+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
69+
session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1");
70+
}
71+
}
72+
73+
@After
74+
public void tearDown() throws Exception {
75+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
76+
session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
77+
}
78+
}
79+
80+
@AfterClass
81+
public static void classTearDown() {
82+
EnvFactory.getEnv().cleanClusterEnvironment();
83+
}
84+
85+
@Test
86+
public void deleteObjectTest()
87+
throws IoTDBConnectionException, StatementExecutionException, IOException {
88+
String testObject =
89+
System.getProperty("user.dir")
90+
+ File.separator
91+
+ "target"
92+
+ File.separator
93+
+ "test-classes"
94+
+ File.separator
95+
+ "object-example.pt";
96+
97+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
98+
session.executeNonQueryStatement("USE \"db1\"");
99+
// insert table data by tablet
100+
List<String> columnNameList =
101+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
102+
List<TSDataType> dataTypeList =
103+
Arrays.asList(
104+
TSDataType.STRING,
105+
TSDataType.STRING,
106+
TSDataType.STRING,
107+
TSDataType.FLOAT,
108+
TSDataType.OBJECT);
109+
List<ColumnCategory> columnTypeList =
110+
new ArrayList<>(
111+
Arrays.asList(
112+
ColumnCategory.TAG,
113+
ColumnCategory.TAG,
114+
ColumnCategory.TAG,
115+
ColumnCategory.FIELD,
116+
ColumnCategory.FIELD));
117+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
118+
int rowIndex = tablet.getRowSize();
119+
tablet.addTimestamp(rowIndex, 1);
120+
tablet.addValue(rowIndex, 0, "1");
121+
tablet.addValue(rowIndex, 1, "5");
122+
tablet.addValue(rowIndex, 2, "3");
123+
tablet.addValue(rowIndex, 3, 37.6F);
124+
tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject)));
125+
session.insert(tablet);
126+
tablet.reset();
127+
128+
try (SessionDataSet dataSet =
129+
session.executeQueryStatement(
130+
"select READ_OBJECT(file) from object_table where time = 1")) {
131+
SessionDataSet.DataIterator iterator = dataSet.iterator();
132+
while (iterator.next()) {
133+
Binary binary = iterator.getBlob(1);
134+
Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues());
135+
}
136+
session.executeNonQueryStatement("DROP TABLE IF EXISTS object_table");
137+
}
138+
}
139+
140+
// test object file path
141+
boolean success = false;
142+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
143+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
144+
File objectDir = new File(objectDirStr);
145+
if (objectDir.exists() && objectDir.isDirectory()) {
146+
File[] regionDirs = objectDir.listFiles();
147+
if (regionDirs != null) {
148+
for (File regionDir : regionDirs) {
149+
if (regionDir.isDirectory()) {
150+
File objectFile =
151+
new File(
152+
regionDir,
153+
convertPathString("object_table")
154+
+ File.separator
155+
+ convertPathString("1")
156+
+ File.separator
157+
+ convertPathString("5")
158+
+ File.separator
159+
+ convertPathString("3")
160+
+ File.separator
161+
+ convertPathString("file")
162+
+ File.separator
163+
+ "1.bin");
164+
if (objectFile.exists() && objectFile.isFile()) {
165+
success = true;
166+
}
167+
}
168+
}
169+
}
170+
}
171+
}
172+
Assert.assertFalse(success);
173+
}
174+
175+
@Test
176+
public void deleteObjectSegmentsTest()
177+
throws IoTDBConnectionException, StatementExecutionException, IOException {
178+
String testObject =
179+
System.getProperty("user.dir")
180+
+ File.separator
181+
+ "target"
182+
+ File.separator
183+
+ "test-classes"
184+
+ File.separator
185+
+ "object-example.pt";
186+
byte[] objectBytes = Files.readAllBytes(Paths.get(testObject));
187+
List<byte[]> objectSegments = new ArrayList<>();
188+
for (int i = 0; i < objectBytes.length; i += 512) {
189+
objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length)));
190+
}
191+
192+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
193+
session.executeNonQueryStatement("USE \"db1\"");
194+
// insert table data by tablet
195+
List<String> columnNameList =
196+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
197+
List<TSDataType> dataTypeList =
198+
Arrays.asList(
199+
TSDataType.STRING,
200+
TSDataType.STRING,
201+
TSDataType.STRING,
202+
TSDataType.FLOAT,
203+
TSDataType.OBJECT);
204+
List<ColumnCategory> columnTypeList =
205+
new ArrayList<>(
206+
Arrays.asList(
207+
ColumnCategory.TAG,
208+
ColumnCategory.TAG,
209+
ColumnCategory.TAG,
210+
ColumnCategory.FIELD,
211+
ColumnCategory.FIELD));
212+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
213+
for (int i = 0; i < objectSegments.size() - 1; i++) {
214+
int rowIndex = tablet.getRowSize();
215+
tablet.addTimestamp(rowIndex, 1);
216+
tablet.addValue(rowIndex, 0, "1");
217+
tablet.addValue(rowIndex, 1, "5");
218+
tablet.addValue(rowIndex, 2, "3");
219+
tablet.addValue(rowIndex, 3, 37.6F);
220+
tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i));
221+
session.insert(tablet);
222+
tablet.reset();
223+
}
224+
session.executeNonQueryStatement("DELETE FROM object_table where time = 1");
225+
226+
try (SessionDataSet dataSet =
227+
session.executeQueryStatement("select file from object_table where time = 1")) {
228+
SessionDataSet.DataIterator iterator = dataSet.iterator();
229+
while (iterator.next()) {
230+
assertNull(iterator.getString(1));
231+
}
232+
}
233+
}
234+
235+
// test object file path
236+
boolean success = false;
237+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
238+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
239+
File objectDir = new File(objectDirStr);
240+
if (objectDir.exists() && objectDir.isDirectory()) {
241+
File[] regionDirs = objectDir.listFiles();
242+
if (regionDirs != null) {
243+
for (File regionDir : regionDirs) {
244+
if (regionDir.isDirectory()) {
245+
File objectTmpFile =
246+
new File(
247+
regionDir,
248+
convertPathString("object_table")
249+
+ File.separator
250+
+ convertPathString("1")
251+
+ File.separator
252+
+ convertPathString("5")
253+
+ File.separator
254+
+ convertPathString("3")
255+
+ File.separator
256+
+ convertPathString("file")
257+
+ File.separator
258+
+ "1.bin.tmp");
259+
if (objectTmpFile.exists() && objectTmpFile.isFile()) {
260+
success = true;
261+
}
262+
}
263+
}
264+
}
265+
}
266+
}
267+
Assert.assertFalse(success);
268+
}
269+
270+
protected String convertPathString(String path) {
271+
return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8));
272+
}
273+
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2959,7 +2959,9 @@ private void deleteObjectFiles(List<File> matchedObjectDirs, List<TableDeletionE
29592959
matchedObjectDir.toPath(),
29602960
Integer.MAX_VALUE,
29612961
(path, attrs) ->
2962-
attrs.isRegularFile() && path.getFileName().toString().endsWith(".bin"))) {
2962+
attrs.isRegularFile()
2963+
&& (path.getFileName().toString().endsWith(".bin")
2964+
|| path.getFileName().toString().endsWith(".tmp")))) {
29632965
paths.forEach(
29642966
path -> {
29652967
Path relativePath = matchedObjectDir.getParentFile().toPath().relativize(path);
@@ -2985,7 +2987,7 @@ private void deleteObjectFiles(List<File> matchedObjectDirs, List<TableDeletionE
29852987
relativePath.getName(relativePath.getNameCount() - 2).toString()),
29862988
StandardCharsets.UTF_8);
29872989
String fileName = path.getFileName().toString();
2988-
long timestamp = Long.parseLong(fileName.substring(0, fileName.lastIndexOf('.')));
2990+
long timestamp = Long.parseLong(fileName.substring(0, fileName.indexOf('.')));
29892991
logger.info(
29902992
"timestamp {}, measurementId {}, ideviceId {}",
29912993
timestamp,

0 commit comments

Comments
 (0)