Skip to content

Commit a6d65cb

Browse files
HTHouHTHou
authored
Support delete object file (#16945)
* deving * dev * remove empty object dir * optimize drop table * add some IT * Fix error * Add IT * fix check object dir error * fix review
1 parent e879c54 commit a6d65cb

File tree

8 files changed

+611
-4
lines changed

8 files changed

+611
-4
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectDeleteIT.java‎

Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.iotdb.relational.it.session;
21+
22+
import org.apache.iotdb.isession.ITableSession;
23+
import org.apache.iotdb.isession.SessionDataSet;
24+
import org.apache.iotdb.it.env.EnvFactory;
25+
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
26+
import org.apache.iotdb.it.framework.IoTDBTestRunner;
27+
import org.apache.iotdb.itbase.category.TableClusterIT;
28+
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
29+
import org.apache.iotdb.rpc.IoTDBConnectionException;
30+
import org.apache.iotdb.rpc.StatementExecutionException;
31+
32+
import com.google.common.io.BaseEncoding;
33+
import org.apache.tsfile.enums.ColumnCategory;
34+
import org.apache.tsfile.enums.TSDataType;
35+
import org.apache.tsfile.utils.Binary;
36+
import org.apache.tsfile.write.record.Tablet;
37+
import org.junit.After;
38+
import org.junit.AfterClass;
39+
import org.junit.Assert;
40+
import org.junit.Before;
41+
import org.junit.BeforeClass;
42+
import org.junit.Test;
43+
import org.junit.experimental.categories.Category;
44+
import org.junit.runner.RunWith;
45+
46+
import java.io.File;
47+
import java.io.IOException;
48+
import java.nio.charset.StandardCharsets;
49+
import java.nio.file.Files;
50+
import java.nio.file.Paths;
51+
import java.util.ArrayList;
52+
import java.util.Arrays;
53+
import java.util.List;
54+
55+
import static org.junit.Assert.assertNull;
56+
57+
@RunWith(IoTDBTestRunner.class)
58+
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
59+
public class IoTDBObjectDeleteIT {
60+
61+
@BeforeClass
62+
public static void classSetUp() throws Exception {
63+
EnvFactory.getEnv().initClusterEnvironment();
64+
}
65+
66+
@Before
67+
public void setUp() throws Exception {
68+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
69+
session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1");
70+
}
71+
}
72+
73+
@After
74+
public void tearDown() throws Exception {
75+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
76+
session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1");
77+
}
78+
}
79+
80+
@AfterClass
81+
public static void classTearDown() {
82+
EnvFactory.getEnv().cleanClusterEnvironment();
83+
}
84+
85+
@Test
86+
public void dropObjectTableTest()
87+
throws IoTDBConnectionException, StatementExecutionException, IOException {
88+
String testObject =
89+
System.getProperty("user.dir")
90+
+ File.separator
91+
+ "target"
92+
+ File.separator
93+
+ "test-classes"
94+
+ File.separator
95+
+ "object-example.pt";
96+
97+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
98+
session.executeNonQueryStatement("USE \"db1\"");
99+
// insert table data by tablet
100+
List<String> columnNameList =
101+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
102+
List<TSDataType> dataTypeList =
103+
Arrays.asList(
104+
TSDataType.STRING,
105+
TSDataType.STRING,
106+
TSDataType.STRING,
107+
TSDataType.FLOAT,
108+
TSDataType.OBJECT);
109+
List<ColumnCategory> columnTypeList =
110+
new ArrayList<>(
111+
Arrays.asList(
112+
ColumnCategory.TAG,
113+
ColumnCategory.TAG,
114+
ColumnCategory.TAG,
115+
ColumnCategory.FIELD,
116+
ColumnCategory.FIELD));
117+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
118+
int rowIndex = tablet.getRowSize();
119+
tablet.addTimestamp(rowIndex, 1);
120+
tablet.addValue(rowIndex, 0, "1");
121+
tablet.addValue(rowIndex, 1, "5");
122+
tablet.addValue(rowIndex, 2, "3");
123+
tablet.addValue(rowIndex, 3, 37.6F);
124+
tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject)));
125+
session.insert(tablet);
126+
tablet.reset();
127+
128+
try (SessionDataSet dataSet =
129+
session.executeQueryStatement(
130+
"select READ_OBJECT(file) from object_table where time = 1")) {
131+
SessionDataSet.DataIterator iterator = dataSet.iterator();
132+
while (iterator.next()) {
133+
Binary binary = iterator.getBlob(1);
134+
Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues());
135+
}
136+
session.executeNonQueryStatement("DROP TABLE IF EXISTS object_table");
137+
}
138+
}
139+
140+
// test object file path
141+
boolean success = false;
142+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
143+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
144+
File objectDir = new File(objectDirStr);
145+
if (objectDir.exists() && objectDir.isDirectory()) {
146+
File[] regionDirs = objectDir.listFiles();
147+
if (regionDirs != null) {
148+
for (File regionDir : regionDirs) {
149+
if (regionDir.isDirectory()) {
150+
File objectFile =
151+
new File(
152+
regionDir,
153+
convertPathString("object_table")
154+
+ File.separator
155+
+ convertPathString("1")
156+
+ File.separator
157+
+ convertPathString("5")
158+
+ File.separator
159+
+ convertPathString("3")
160+
+ File.separator
161+
+ convertPathString("file")
162+
+ File.separator
163+
+ "1.bin");
164+
if (objectFile.exists() && objectFile.isFile()) {
165+
success = true;
166+
}
167+
}
168+
}
169+
}
170+
}
171+
}
172+
Assert.assertFalse(success);
173+
}
174+
175+
@Test
176+
public void dropObjectColumnTest()
177+
throws IoTDBConnectionException, StatementExecutionException, IOException {
178+
String testObject =
179+
System.getProperty("user.dir")
180+
+ File.separator
181+
+ "target"
182+
+ File.separator
183+
+ "test-classes"
184+
+ File.separator
185+
+ "object-example.pt";
186+
187+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
188+
session.executeNonQueryStatement("USE \"db1\"");
189+
// insert table data by tablet
190+
List<String> columnNameList =
191+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
192+
List<TSDataType> dataTypeList =
193+
Arrays.asList(
194+
TSDataType.STRING,
195+
TSDataType.STRING,
196+
TSDataType.STRING,
197+
TSDataType.FLOAT,
198+
TSDataType.OBJECT);
199+
List<ColumnCategory> columnTypeList =
200+
new ArrayList<>(
201+
Arrays.asList(
202+
ColumnCategory.TAG,
203+
ColumnCategory.TAG,
204+
ColumnCategory.TAG,
205+
ColumnCategory.FIELD,
206+
ColumnCategory.FIELD));
207+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
208+
int rowIndex = tablet.getRowSize();
209+
tablet.addTimestamp(rowIndex, 1);
210+
tablet.addValue(rowIndex, 0, "1");
211+
tablet.addValue(rowIndex, 1, "5");
212+
tablet.addValue(rowIndex, 2, "3");
213+
tablet.addValue(rowIndex, 3, 37.6F);
214+
tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject)));
215+
session.insert(tablet);
216+
tablet.reset();
217+
218+
try (SessionDataSet dataSet =
219+
session.executeQueryStatement(
220+
"select READ_OBJECT(file) from object_table where time = 1")) {
221+
SessionDataSet.DataIterator iterator = dataSet.iterator();
222+
while (iterator.next()) {
223+
Binary binary = iterator.getBlob(1);
224+
Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues());
225+
}
226+
session.executeNonQueryStatement("ALTER TABLE object_table drop column file");
227+
}
228+
}
229+
230+
// test object file path
231+
boolean success = false;
232+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
233+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
234+
File objectDir = new File(objectDirStr);
235+
if (objectDir.exists() && objectDir.isDirectory()) {
236+
File[] regionDirs = objectDir.listFiles();
237+
if (regionDirs != null) {
238+
for (File regionDir : regionDirs) {
239+
if (regionDir.isDirectory()) {
240+
File objectFile =
241+
new File(
242+
regionDir,
243+
convertPathString("object_table")
244+
+ File.separator
245+
+ convertPathString("1")
246+
+ File.separator
247+
+ convertPathString("5")
248+
+ File.separator
249+
+ convertPathString("3")
250+
+ File.separator
251+
+ convertPathString("file")
252+
+ File.separator
253+
+ "1.bin");
254+
if (objectFile.exists() && objectFile.isFile()) {
255+
success = true;
256+
}
257+
}
258+
}
259+
}
260+
}
261+
}
262+
Assert.assertFalse(success);
263+
}
264+
265+
@Test
266+
public void deleteObjectSegmentsTest()
267+
throws IoTDBConnectionException, StatementExecutionException, IOException {
268+
String testObject =
269+
System.getProperty("user.dir")
270+
+ File.separator
271+
+ "target"
272+
+ File.separator
273+
+ "test-classes"
274+
+ File.separator
275+
+ "object-example.pt";
276+
byte[] objectBytes = Files.readAllBytes(Paths.get(testObject));
277+
List<byte[]> objectSegments = new ArrayList<>();
278+
for (int i = 0; i < objectBytes.length; i += 512) {
279+
objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length)));
280+
}
281+
282+
try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
283+
session.executeNonQueryStatement("USE \"db1\"");
284+
// insert table data by tablet
285+
List<String> columnNameList =
286+
Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file");
287+
List<TSDataType> dataTypeList =
288+
Arrays.asList(
289+
TSDataType.STRING,
290+
TSDataType.STRING,
291+
TSDataType.STRING,
292+
TSDataType.FLOAT,
293+
TSDataType.OBJECT);
294+
List<ColumnCategory> columnTypeList =
295+
new ArrayList<>(
296+
Arrays.asList(
297+
ColumnCategory.TAG,
298+
ColumnCategory.TAG,
299+
ColumnCategory.TAG,
300+
ColumnCategory.FIELD,
301+
ColumnCategory.FIELD));
302+
Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1);
303+
for (int i = 0; i < objectSegments.size() - 1; i++) {
304+
int rowIndex = tablet.getRowSize();
305+
tablet.addTimestamp(rowIndex, 1);
306+
tablet.addValue(rowIndex, 0, "1");
307+
tablet.addValue(rowIndex, 1, "5");
308+
tablet.addValue(rowIndex, 2, "3");
309+
tablet.addValue(rowIndex, 3, 37.6F);
310+
tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i));
311+
session.insert(tablet);
312+
tablet.reset();
313+
}
314+
session.executeNonQueryStatement("DELETE FROM object_table where time = 1");
315+
316+
try (SessionDataSet dataSet =
317+
session.executeQueryStatement("select file from object_table where time = 1")) {
318+
SessionDataSet.DataIterator iterator = dataSet.iterator();
319+
while (iterator.next()) {
320+
assertNull(iterator.getString(1));
321+
}
322+
}
323+
}
324+
325+
// test object file path
326+
boolean success = false;
327+
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
328+
String objectDirStr = dataNodeWrapper.getDataNodeObjectDir();
329+
File objectDir = new File(objectDirStr);
330+
if (objectDir.exists() && objectDir.isDirectory()) {
331+
File[] regionDirs = objectDir.listFiles();
332+
if (regionDirs != null) {
333+
for (File regionDir : regionDirs) {
334+
if (regionDir.isDirectory()) {
335+
File objectTmpFile =
336+
new File(
337+
regionDir,
338+
convertPathString("object_table")
339+
+ File.separator
340+
+ convertPathString("1")
341+
+ File.separator
342+
+ convertPathString("5")
343+
+ File.separator
344+
+ convertPathString("3")
345+
+ File.separator
346+
+ convertPathString("file")
347+
+ File.separator
348+
+ "1.bin.tmp");
349+
if (objectTmpFile.exists() && objectTmpFile.isFile()) {
350+
success = true;
351+
}
352+
}
353+
}
354+
}
355+
}
356+
}
357+
Assert.assertFalse(success);
358+
}
359+
360+
protected String convertPathString(String path) {
361+
return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8));
362+
}
363+
}

0 commit comments

Comments
 (0)