Skip to content

Commit a7b15f5

Browse files
HTHouHTHou
committed
add base32 Object Path
1 parent 7249a52 commit a7b15f5

File tree

9 files changed

+253
-76
lines changed

9 files changed

+253
-76
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,9 @@
3333
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
3434
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType;
3535
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
36-
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils;
3736
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
37+
import org.apache.iotdb.db.storageengine.dataregion.IObjectPath;
38+
import org.apache.iotdb.db.storageengine.dataregion.IObjectPath.Deserializer;
3839

3940
import org.apache.tsfile.utils.PublicBAOS;
4041
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -60,15 +61,15 @@ public class ObjectNode extends SearchNode implements WALEntryValue {
6061

6162
private byte[] content;
6263

63-
private String filePath;
64+
private IObjectPath filePath;
6465

6566
private final int contentLength;
6667

6768
private TRegionReplicaSet dataRegionReplicaSet;
6869

6970
private boolean isGeneratedByRemoteConsensusLeader;
7071

71-
public ObjectNode(boolean isEOF, long offset, byte[] content, String filePath) {
72+
public ObjectNode(boolean isEOF, long offset, byte[] content, IObjectPath filePath) {
7273
super(new PlanNodeId(""));
7374
this.isEOF = isEOF;
7475
this.offset = offset;
@@ -77,7 +78,7 @@ public ObjectNode(boolean isEOF, long offset, byte[] content, String filePath) {
7778
this.contentLength = content.length;
7879
}
7980

80-
public ObjectNode(boolean isEOF, long offset, int contentLength, String filePath) {
81+
public ObjectNode(boolean isEOF, long offset, int contentLength, IObjectPath filePath) {
8182
super(new PlanNodeId(""));
8283
this.isEOF = isEOF;
8384
this.offset = offset;
@@ -97,12 +98,12 @@ public long getOffset() {
9798
return offset;
9899
}
99100

100-
public void setFilePath(String filePath) {
101+
public void setFilePath(IObjectPath filePath) {
101102
this.filePath = filePath;
102103
}
103104

104-
public String getFilePath() {
105-
return filePath;
105+
public String getFilePathString() {
106+
return filePath.toString();
106107
}
107108

108109
@Override
@@ -111,7 +112,11 @@ public void serializeToWAL(IWALByteBufferView buffer) {
111112
buffer.putLong(searchIndex);
112113
buffer.put((byte) (isEOF ? 1 : 0));
113114
buffer.putLong(offset);
114-
WALWriteUtils.write(filePath, buffer);
115+
try {
116+
filePath.serialize(buffer);
117+
} catch (IOException e) {
118+
throw new RuntimeException(e);
119+
}
115120
buffer.putInt(content.length);
116121
}
117122

@@ -122,14 +127,14 @@ public int serializedSize() {
122127
+ Byte.BYTES
123128
+ Long.BYTES
124129
+ Integer.BYTES
125-
+ ReadWriteIOUtils.sizeToWrite(filePath);
130+
+ filePath.getSerializedSize();
126131
}
127132

128133
public static ObjectNode deserializeFromWAL(DataInputStream stream) throws IOException {
129134
long searchIndex = stream.readLong();
130135
boolean isEOF = stream.readByte() == 1;
131136
long offset = stream.readLong();
132-
String filePath = ReadWriteIOUtils.readString(stream);
137+
IObjectPath filePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(stream);
133138
int contentLength = stream.readInt();
134139
ObjectNode objectNode = new ObjectNode(isEOF, offset, contentLength, filePath);
135140
objectNode.setSearchIndex(searchIndex);
@@ -140,8 +145,9 @@ public static ObjectNode deserializeFromWAL(ByteBuffer buffer) {
140145
long searchIndex = buffer.getLong();
141146
boolean isEOF = buffer.get() == 1;
142147
long offset = buffer.getLong();
143-
String filePath = ReadWriteIOUtils.readString(buffer);
144-
Optional<File> objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath);
148+
IObjectPath filePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(buffer);
149+
Optional<File> objectFile =
150+
TierManager.getInstance().getAbsoluteObjectFilePath(filePath.toString());
145151
int contentLength = buffer.getInt();
146152
byte[] contents = new byte[contentLength];
147153
if (objectFile.isPresent()) {
@@ -152,7 +158,7 @@ public static ObjectNode deserializeFromWAL(ByteBuffer buffer) {
152158
throw new RuntimeException(e);
153159
}
154160
} else {
155-
throw new ObjectFileNotExist(filePath);
161+
throw new ObjectFileNotExist(filePath.toString());
156162
}
157163

158164
ObjectNode objectNode = new ObjectNode(isEOF, offset, contents, filePath);
@@ -163,7 +169,7 @@ public static ObjectNode deserializeFromWAL(ByteBuffer buffer) {
163169
public static ObjectNode deserialize(ByteBuffer byteBuffer) {
164170
boolean isEoF = ReadWriteIOUtils.readBool(byteBuffer);
165171
long offset = ReadWriteIOUtils.readLong(byteBuffer);
166-
String filePath = ReadWriteIOUtils.readString(byteBuffer);
172+
IObjectPath filePath = Deserializer.DEFAULT_DESERIALIZER.deserializeFrom(byteBuffer);
167173
int contentLength = ReadWriteIOUtils.readInt(byteBuffer);
168174
byte[] content = ReadWriteIOUtils.readBytes(byteBuffer, contentLength);
169175
return new ObjectNode(isEoF, offset, content, filePath);
@@ -227,7 +233,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) {
227233
getType().serialize(byteBuffer);
228234
ReadWriteIOUtils.write(isEOF, byteBuffer);
229235
ReadWriteIOUtils.write(offset, byteBuffer);
230-
ReadWriteIOUtils.write(filePath, byteBuffer);
236+
filePath.serialize(byteBuffer);
231237
ReadWriteIOUtils.write(contentLength, byteBuffer);
232238
byteBuffer.put(content);
233239
}
@@ -237,7 +243,7 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException {
237243
getType().serialize(stream);
238244
ReadWriteIOUtils.write(isEOF, stream);
239245
ReadWriteIOUtils.write(offset, stream);
240-
ReadWriteIOUtils.write(filePath, stream);
246+
filePath.serialize(stream);
241247
ReadWriteIOUtils.write(contentLength, stream);
242248
stream.write(content);
243249
}
@@ -251,7 +257,8 @@ public ByteBuffer serialize() {
251257
byte[] contents = new byte[contentLength];
252258
boolean readSuccess = false;
253259
for (int i = 0; i < 2; i++) {
254-
Optional<File> objectFile = TierManager.getInstance().getAbsoluteObjectFilePath(filePath);
260+
Optional<File> objectFile =
261+
TierManager.getInstance().getAbsoluteObjectFilePath(filePath.toString());
255262
if (objectFile.isPresent()) {
256263
try {
257264
readContentFromFile(objectFile.get(), contents);
@@ -279,7 +286,7 @@ public ByteBuffer serialize() {
279286
}
280287
ReadWriteIOUtils.write(readSuccess && isEOF, stream);
281288
ReadWriteIOUtils.write(offset, stream);
282-
ReadWriteIOUtils.write(filePath, stream);
289+
filePath.serialize(stream);
283290
ReadWriteIOUtils.write(contentLength, stream);
284291
stream.write(contents);
285292
return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,17 @@
2727
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
2828
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
2929
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
30-
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
30+
import org.apache.iotdb.db.storageengine.dataregion.IObjectPath;
3131

3232
import org.apache.tsfile.enums.TSDataType;
3333
import org.apache.tsfile.file.metadata.IDeviceID;
3434
import org.apache.tsfile.file.metadata.IDeviceID.Factory;
3535
import org.apache.tsfile.utils.Binary;
36-
import org.apache.tsfile.utils.BytesUtils;
3736
import org.apache.tsfile.utils.ReadWriteIOUtils;
3837

3938
import java.io.DataInputStream;
4039
import java.io.IOException;
4140
import java.nio.ByteBuffer;
42-
import java.nio.charset.StandardCharsets;
4341
import java.util.ArrayList;
4442
import java.util.HashMap;
4543
import java.util.List;
@@ -212,22 +210,25 @@ private void handleObjectValue(
212210
boolean isEoF = buffer.get() == 1;
213211
long offset = buffer.getLong();
214212
byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining());
215-
String relativePath =
216-
TsFileNameGenerator.generateObjectFilePath(
213+
IObjectPath relativePath =
214+
IObjectPath.Factory.DEFAULT_FACTORY.create(
217215
dataRegionReplicaSet.getRegionId().getId(),
218216
insertRowNode.getTime(),
219217
insertRowNode.getDeviceID(),
220218
insertRowNode.getMeasurements()[j]);
221219
ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath);
222220
objectNode.setDataRegionReplicaSet(dataRegionReplicaSet);
223-
byte[] filePathBytes = relativePath.getBytes(StandardCharsets.UTF_8);
224-
byte[] valueBytes = new byte[filePathBytes.length + Long.BYTES];
225-
System.arraycopy(
226-
BytesUtils.longToBytes(offset + content.length), 0, valueBytes, 0, Long.BYTES);
227-
System.arraycopy(filePathBytes, 0, valueBytes, Long.BYTES, filePathBytes.length);
228-
((Binary) values[j]).setValues(valueBytes);
229-
insertRowNode.setValues(values);
230221
writePlanNodeList.add(objectNode);
222+
if (isEoF) {
223+
ByteBuffer valueBytes =
224+
ByteBuffer.allocate(relativePath.getSerializedSize() + Long.BYTES);
225+
valueBytes.putLong(offset + content.length);
226+
relativePath.serialize(valueBytes);
227+
((Binary) values[j]).setValues(valueBytes.array());
228+
insertRowNode.setValues(values);
229+
} else {
230+
values[j] = null;
231+
}
231232
}
232233
}
233234
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,15 @@
3232
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
3333
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
3434
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache;
35-
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
3635
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
36+
import org.apache.iotdb.db.storageengine.dataregion.IObjectPath;
3737

3838
import org.apache.tsfile.enums.TSDataType;
3939
import org.apache.tsfile.file.metadata.IDeviceID;
4040
import org.apache.tsfile.file.metadata.IDeviceID.Factory;
4141
import org.apache.tsfile.read.TimeValuePair;
4242
import org.apache.tsfile.utils.Binary;
4343
import org.apache.tsfile.utils.BitMap;
44-
import org.apache.tsfile.utils.BytesUtils;
4544
import org.apache.tsfile.utils.Pair;
4645
import org.apache.tsfile.utils.ReadWriteIOUtils;
4746
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -50,7 +49,6 @@
5049
import java.io.DataOutputStream;
5150
import java.io.IOException;
5251
import java.nio.ByteBuffer;
53-
import java.nio.charset.StandardCharsets;
5452
import java.util.ArrayList;
5553
import java.util.HashMap;
5654
import java.util.List;
@@ -470,19 +468,17 @@ private void handleObjectValue(
470468
boolean isEoF = buffer.get() == 1;
471469
long offset = buffer.getLong();
472470
byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining());
473-
String relativePath =
474-
TsFileNameGenerator.generateObjectFilePath(
471+
IObjectPath relativePath =
472+
IObjectPath.Factory.DEFAULT_FACTORY.create(
475473
entry.getKey().getRegionId().getId(), times[j], getDeviceID(j), measurements[column]);
476474
ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath);
477475
objectNode.setDataRegionReplicaSet(entry.getKey());
478476
result.add(objectNode);
479477
if (isEoF) {
480-
byte[] filePathBytes = relativePath.getBytes(StandardCharsets.UTF_8);
481-
byte[] valueBytes = new byte[filePathBytes.length + Long.BYTES];
482-
System.arraycopy(
483-
BytesUtils.longToBytes(offset + content.length), 0, valueBytes, 0, Long.BYTES);
484-
System.arraycopy(filePathBytes, 0, valueBytes, Long.BYTES, filePathBytes.length);
485-
((Binary[]) columns[column])[j] = new Binary(valueBytes);
478+
ByteBuffer valueBytes = ByteBuffer.allocate(relativePath.getSerializedSize() + Long.BYTES);
479+
valueBytes.putLong(offset + content.length);
480+
relativePath.serialize(valueBytes);
481+
((Binary[]) columns[column])[j] = new Binary(valueBytes.array());
486482
} else {
487483
((Binary[]) columns[column])[j] = null;
488484
if (bitMaps == null) {

0 commit comments

Comments
 (0)