Skip to content

Commit ebc7d7f

Browse files
shuwenweishuwenwei
authored
Further optimize the efficiency of memtable region scan (#16976)
1 parent c531044 commit ebc7d7f

File tree

4 files changed

+183
-60
lines changed

4 files changed

+183
-60
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java‎

Lines changed: 61 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import java.util.concurrent.BlockingQueue;
5959
import java.util.concurrent.atomic.AtomicInteger;
6060

61+
import static org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager.ARRAY_SIZE;
6162
import static org.apache.iotdb.db.utils.ModificationUtils.isPointDeleted;
6263

6364
public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
@@ -365,62 +366,74 @@ private void getAnySatisfiedTimestamp(
365366
// timestamp: 1 bitmap: 011
366367
// timestamp: 2 bitmap: 101
367368
// timestamp: 4 bitmap: 110
368-
for (int row = 0; row < rowCount; row++) {
369-
// the row is deleted
370-
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
371-
continue;
372-
}
373-
long timestamp = alignedTVList.getTime(row);
374-
if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp, null)) {
375-
continue;
376-
}
377-
378-
// Note that this method will only perform bitmap unmarking on the first occurrence of a
379-
// non-null value in multiple timestamps for the same column.
380-
BitMap currentRowNullValueBitmap = null;
381-
382-
for (int column = 0; column < schemaList.size(); column++) {
383-
if (alignedTVList.isNullValue(alignedTVList.getValueIndex(row), column)) {
369+
List<long[]> timestampsList = alignedTVList.getTimestamps();
370+
List<int[]> indicesList = alignedTVList.getIndices();
371+
int row = -1;
372+
for (int i = 0; i < timestampsList.size(); i++) {
373+
long[] timestamps = timestampsList.get(i);
374+
int[] indices = indicesList == null ? null : indicesList.get(i);
375+
int limit = (i == timestampsList.size() - 1) ? rowCount - i * ARRAY_SIZE : ARRAY_SIZE;
376+
for (int j = 0; j < limit; j++) {
377+
row++;
378+
// the row is deleted
379+
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
384380
continue;
385381
}
386-
387-
// skip deleted row
388-
if (valueColumnsDeletionList != null
389-
&& !valueColumnsDeletionList.isEmpty()
390-
&& isPointDeleted(
391-
timestamp,
392-
valueColumnsDeletionList.get(column),
393-
valueColumnDeleteCursor.get(column))) {
382+
long timestamp = timestamps[j];
383+
if (globalTimeFilter != null && !globalTimeFilter.satisfy(timestamp, null)) {
394384
continue;
395385
}
396-
if (!columnHasNonNullValue.isMarked(column)) {
397-
hasNonNullValueColumnCount.incrementAndGet();
398-
columnHasNonNullValue.mark(column);
399-
currentRowNullValueBitmap =
386+
387+
// Note that this method will only perform bitmap unmarking on the first occurrence of a
388+
// non-null value in multiple timestamps for the same column.
389+
BitMap currentRowNullValueBitmap = null;
390+
for (int column = 0; column < schemaList.size(); column++) {
391+
if (alignedTVList.isNullValue(indices == null ? row : indices[j], column)) {
392+
continue;
393+
}
394+
395+
// skip deleted row
396+
if (valueColumnsDeletionList != null && !valueColumnsDeletionList.isEmpty()) {
397+
List<TimeRange> columnDeletionList = valueColumnsDeletionList.get(column);
398+
int[] deleteCursor = valueColumnDeleteCursor.get(column);
399+
if (columnDeletionList != null && !columnDeletionList.isEmpty()) {
400+
if (!alignedTVList.isSorted()) {
401+
deleteCursor[0] = 0;
402+
}
403+
if (isPointDeleted(timestamp, columnDeletionList, deleteCursor)) {
404+
continue;
405+
}
406+
}
407+
}
408+
if (!columnHasNonNullValue.isMarked(column)) {
409+
hasNonNullValueColumnCount.incrementAndGet();
410+
columnHasNonNullValue.mark(column);
411+
currentRowNullValueBitmap =
412+
currentRowNullValueBitmap != null
413+
? currentRowNullValueBitmap
414+
: timestampWithBitmap.computeIfAbsent(
415+
timestamp, k -> getAllMarkedBitmap(schemaList.size()));
416+
currentRowNullValueBitmap.unmark(column);
417+
}
418+
}
419+
420+
if (!ignoreAllNullRows) {
421+
timestampWithBitmap.put(
422+
timestamp,
400423
currentRowNullValueBitmap != null
401424
? currentRowNullValueBitmap
402-
: timestampWithBitmap.computeIfAbsent(
403-
timestamp, k -> getAllMarkedBitmap(schemaList.size()));
404-
currentRowNullValueBitmap.unmark(column);
425+
: getAllMarkedBitmap(schemaList.size()));
426+
return;
405427
}
406-
}
407-
408-
if (!ignoreAllNullRows) {
409-
timestampWithBitmap.put(
410-
timestamp,
411-
currentRowNullValueBitmap != null
412-
? currentRowNullValueBitmap
413-
: getAllMarkedBitmap(schemaList.size()));
414-
return;
415-
}
416-
if (currentRowNullValueBitmap == null) {
417-
continue;
418-
}
419-
// found new column with non-null value
420-
timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
428+
if (currentRowNullValueBitmap == null) {
429+
continue;
430+
}
431+
// found new column with non-null value
432+
timestampWithBitmap.put(timestamp, currentRowNullValueBitmap);
421433

422-
if (hasNonNullValueColumnCount.get() == schemaList.size()) {
423-
return;
434+
if (hasNonNullValueColumnCount.get() == schemaList.size()) {
435+
return;
436+
}
424437
}
425438
}
426439
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java‎

Lines changed: 32 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.iotdb.db.conf.IoTDBConfig;
2525
import org.apache.iotdb.db.conf.IoTDBDescriptor;
2626
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView;
27+
import org.apache.iotdb.db.storageengine.rescon.memory.PrimitiveArrayManager;
2728
import org.apache.iotdb.db.utils.ModificationUtils;
2829
import org.apache.iotdb.db.utils.datastructure.BatchEncodeInfo;
2930
import org.apache.iotdb.db.utils.datastructure.MemPointIterator;
@@ -599,19 +600,38 @@ private Optional<Long> getAnySatisfiedTimestamp(
599600
&& !globalTimeFilter.satisfyStartEndTime(tvlist.getMinTime(), tvlist.getMaxTime())) {
600601
return Optional.empty();
601602
}
602-
for (int i = 0; i < rowCount; i++) {
603-
if (tvlist.getBitMap() != null && tvlist.isNullValue(tvlist.getValueIndex(i))) {
604-
continue;
605-
}
606-
long curTime = tvlist.getTime(i);
607-
if (deletionList != null
608-
&& ModificationUtils.isPointDeleted(curTime, deletionList, deletionCursor)) {
609-
continue;
610-
}
611-
if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime, null)) {
612-
continue;
603+
604+
List<long[]> timestampsList = tvlist.getTimestamps();
605+
List<BitMap> bitMaps = tvlist.getBitMap();
606+
List<int[]> indicesList = tvlist.getIndices();
607+
for (int i = 0; i < timestampsList.size(); i++) {
608+
long[] timestamps = timestampsList.get(i);
609+
BitMap bitMap = bitMaps == null ? null : bitMaps.get(i);
610+
int[] indices = indicesList == null ? null : indicesList.get(i);
611+
int limit =
612+
(i == timestampsList.size() - 1)
613+
? rowCount - i * PrimitiveArrayManager.ARRAY_SIZE
614+
: PrimitiveArrayManager.ARRAY_SIZE;
615+
for (int j = 0; j < limit; j++) {
616+
if (bitMap != null
617+
&& (indices == null ? bitMap.isMarked(j) : tvlist.isNullValue(indices[j]))) {
618+
continue;
619+
}
620+
long curTime = timestamps[j];
621+
if (deletionList != null && !deletionList.isEmpty()) {
622+
if (!tvlist.isSorted()) {
623+
deletionCursor[0] = 0;
624+
}
625+
if (ModificationUtils.isPointDeleted(curTime, deletionList, deletionCursor)) {
626+
continue;
627+
}
628+
}
629+
if (globalTimeFilter != null && !globalTimeFilter.satisfy(curTime, null)) {
630+
continue;
631+
}
632+
633+
return Optional.of(curTime);
613634
}
614-
return Optional.of(curTime);
615635
}
616636
return Optional.empty();
617637
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -745,6 +745,10 @@ public List<long[]> getTimestamps() {
745745
return timestamps;
746746
}
747747

748+
public List<int[]> getIndices() {
749+
return indices;
750+
}
751+
748752
public void setOwnerQuery(QueryContext queryCtx) {
749753
this.ownerQuery = queryCtx;
750754
}

‎iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919

2020
package org.apache.iotdb.db.storageengine.dataregion.memtable;
2121

22+
import org.apache.iotdb.commons.exception.IllegalPathException;
2223
import org.apache.iotdb.commons.path.AlignedFullPath;
24+
import org.apache.iotdb.commons.path.MeasurementPath;
2325
import org.apache.iotdb.commons.path.NonAlignedFullPath;
2426
import org.apache.iotdb.db.conf.IoTDBDescriptor;
27+
import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
2528
import org.apache.iotdb.db.storageengine.dataregion.read.filescan.IChunkHandle;
2629

2730
import org.apache.tsfile.enums.TSDataType;
@@ -238,6 +241,53 @@ public void testAlignedWritableMemChunkRegionScan() {
238241
}
239242
}
240243

244+
@Test
245+
public void testAlignedWritableMemChunkRegionScan2() throws IllegalPathException {
246+
PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
247+
try {
248+
List<IMeasurementSchema> measurementSchemas =
249+
Arrays.asList(
250+
new MeasurementSchema("s1", TSDataType.INT32),
251+
new MeasurementSchema("s2", TSDataType.INT32),
252+
new MeasurementSchema("s3", TSDataType.INT32));
253+
AlignedWritableMemChunk writableMemChunk = null;
254+
for (int i = 1000; i < 2000; i++) {
255+
memTable.writeAlignedRow(
256+
new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i});
257+
}
258+
for (int i = 1; i < 100; i++) {
259+
memTable.writeAlignedRow(
260+
new StringArrayDeviceID("root.test.d1"),
261+
measurementSchemas,
262+
i,
263+
new Object[] {i, null, i});
264+
}
265+
266+
memTable.delete(
267+
new TreeDeletionEntry(
268+
new MeasurementPath(new StringArrayDeviceID("root.test.d1"), "s1"),
269+
new TimeRange(1, 1500)));
270+
writableMemChunk =
271+
(AlignedWritableMemChunk)
272+
memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), "");
273+
writableMemChunk.sortTvListForFlush();
274+
List<BitMap> bitMaps = new ArrayList<>();
275+
long[] timestamps =
276+
writableMemChunk.getAnySatisfiedTimestamp(
277+
Arrays.asList(
278+
Collections.emptyList(), Collections.emptyList(), Collections.emptyList()),
279+
bitMaps,
280+
true,
281+
null);
282+
Assert.assertEquals(3, timestamps.length);
283+
Assert.assertEquals(1, timestamps[0]);
284+
Assert.assertEquals(1000, timestamps[1]);
285+
Assert.assertEquals(1501, timestamps[2]);
286+
} finally {
287+
memTable.release();
288+
}
289+
}
290+
241291
@Test
242292
public void testTableWritableMemChunkRegionScan() {
243293
List<IMeasurementSchema> measurementSchemas =
@@ -356,4 +406,40 @@ public void testNonAlignedWritableMemChunkRegionScan() {
356406
memTable.release();
357407
}
358408
}
409+
410+
@Test
411+
public void testNonAlignedWritableMemChunkRegionScan2() throws IllegalPathException {
412+
PrimitiveMemTable memTable = new PrimitiveMemTable("root.test", "0");
413+
try {
414+
MeasurementSchema measurementSchema = new MeasurementSchema("s1", TSDataType.INT32);
415+
for (int i = 1000; i < 2000; i++) {
416+
memTable.write(
417+
new StringArrayDeviceID("root.test.d1"),
418+
Collections.singletonList(measurementSchema),
419+
i,
420+
new Object[] {i});
421+
}
422+
for (int i = 1; i < 100; i++) {
423+
memTable.write(
424+
new StringArrayDeviceID("root.test.d1"),
425+
Collections.singletonList(measurementSchema),
426+
i,
427+
new Object[] {i});
428+
}
429+
memTable.delete(
430+
new TreeDeletionEntry(
431+
new MeasurementPath(new StringArrayDeviceID("root.test.d1"), "s1"),
432+
new TimeRange(1, 1500)));
433+
WritableMemChunk writableMemChunk =
434+
(WritableMemChunk)
435+
memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), "s1");
436+
writableMemChunk.sortTvListForFlush();
437+
Optional<Long> timestamp = writableMemChunk.getAnySatisfiedTimestamp(null, null);
438+
Assert.assertTrue(timestamp.isPresent());
439+
Assert.assertEquals(1501, timestamp.get().longValue());
440+
441+
} finally {
442+
memTable.release();
443+
}
444+
}
359445
}

0 commit comments

Comments
 (0)