Skip to content

Commit 524cc4c

Browse files
zerolbsonyzerolbsony
committed
Occur incompatible exception when merging statistics, need to rewrite statistics in current chunk metadata so that resolve "Statistics classes mismatched: class org.apache.tsfile.file.metadata.statistics.BinaryStatistics vs. class org.apache.tsfile.file.metadata.statistics.IntegerStatistics" exception.
1 parent e570fda commit 524cc4c

File tree

2 files changed

+119
-102
lines changed

2 files changed

+119
-102
lines changed

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,14 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata(
542542
boolean isModified = false;
543543
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
544544
isModified = (isModified || chunkMetadata.isModified());
545+
TSDataType targetDataType = fullPath.getMeasurementSchema().getType();
546+
if (targetDataType.equals(TSDataType.STRING)
547+
&& (chunkMetadata.getDataType() != targetDataType)) {
548+
// create new statistics object via new data type, and merge statistics information
549+
SchemaUtils.rewriteNonAlignedChunkMetadataStatistics(
550+
(ChunkMetadata) chunkMetadata, targetDataType);
551+
chunkMetadata.setModified(true);
552+
}
545553
if (!useFakeStatistics) {
546554
seriesStatistics.mergeStatistics(chunkMetadata.getStatistics());
547555
continue;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java‎

Lines changed: 111 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -404,108 +404,7 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics
404404
List<IChunkMetadata> newValueChunkMetadataList = new ArrayList<>();
405405
for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) {
406406
Statistics<?> statistics = Statistics.getStatsByType(targetDataType);
407-
switch (valueChunkMetadata.getDataType()) {
408-
case INT32:
409-
case DATE:
410-
case INT64:
411-
case TIMESTAMP:
412-
case FLOAT:
413-
case DOUBLE:
414-
case BOOLEAN:
415-
if (targetDataType == TSDataType.STRING) {
416-
Binary[] binaryValues = new Binary[4];
417-
binaryValues[0] =
418-
new Binary(
419-
valueChunkMetadata.getStatistics().getFirstValue().toString(),
420-
StandardCharsets.UTF_8);
421-
binaryValues[1] =
422-
new Binary(
423-
valueChunkMetadata.getStatistics().getLastValue().toString(),
424-
StandardCharsets.UTF_8);
425-
if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) {
426-
binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8);
427-
binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8);
428-
} else {
429-
binaryValues[2] =
430-
new Binary(
431-
valueChunkMetadata.getStatistics().getMinValue().toString(),
432-
StandardCharsets.UTF_8);
433-
binaryValues[3] =
434-
new Binary(
435-
valueChunkMetadata.getStatistics().getMaxValue().toString(),
436-
StandardCharsets.UTF_8);
437-
}
438-
long[] longValues = new long[4];
439-
longValues[0] = valueChunkMetadata.getStatistics().getStartTime();
440-
longValues[1] = valueChunkMetadata.getStatistics().getEndTime();
441-
longValues[2] = longValues[1];
442-
longValues[3] = longValues[1];
443-
statistics.update(longValues, binaryValues, binaryValues.length);
444-
} else if (targetDataType == TSDataType.TEXT) {
445-
Binary[] binaryValues = new Binary[2];
446-
if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) {
447-
binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8);
448-
binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8);
449-
} else {
450-
binaryValues[0] =
451-
new Binary(
452-
valueChunkMetadata.getStatistics().getMinValue().toString(),
453-
StandardCharsets.UTF_8);
454-
binaryValues[1] =
455-
new Binary(
456-
valueChunkMetadata.getStatistics().getMaxValue().toString(),
457-
StandardCharsets.UTF_8);
458-
}
459-
long[] longValues = new long[2];
460-
longValues[0] = valueChunkMetadata.getStatistics().getStartTime();
461-
longValues[1] = valueChunkMetadata.getStatistics().getEndTime();
462-
statistics.update(longValues, binaryValues, binaryValues.length);
463-
} else {
464-
statistics = valueChunkMetadata.getStatistics();
465-
}
466-
break;
467-
case STRING:
468-
if (targetDataType == TSDataType.TEXT) {
469-
Binary[] binaryValues = new Binary[2];
470-
binaryValues[0] =
471-
new Binary(
472-
Arrays.asList(TSDataType.TEXT, TSDataType.BLOB)
473-
.contains(valueChunkMetadata.getDataType())
474-
? ""
475-
: valueChunkMetadata.getStatistics().getMinValue().toString(),
476-
StandardCharsets.UTF_8);
477-
binaryValues[1] =
478-
new Binary(
479-
Arrays.asList(TSDataType.TEXT, TSDataType.BLOB)
480-
.contains(valueChunkMetadata.getDataType())
481-
? ""
482-
: valueChunkMetadata.getStatistics().getMaxValue().toString(),
483-
StandardCharsets.UTF_8);
484-
long[] longValues = new long[2];
485-
longValues[0] = valueChunkMetadata.getStatistics().getStartTime();
486-
longValues[1] = valueChunkMetadata.getStatistics().getEndTime();
487-
statistics.update(longValues, binaryValues, binaryValues.length);
488-
} else {
489-
statistics = valueChunkMetadata.getStatistics();
490-
}
491-
break;
492-
case TEXT:
493-
case BLOB:
494-
if (targetDataType == TSDataType.STRING) {
495-
Binary[] binaryValues = new Binary[2];
496-
binaryValues[0] = new Binary("", StandardCharsets.UTF_8);
497-
binaryValues[1] = new Binary("", StandardCharsets.UTF_8);
498-
long[] longValues = new long[2];
499-
longValues[0] = valueChunkMetadata.getStatistics().getStartTime();
500-
longValues[1] = valueChunkMetadata.getStatistics().getEndTime();
501-
statistics.update(longValues, binaryValues, binaryValues.length);
502-
} else {
503-
statistics = valueChunkMetadata.getStatistics();
504-
}
505-
break;
506-
default:
507-
break;
508-
}
407+
statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics);
509408

510409
ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata;
511410
newChunkMetadata.setTsDataType(targetDataType);
@@ -516,10 +415,120 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics
516415
alignedChunkMetadata.getTimeChunkMetadata(), newValueChunkMetadataList);
517416
}
518417

418+
public static void rewriteNonAlignedChunkMetadataStatistics(
419+
ChunkMetadata chunkMetadata, TSDataType targetDataType) {
420+
Statistics<?> statistics = Statistics.getStatsByType(targetDataType);
421+
statistics = getNewStatistics(chunkMetadata, targetDataType, statistics);
422+
423+
chunkMetadata.setTsDataType(targetDataType);
424+
chunkMetadata.setStatistics(statistics);
425+
}
426+
519427
public static TSEncoding getDataTypeCompatibleEncoding(TSDataType dataType, TSEncoding encoding) {
520428
if (!encoding.isSupported(dataType)) {
521429
return EncodingInferenceUtils.getDefaultEncoding(dataType);
522430
}
523431
return encoding;
524432
}
433+
434+
public static Statistics<?> getNewStatistics(
435+
IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics<?> statistics) {
436+
switch (chunkMetadata.getDataType()) {
437+
case INT32:
438+
case DATE:
439+
case INT64:
440+
case TIMESTAMP:
441+
case FLOAT:
442+
case DOUBLE:
443+
case BOOLEAN:
444+
if (targetDataType == TSDataType.STRING) {
445+
Binary[] binaryValues = new Binary[4];
446+
binaryValues[0] =
447+
new Binary(
448+
chunkMetadata.getStatistics().getFirstValue().toString(), StandardCharsets.UTF_8);
449+
binaryValues[1] =
450+
new Binary(
451+
chunkMetadata.getStatistics().getLastValue().toString(), StandardCharsets.UTF_8);
452+
if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) {
453+
binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8);
454+
binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8);
455+
} else {
456+
binaryValues[2] =
457+
new Binary(
458+
chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8);
459+
binaryValues[3] =
460+
new Binary(
461+
chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8);
462+
}
463+
long[] longValues = new long[4];
464+
longValues[0] = chunkMetadata.getStatistics().getStartTime();
465+
longValues[1] = chunkMetadata.getStatistics().getEndTime();
466+
longValues[2] = longValues[1];
467+
longValues[3] = longValues[1];
468+
statistics.update(longValues, binaryValues, binaryValues.length);
469+
} else if (targetDataType == TSDataType.TEXT) {
470+
Binary[] binaryValues = new Binary[2];
471+
if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) {
472+
binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8);
473+
binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8);
474+
} else {
475+
binaryValues[0] =
476+
new Binary(
477+
chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8);
478+
binaryValues[1] =
479+
new Binary(
480+
chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8);
481+
}
482+
long[] longValues = new long[2];
483+
longValues[0] = chunkMetadata.getStatistics().getStartTime();
484+
longValues[1] = chunkMetadata.getStatistics().getEndTime();
485+
statistics.update(longValues, binaryValues, binaryValues.length);
486+
} else {
487+
statistics = chunkMetadata.getStatistics();
488+
}
489+
break;
490+
case STRING:
491+
if (targetDataType == TSDataType.TEXT) {
492+
Binary[] binaryValues = new Binary[2];
493+
binaryValues[0] =
494+
new Binary(
495+
Arrays.asList(TSDataType.TEXT, TSDataType.BLOB)
496+
.contains(chunkMetadata.getDataType())
497+
? ""
498+
: chunkMetadata.getStatistics().getMinValue().toString(),
499+
StandardCharsets.UTF_8);
500+
binaryValues[1] =
501+
new Binary(
502+
Arrays.asList(TSDataType.TEXT, TSDataType.BLOB)
503+
.contains(chunkMetadata.getDataType())
504+
? ""
505+
: chunkMetadata.getStatistics().getMaxValue().toString(),
506+
StandardCharsets.UTF_8);
507+
long[] longValues = new long[2];
508+
longValues[0] = chunkMetadata.getStatistics().getStartTime();
509+
longValues[1] = chunkMetadata.getStatistics().getEndTime();
510+
statistics.update(longValues, binaryValues, binaryValues.length);
511+
} else {
512+
statistics = chunkMetadata.getStatistics();
513+
}
514+
break;
515+
case TEXT:
516+
case BLOB:
517+
if (targetDataType == TSDataType.STRING) {
518+
Binary[] binaryValues = new Binary[2];
519+
binaryValues[0] = new Binary("", StandardCharsets.UTF_8);
520+
binaryValues[1] = new Binary("", StandardCharsets.UTF_8);
521+
long[] longValues = new long[2];
522+
longValues[0] = chunkMetadata.getStatistics().getStartTime();
523+
longValues[1] = chunkMetadata.getStatistics().getEndTime();
524+
statistics.update(longValues, binaryValues, binaryValues.length);
525+
} else {
526+
statistics = chunkMetadata.getStatistics();
527+
}
528+
break;
529+
default:
530+
break;
531+
}
532+
return statistics;
533+
}
525534
}

0 commit comments

Comments
 (0)