Skip to content

Commit aac2d0b

Browse files
cecemeicecemei
authored
Fix bug for NestedCommonFormatColumnPartSerde (#18850)
1 parent 313ba8e commit aac2d0b

File tree

2 files changed

+63
-69
lines changed

2 files changed

+63
-69
lines changed

‎processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -482,8 +482,8 @@ public FormatSpec(
482482
@JsonProperty("longColumnCompression")@Nullable CompressionStrategy longColumnCompression,
483483
@JsonProperty("doubleColumnCompression")@Nullable CompressionStrategy doubleColumnCompression,
484484
@JsonProperty("bitmapEncoding") @Nullable BitmapSerdeFactory bitmapEncoding,
485-
@JsonProperty("longFieldBitmapIndex")@Nullable BitmapIndexType longFieldBitmapIndex,
486-
@JsonProperty("doubleFieldBitmapIndex")@Nullable BitmapIndexType doubleFieldBitmapIndex
485+
@JsonProperty("longFieldBitmapIndexType")@Nullable BitmapIndexType longFieldBitmapIndex,
486+
@JsonProperty("doubleFieldBitmapIndexType")@Nullable BitmapIndexType doubleFieldBitmapIndex
487487
)
488488
{
489489
super(

‎processing/src/test/java/org/apache/druid/segment/nested/NestedCommonFormatColumnFormatSpecTest.java‎

Lines changed: 61 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
3131
import org.apache.druid.segment.data.FrontCodedIndexed;
3232
import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
33+
import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde;
3334
import org.junit.Assert;
3435
import org.junit.jupiter.api.Test;
3536

@@ -39,26 +40,44 @@ public class NestedCommonFormatColumnFormatSpecTest
3940
public void testSerde() throws JsonProcessingException
4041
{
4142
NestedCommonFormatColumnFormatSpec spec =
42-
NestedCommonFormatColumnFormatSpec.builder()
43-
.setObjectFieldsDictionaryEncoding(
44-
new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)
45-
)
46-
.setObjectStorageCompression(CompressionStrategy.ZSTD)
47-
.setStringDictionaryEncoding(
48-
new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1)
49-
)
50-
.setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
51-
.setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
52-
.build();
53-
Assert.assertEquals(
54-
spec,
43+
NestedCommonFormatColumnFormatSpec
44+
.builder()
45+
.setBitmapEncoding(new ConciseBitmapSerdeFactory())
46+
.setObjectFieldsDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1))
47+
.setObjectStorageCompression(CompressionStrategy.ZSTD)
48+
.setStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1))
49+
.setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
50+
.setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
51+
.build();
52+
// NestedCommonFormatColumnFormatSpec does not support serde for BitmapEncoding, value would be ignored
53+
Assert.assertEquals(
54+
NestedCommonFormatColumnFormatSpec.builder(spec).setBitmapEncoding(null).build(),
5555
TestHelper.JSON_MAPPER.readValue(
5656
TestHelper.JSON_MAPPER.writeValueAsString(spec),
5757
NestedCommonFormatColumnFormatSpec.class
5858
)
5959
);
6060
}
6161

62+
@Test
63+
public void testSerdeFormatSpec() throws JsonProcessingException
64+
{
65+
NestedCommonFormatColumnFormatSpec spec =
66+
NestedCommonFormatColumnFormatSpec
67+
.builder()
68+
.setObjectFieldsDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1))
69+
.setObjectStorageCompression(CompressionStrategy.ZSTD)
70+
.setStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1))
71+
.setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
72+
.setDoubleFieldBitmapIndexType(BitmapIndexType.DictionaryEncodedValueIndex.INSTANCE)
73+
.build();
74+
NestedCommonFormatColumnPartSerde.FormatSpec formatSpec = TestHelper.JSON_MAPPER.readValue(
75+
TestHelper.JSON_MAPPER.writeValueAsString(spec),
76+
NestedCommonFormatColumnPartSerde.FormatSpec.class
77+
);
78+
Assert.assertEquals(spec.toString(), formatSpec.toString());
79+
}
80+
6281
@Test
6382
public void testGetEffectiveSpecDefaults()
6483
{
@@ -67,18 +86,9 @@ public void testGetEffectiveSpecDefaults()
6786
IndexSpec.getDefault().getEffectiveSpec()
6887
);
6988

70-
Assert.assertEquals(
71-
StringEncodingStrategy.UTF8_STRATEGY,
72-
defaults.getObjectFieldsDictionaryEncoding()
73-
);
74-
Assert.assertEquals(
75-
ObjectStorageEncoding.SMILE,
76-
defaults.getObjectStorageEncoding()
77-
);
78-
Assert.assertEquals(
79-
CompressionStrategy.LZ4,
80-
defaults.getObjectStorageCompression()
81-
);
89+
Assert.assertEquals(StringEncodingStrategy.UTF8_STRATEGY, defaults.getObjectFieldsDictionaryEncoding());
90+
Assert.assertEquals(ObjectStorageEncoding.SMILE, defaults.getObjectStorageEncoding());
91+
Assert.assertEquals(CompressionStrategy.LZ4, defaults.getObjectStorageCompression());
8292
Assert.assertEquals(
8393
IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(),
8494
defaults.getDictionaryEncodedColumnCompression()
@@ -95,6 +105,8 @@ public void testGetEffectiveSpecDefaults()
95105
IndexSpec.getDefault().getEffectiveSpec().getMetricCompression(),
96106
defaults.getDoubleColumnCompression()
97107
);
108+
Assert.assertEquals(BitmapIndexType.DictionaryEncodedValueIndex.INSTANCE, defaults.getLongFieldBitmapIndexType());
109+
Assert.assertEquals(BitmapIndexType.DictionaryEncodedValueIndex.INSTANCE, defaults.getLongFieldBitmapIndexType());
98110
}
99111

100112
@Test
@@ -108,25 +120,18 @@ public void testEffectiveSpecIndexSpecOverrides()
108120
NestedCommonFormatColumnFormatSpec.builder()
109121
.setObjectFieldsDictionaryEncoding(frontcoded)
110122
.setObjectStorageEncoding(ObjectStorageEncoding.NONE)
123+
.setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
124+
.setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
111125
.build()
112126
)
113127
.withMetricCompression(CompressionStrategy.LZF)
114128
.build()
115129
.getEffectiveSpec()
116130
);
117131

118-
Assert.assertEquals(
119-
frontcoded,
120-
defaults.getObjectFieldsDictionaryEncoding()
121-
);
122-
Assert.assertEquals(
123-
ObjectStorageEncoding.NONE,
124-
defaults.getObjectStorageEncoding()
125-
);
126-
Assert.assertEquals(
127-
CompressionStrategy.LZ4,
128-
defaults.getObjectStorageCompression()
129-
);
132+
Assert.assertEquals(frontcoded, defaults.getObjectFieldsDictionaryEncoding());
133+
Assert.assertEquals(ObjectStorageEncoding.NONE, defaults.getObjectStorageEncoding());
134+
Assert.assertEquals(CompressionStrategy.LZ4, defaults.getObjectStorageCompression());
130135
Assert.assertEquals(
131136
IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(),
132137
defaults.getDictionaryEncodedColumnCompression()
@@ -135,30 +140,26 @@ public void testEffectiveSpecIndexSpecOverrides()
135140
IndexSpec.getDefault().getEffectiveSpec().getStringDictionaryEncoding(),
136141
defaults.getStringDictionaryEncoding()
137142
);
138-
Assert.assertEquals(
139-
CompressionStrategy.LZF,
140-
defaults.getLongColumnCompression()
141-
);
142-
Assert.assertEquals(
143-
CompressionStrategy.LZF,
144-
defaults.getDoubleColumnCompression()
145-
);
143+
Assert.assertEquals(CompressionStrategy.LZF, defaults.getLongColumnCompression());
144+
Assert.assertEquals(CompressionStrategy.LZF, defaults.getDoubleColumnCompression());
145+
Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, defaults.getLongFieldBitmapIndexType());
146+
Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, defaults.getDoubleFieldBitmapIndexType());
146147
}
147148

148149
@Test
149150
public void testGetEffectiveSpecMerge()
150151
{
151152
NestedCommonFormatColumnFormatSpec merged = NestedCommonFormatColumnFormatSpec.getEffectiveFormatSpec(
152-
NestedCommonFormatColumnFormatSpec.builder()
153-
.setObjectFieldsDictionaryEncoding(
154-
new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)
155-
)
156-
.setObjectStorageCompression(CompressionStrategy.ZSTD)
157-
.setStringDictionaryEncoding(
158-
new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1)
159-
)
160-
.setDoubleColumnCompression(CompressionStrategy.ZSTD)
161-
.build(),
153+
NestedCommonFormatColumnFormatSpec
154+
.builder()
155+
.setObjectFieldsDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1))
156+
.setObjectStorageCompression(CompressionStrategy.ZSTD)
157+
.setObjectStorageEncoding(ObjectStorageEncoding.NONE)
158+
.setStringDictionaryEncoding(new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1))
159+
.setDoubleColumnCompression(CompressionStrategy.ZSTD)
160+
.setDoubleFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
161+
.setLongFieldBitmapIndexType(BitmapIndexType.NullValueIndex.INSTANCE)
162+
.build(),
162163
IndexSpec.getDefault().getEffectiveSpec()
163164
);
164165

@@ -170,26 +171,19 @@ public void testGetEffectiveSpecMerge()
170171
new StringEncodingStrategy.FrontCoded(4, FrontCodedIndexed.V1),
171172
merged.getStringDictionaryEncoding()
172173
);
173-
Assert.assertEquals(
174-
ObjectStorageEncoding.SMILE,
175-
merged.getObjectStorageEncoding()
176-
);
174+
Assert.assertEquals(ObjectStorageEncoding.NONE, merged.getObjectStorageEncoding());
177175
Assert.assertEquals(
178176
IndexSpec.getDefault().getEffectiveSpec().getDimensionCompression(),
179177
merged.getDictionaryEncodedColumnCompression()
180178
);
181-
Assert.assertEquals(
182-
CompressionStrategy.ZSTD,
183-
merged.getObjectStorageCompression()
184-
);
179+
Assert.assertEquals(CompressionStrategy.ZSTD, merged.getObjectStorageCompression());
185180
Assert.assertEquals(
186181
IndexSpec.getDefault().getEffectiveSpec().getMetricCompression(),
187182
merged.getLongColumnCompression()
188183
);
189-
Assert.assertEquals(
190-
CompressionStrategy.ZSTD,
191-
merged.getDoubleColumnCompression()
192-
);
184+
Assert.assertEquals(CompressionStrategy.ZSTD, merged.getDoubleColumnCompression());
185+
Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, merged.getLongFieldBitmapIndexType());
186+
Assert.assertEquals(BitmapIndexType.NullValueIndex.INSTANCE, merged.getDoubleFieldBitmapIndexType());
193187
}
194188

195189
@Test

0 commit comments

Comments
 (0)