Skip to content

Commit 71b3b1e

Browse files
clintropolisclintropolis
committed
segment format v10
1 parent b9cf125 commit 71b3b1e

File tree

54 files changed

+4654
-1849
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+4654
-1849
lines changed

‎embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ public EmbeddedDruidCluster createCluster()
102102
.useDefaultTimeoutForLatchableEmitter(20)
103103
.addResource(storageResource)
104104
.addCommonProperty("druid.storage.zip", "false")
105+
.addCommonProperty("druid.indexer.task.buildV10", "true")
105106
.addCommonProperty("druid.monitoring.emissionPeriod", "PT1s")
106107
.addCommonProperty(
107108
"druid.monitoring.monitors",

‎indexing-service/src/main/java/org/apache/druid/indexing/common/TaskToolboxFactory.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
import org.apache.druid.rpc.StandardRetryPolicy;
5252
import org.apache.druid.rpc.indexing.OverlordClient;
5353
import org.apache.druid.segment.IndexIO;
54+
import org.apache.druid.segment.IndexMergerV10Factory;
5455
import org.apache.druid.segment.IndexMergerV9Factory;
5556
import org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory;
5657
import org.apache.druid.segment.incremental.RowIngestionMetersFactory;
@@ -103,6 +104,7 @@ public class TaskToolboxFactory
103104
private final CacheConfig cacheConfig;
104105
private final CachePopulatorStats cachePopulatorStats;
105106
private final IndexMergerV9Factory indexMergerV9Factory;
107+
private final IndexMergerV10Factory indexMergerV10Factory;
106108
private final DruidNodeAnnouncer druidNodeAnnouncer;
107109
private final DruidNode druidNode;
108110
private final LookupNodeService lookupNodeService;
@@ -151,6 +153,7 @@ public TaskToolboxFactory(
151153
CacheConfig cacheConfig,
152154
CachePopulatorStats cachePopulatorStats,
153155
IndexMergerV9Factory indexMergerV9Factory,
156+
IndexMergerV10Factory indexMergerV10Factory,
154157
DruidNodeAnnouncer druidNodeAnnouncer,
155158
@RemoteChatHandler DruidNode druidNode,
156159
LookupNodeService lookupNodeService,
@@ -196,6 +199,7 @@ public TaskToolboxFactory(
196199
this.cacheConfig = cacheConfig;
197200
this.cachePopulatorStats = cachePopulatorStats;
198201
this.indexMergerV9Factory = indexMergerV9Factory;
202+
this.indexMergerV10Factory = indexMergerV10Factory;
199203
this.druidNodeAnnouncer = druidNodeAnnouncer;
200204
this.druidNode = druidNode;
201205
this.lookupNodeService = lookupNodeService;
@@ -256,7 +260,9 @@ public TaskToolbox build(TaskConfig config, Task task)
256260
.cacheConfig(cacheConfig)
257261
.cachePopulatorStats(cachePopulatorStats)
258262
.indexMerger(
259-
indexMergerV9Factory.create(
263+
config.buildV10()
264+
? indexMergerV10Factory.create()
265+
: indexMergerV9Factory.create(
260266
task.getContextValue(Tasks.STORE_EMPTY_COLUMNS_KEY, config.isStoreEmptyColumns())
261267
)
262268
)

‎indexing-service/src/main/java/org/apache/druid/indexing/common/config/TaskConfig.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,9 @@ public class TaskConfig implements TaskDirectory
8080
@JsonProperty
8181
private final boolean allowHadoopTaskExecution;
8282

83+
@JsonProperty
84+
private final boolean buildV10;
85+
8386
@JsonCreator
8487
public TaskConfig(
8588
@JsonProperty("baseDir") String baseDir,
@@ -92,7 +95,8 @@ public TaskConfig(
9295
@JsonProperty("storeEmptyColumns") @Nullable Boolean storeEmptyColumns,
9396
@JsonProperty("encapsulatedTask") boolean enableTaskLevelLogPush,
9497
@JsonProperty("tmpStorageBytesPerTask") @Nullable Long tmpStorageBytesPerTask,
95-
@JsonProperty("allowHadoopTaskExecution") boolean allowHadoopTaskExecution
98+
@JsonProperty("allowHadoopTaskExecution") boolean allowHadoopTaskExecution,
99+
@JsonProperty("buildV10") boolean buildV10
96100
)
97101
{
98102
this.baseDir = Configs.valueOrDefault(baseDir, System.getProperty("java.io.tmpdir"));
@@ -119,6 +123,7 @@ public TaskConfig(
119123
this.storeEmptyColumns = Configs.valueOrDefault(storeEmptyColumns, DEFAULT_STORE_EMPTY_COLUMNS);
120124
this.tmpStorageBytesPerTask = Configs.valueOrDefault(tmpStorageBytesPerTask, DEFAULT_TMP_STORAGE_BYTES_PER_TASK);
121125
this.allowHadoopTaskExecution = allowHadoopTaskExecution;
126+
this.buildV10 = buildV10;
122127
}
123128

124129
private TaskConfig(
@@ -132,7 +137,8 @@ private TaskConfig(
132137
boolean storeEmptyColumns,
133138
boolean encapsulatedTask,
134139
long tmpStorageBytesPerTask,
135-
boolean allowHadoopTaskExecution
140+
boolean allowHadoopTaskExecution,
141+
boolean buildV10
136142
)
137143
{
138144
this.baseDir = baseDir;
@@ -146,6 +152,7 @@ private TaskConfig(
146152
this.encapsulatedTask = encapsulatedTask;
147153
this.tmpStorageBytesPerTask = tmpStorageBytesPerTask;
148154
this.allowHadoopTaskExecution = allowHadoopTaskExecution;
155+
this.buildV10 = buildV10;
149156
}
150157

151158
@JsonProperty
@@ -244,6 +251,12 @@ public boolean isAllowHadoopTaskExecution()
244251
return allowHadoopTaskExecution;
245252
}
246253

254+
@JsonProperty
255+
public boolean buildV10()
256+
{
257+
return buildV10;
258+
}
259+
247260
private String defaultDir(@Nullable String configParameter, final String defaultVal)
248261
{
249262
if (configParameter == null) {
@@ -266,7 +279,8 @@ public TaskConfig withBaseTaskDir(File baseTaskDir)
266279
storeEmptyColumns,
267280
encapsulatedTask,
268281
tmpStorageBytesPerTask,
269-
allowHadoopTaskExecution
282+
allowHadoopTaskExecution,
283+
buildV10
270284
);
271285
}
272286

@@ -283,7 +297,8 @@ public TaskConfig withTmpStorageBytesPerTask(long tmpStorageBytesPerTask)
283297
storeEmptyColumns,
284298
encapsulatedTask,
285299
tmpStorageBytesPerTask,
286-
allowHadoopTaskExecution
300+
allowHadoopTaskExecution,
301+
buildV10
287302
);
288303
}
289304
}

‎indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
import org.apache.druid.segment.indexing.DataSchema;
9090
import org.apache.druid.segment.indexing.TuningConfig;
9191
import org.apache.druid.segment.loading.SegmentCacheManager;
92+
import org.apache.druid.segment.projections.AggregateProjectionSchema;
9293
import org.apache.druid.segment.realtime.appenderator.AppenderatorsManager;
9394
import org.apache.druid.segment.transform.CompactionTransformSpec;
9495
import org.apache.druid.segment.transform.TransformSpec;
@@ -1134,7 +1135,7 @@ private void processProjections(final QueryableIndex index)
11341135
if (metadata != null && metadata.getProjections() != null && !metadata.getProjections().isEmpty()) {
11351136
projections = new HashMap<>();
11361137
for (AggregateProjectionMetadata projectionMetadata : metadata.getProjections()) {
1137-
final AggregateProjectionMetadata.Schema schema = projectionMetadata.getSchema();
1138+
final AggregateProjectionSchema schema = projectionMetadata.getSchema();
11381139
final QueryableIndex projectionIndex = Preconditions.checkNotNull(
11391140
index.getProjectionQueryableIndex(schema.getName())
11401141
);

‎indexing-service/src/test/java/org/apache/druid/indexing/common/TaskToolboxTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ public void setUp() throws IOException
148148
null,
149149
null,
150150
null,
151+
null,
151152
new NoopTestTaskReportFileWriter(),
152153
null,
153154
AuthTestUtils.TEST_AUTHORIZER_MAPPER,

‎indexing-service/src/test/java/org/apache/druid/indexing/common/config/TaskConfigBuilder.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public class TaskConfigBuilder
3737
private boolean enableTaskLevelLogPush;
3838
private Long tmpStorageBytesPerTask;
3939
private boolean allowHadoopTaskExecution;
40+
private boolean buildV10;
4041

4142
public TaskConfigBuilder setBaseDir(String baseDir)
4243
{
@@ -104,6 +105,12 @@ public TaskConfigBuilder setAllowHadoopTaskExecution(boolean allowHadoopTaskExec
104105
return this;
105106
}
106107

108+
public TaskConfigBuilder setBuildV10(boolean buildV10)
109+
{
110+
this.buildV10 = buildV10;
111+
return this;
112+
}
113+
107114
public TaskConfig build()
108115
{
109116
return new TaskConfig(
@@ -117,7 +124,8 @@ public TaskConfig build()
117124
storeEmptyColumns,
118125
enableTaskLevelLogPush,
119126
tmpStorageBytesPerTask,
120-
allowHadoopTaskExecution
127+
allowHadoopTaskExecution,
128+
buildV10
121129
);
122130
}
123131
}

‎indexing-service/src/test/java/org/apache/druid/indexing/overlord/SingleTaskBackgroundRunnerTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ public void setup() throws IOException
126126
null,
127127
utils.getIndexMergerV9Factory(),
128128
null,
129+
null,
129130
node,
130131
null,
131132
null,

‎indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLifecycleTest.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,7 @@ public void announceSegment(DataSegment segment)
599599
new CacheConfig(),
600600
new CachePopulatorStats(),
601601
INDEX_MERGER_V9_FACTORY,
602+
null,
602603
EasyMock.createNiceMock(DruidNodeAnnouncer.class),
603604
EasyMock.createNiceMock(DruidNode.class),
604605
new LookupNodeService("tier"),

‎indexing-service/src/test/java/org/apache/druid/indexing/overlord/TestTaskToolboxFactory.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ public TestTaskToolboxFactory(
107107
bob.cacheConfig,
108108
bob.cachePopulatorStats,
109109
bob.indexMergerV9Factory,
110+
null,
110111
bob.druidNodeAnnouncer,
111112
bob.druidNode,
112113
bob.lookupNodeService,

‎indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,7 @@ public void close()
697697
new CacheConfig(),
698698
new CachePopulatorStats(),
699699
testUtils.getIndexMergerV9Factory(),
700+
null,
700701
EasyMock.createNiceMock(DruidNodeAnnouncer.class),
701702
EasyMock.createNiceMock(DruidNode.class),
702703
new LookupNodeService("tier"),

0 commit comments

Comments
 (0)