Skip to content

Commit 6ee1b72

Browse files
SteveYurongSuSteveYurongSu
committed
Pipe: Disable unstable features in the distribution
1 parent f6e8aa7 commit 6ee1b72

File tree

10 files changed

+13
-616
lines changed

10 files changed

+13
-616
lines changed

‎.github/workflows/pipe-it-2cluster.yml‎

Lines changed: 0 additions & 466 deletions
This file was deleted.

‎integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -342,11 +342,6 @@ public void testAsyncConnectorUseNodeUrls() throws Exception {
342342
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName());
343343
}
344344

345-
@Test
346-
public void testAirGapConnectorUseNodeUrls() throws Exception {
347-
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName());
348-
}
349-
350345
private void doTestUseNodeUrls(String connectorName) throws Exception {
351346
senderEnv
352347
.getConfig()
@@ -381,16 +376,7 @@ private void doTestUseNodeUrls(String connectorName) throws Exception {
381376

382377
final StringBuilder nodeUrlsBuilder = new StringBuilder();
383378
for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) {
384-
if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) {
385-
// Use default port for convenience
386-
nodeUrlsBuilder
387-
.append(wrapper.getIp())
388-
.append(":")
389-
.append(wrapper.getPipeAirGapReceiverPort())
390-
.append(",");
391-
} else {
392-
nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
393-
}
379+
nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
394380
}
395381

396382
try (final SyncConfigNodeIServiceClient client =

‎iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionConnectorConstructor.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
2323
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.donothing.DoNothingConnector;
2424
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeConnectorConstructor;
25-
import org.apache.iotdb.confignode.manager.pipe.connector.protocol.IoTDBConfigRegionAirGapConnector;
2625
import org.apache.iotdb.confignode.manager.pipe.connector.protocol.IoTDBConfigRegionConnector;
2726
import org.apache.iotdb.pipe.api.PipeConnector;
2827

@@ -42,9 +41,6 @@ protected void initConstructors() {
4241
pluginConstructors.put(
4342
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(),
4443
IoTDBConfigRegionConnector::new);
45-
pluginConstructors.put(
46-
BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
47-
IoTDBConfigRegionAirGapConnector::new);
4844
pluginConstructors.put(
4945
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingConnector::new);
5046

@@ -59,9 +55,6 @@ protected void initConstructors() {
5955
pluginConstructors.put(
6056
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(),
6157
IoTDBConfigRegionConnector::new);
62-
pluginConstructors.put(
63-
BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(),
64-
IoTDBConfigRegionAirGapConnector::new);
6558
pluginConstructors.put(
6659
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingConnector::new);
6760
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionConnectorConstructor.java‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,7 @@
2323
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.donothing.DoNothingConnector;
2424
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeConnectorConstructor;
2525
import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
26-
import org.apache.iotdb.db.pipe.connector.protocol.airgap.IoTDBDataRegionAirGapConnector;
2726
import org.apache.iotdb.db.pipe.connector.protocol.legacy.IoTDBLegacyPipeConnector;
28-
import org.apache.iotdb.db.pipe.connector.protocol.opcda.OpcDaConnector;
29-
import org.apache.iotdb.db.pipe.connector.protocol.opcua.OpcUaConnector;
3027
import org.apache.iotdb.db.pipe.connector.protocol.pipeconsensus.PipeConsensusAsyncConnector;
3128
import org.apache.iotdb.db.pipe.connector.protocol.thrift.async.IoTDBDataRegionAsyncConnector;
3229
import org.apache.iotdb.db.pipe.connector.protocol.thrift.sync.IoTDBDataRegionSyncConnector;
@@ -59,15 +56,8 @@ protected void initConstructors() {
5956
pluginConstructors.put(
6057
BuiltinPipePlugin.IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName(),
6158
IoTDBLegacyPipeConnector::new);
62-
pluginConstructors.put(
63-
BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
64-
IoTDBDataRegionAirGapConnector::new);
6559
pluginConstructors.put(
6660
BuiltinPipePlugin.WEBSOCKET_CONNECTOR.getPipePluginName(), WebSocketConnector::new);
67-
pluginConstructors.put(
68-
BuiltinPipePlugin.OPC_UA_CONNECTOR.getPipePluginName(), OpcUaConnector::new);
69-
pluginConstructors.put(
70-
BuiltinPipePlugin.OPC_DA_CONNECTOR.getPipePluginName(), OpcDaConnector::new);
7161
pluginConstructors.put(
7262
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingConnector::new);
7363
pluginConstructors.put(
@@ -88,13 +78,8 @@ protected void initConstructors() {
8878
pluginConstructors.put(
8979
BuiltinPipePlugin.IOTDB_LEGACY_PIPE_SINK.getPipePluginName(),
9080
IoTDBLegacyPipeConnector::new);
91-
pluginConstructors.put(
92-
BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(),
93-
IoTDBDataRegionAirGapConnector::new);
9481
pluginConstructors.put(
9582
BuiltinPipePlugin.WEBSOCKET_SINK.getPipePluginName(), WebSocketConnector::new);
96-
pluginConstructors.put(BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName(), OpcUaConnector::new);
97-
pluginConstructors.put(BuiltinPipePlugin.OPC_DA_SINK.getPipePluginName(), OpcDaConnector::new);
9883
pluginConstructors.put(
9984
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingConnector::new);
10085
pluginConstructors.put(

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java‎

Lines changed: 0 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,9 @@
2121

2222
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
2323
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor;
24-
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor;
2524
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeProcessorConstructor;
2625
import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
27-
import org.apache.iotdb.db.pipe.processor.aggregate.AggregateProcessor;
28-
import org.apache.iotdb.db.pipe.processor.aggregate.operator.processor.StandardStatisticsOperatorProcessor;
29-
import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.TumblingWindowingProcessor;
30-
import org.apache.iotdb.db.pipe.processor.downsampling.changing.ChangingValueSamplingProcessor;
31-
import org.apache.iotdb.db.pipe.processor.downsampling.sdt.SwingingDoorTrendingSamplingProcessor;
32-
import org.apache.iotdb.db.pipe.processor.downsampling.tumbling.TumblingTimeSamplingProcessor;
3326
import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor;
34-
import org.apache.iotdb.db.pipe.processor.twostage.plugin.TwoStageCountProcessor;
3527

3628
class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor {
3729

@@ -43,28 +35,6 @@ class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor {
4335
protected void initConstructors() {
4436
pluginConstructors.put(
4537
BuiltinPipePlugin.DO_NOTHING_PROCESSOR.getPipePluginName(), DoNothingProcessor::new);
46-
pluginConstructors.put(
47-
BuiltinPipePlugin.TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName(),
48-
TumblingTimeSamplingProcessor::new);
49-
pluginConstructors.put(
50-
BuiltinPipePlugin.SDT_SAMPLING_PROCESSOR.getPipePluginName(),
51-
SwingingDoorTrendingSamplingProcessor::new);
52-
pluginConstructors.put(
53-
BuiltinPipePlugin.CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName(),
54-
ChangingValueSamplingProcessor::new);
55-
pluginConstructors.put(
56-
BuiltinPipePlugin.THROWING_EXCEPTION_PROCESSOR.getPipePluginName(),
57-
ThrowingExceptionProcessor::new);
58-
pluginConstructors.put(
59-
BuiltinPipePlugin.AGGREGATE_PROCESSOR.getPipePluginName(), AggregateProcessor::new);
60-
pluginConstructors.put(
61-
BuiltinPipePlugin.STANDARD_STATISTICS_PROCESSOR.getPipePluginName(),
62-
StandardStatisticsOperatorProcessor::new);
63-
pluginConstructors.put(
64-
BuiltinPipePlugin.TUMBLING_WINDOWING_PROCESSOR.getPipePluginName(),
65-
TumblingWindowingProcessor::new);
66-
pluginConstructors.put(
67-
BuiltinPipePlugin.COUNT_POINT_PROCESSOR.getPipePluginName(), TwoStageCountProcessor::new);
6838
pluginConstructors.put(
6939
BuiltinPipePlugin.PIPE_CONSENSUS_PROCESSOR.getPipePluginName(),
7040
PipeConsensusProcessor::new);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionConnectorConstructor.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
2323
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.donothing.DoNothingConnector;
2424
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeConnectorConstructor;
25-
import org.apache.iotdb.db.pipe.connector.protocol.airgap.IoTDBSchemaRegionAirGapConnector;
2625
import org.apache.iotdb.db.pipe.connector.protocol.thrift.sync.IoTDBSchemaRegionConnector;
2726
import org.apache.iotdb.pipe.api.PipeConnector;
2827

@@ -42,9 +41,6 @@ protected void initConstructors() {
4241
pluginConstructors.put(
4342
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(),
4443
IoTDBSchemaRegionConnector::new);
45-
pluginConstructors.put(
46-
BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
47-
IoTDBSchemaRegionAirGapConnector::new);
4844
pluginConstructors.put(
4945
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingConnector::new);
5046

@@ -59,9 +55,6 @@ protected void initConstructors() {
5955
pluginConstructors.put(
6056
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(),
6157
IoTDBSchemaRegionConnector::new);
62-
pluginConstructors.put(
63-
BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(),
64-
IoTDBSchemaRegionAirGapConnector::new);
6558
pluginConstructors.put(
6659
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingConnector::new);
6760
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.iotdb.commons.consensus.DataRegionId;
2323
import org.apache.iotdb.commons.exception.IllegalPathException;
24+
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
2425
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBPipePattern;
2526
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
2627
import org.apache.iotdb.commons.pipe.extractor.IoTDBExtractor;
@@ -171,26 +172,6 @@ public void validate(final PipeParameterValidator validator) throws Exception {
171172
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
172173
EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE));
173174

174-
// Validate extractor.realtime.mode
175-
if (validator
176-
.getParameters()
177-
.getBooleanOrDefault(
178-
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
179-
EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)
180-
|| validator.getParameters().hasAnyAttributes(SOURCE_START_TIME_KEY, SOURCE_END_TIME_KEY)) {
181-
validator.validateAttributeValueRange(
182-
validator.getParameters().hasAttribute(EXTRACTOR_REALTIME_MODE_KEY)
183-
? EXTRACTOR_REALTIME_MODE_KEY
184-
: SOURCE_REALTIME_MODE_KEY,
185-
true,
186-
EXTRACTOR_REALTIME_MODE_FILE_VALUE,
187-
EXTRACTOR_REALTIME_MODE_HYBRID_VALUE,
188-
EXTRACTOR_REALTIME_MODE_LOG_VALUE,
189-
EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE,
190-
EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE,
191-
EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE);
192-
}
193-
194175
// Validate source.start-time and source.end-time
195176
if (validator.getParameters().hasAnyAttributes(SOURCE_START_TIME_KEY, SOURCE_END_TIME_KEY)
196177
&& validator
@@ -267,6 +248,13 @@ private void constructRealtimeExtractor(final PipeParameters parameters)
267248
return;
268249
}
269250

251+
if (!(pipeName != null
252+
&& (pipeName.startsWith(PipeStaticMeta.SUBSCRIPTION_PIPE_PREFIX)
253+
|| pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)))) {
254+
realtimeExtractor = new PipeRealtimeDataRegionTsFileExtractor();
255+
return;
256+
}
257+
270258
// Use hybrid mode by default
271259
if (!parameters.hasAnyAttributes(EXTRACTOR_REALTIME_MODE_KEY, SOURCE_REALTIME_MODE_KEY)) {
272260
checkWalEnable(parameters);

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/realtime/PipeRealtimeDataRegionExtractor.java‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.iotdb.commons.exception.pipe.PipeRuntimeNonCriticalException;
2424
import org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
2525
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
26-
import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant;
2726
import org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskExtractorRuntimeEnvironment;
2827
import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
2928
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
@@ -103,7 +102,7 @@ public abstract class PipeRealtimeDataRegionExtractor implements PipeExtractor {
103102
private final AtomicReference<Pair<Long, Long>> dataRegionTimePartitionIdBound =
104103
new AtomicReference<>();
105104

106-
protected boolean isForwardingPipeRequests;
105+
protected boolean isForwardingPipeRequests = true;
107106

108107
private boolean shouldTransferModFile; // Whether to transfer mods
109108

@@ -226,12 +225,7 @@ public void customize(
226225
? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime)
227226
: TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1;
228227

229-
isForwardingPipeRequests =
230-
parameters.getBooleanOrDefault(
231-
Arrays.asList(
232-
PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY,
233-
PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY),
234-
PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
228+
isForwardingPipeRequests = true;
235229

236230
shouldTransferModFile =
237231
parameters.getBooleanOrDefault(

0 commit comments

Comments
 (0)