chore(query): improve distributed runtime filter (#16862)

* chore(query): improve distributed runtime filter

* chore(code): refine code

* chore(code): refine code

* chore(code); fix make lint

* chore(query): add scan id to fix same table index

* chore(code): make lint

* chore(pipeline): fix deserializer runtime filter key

Author: Dousir9

src/query/catalog/src/plan/datasource/datasource_plan.rs

@@ -47,6 +47,7 @@ pub struct DataSourcePlan {
     pub data_mask_policy: Option<BTreeMap<FieldIndex, RemoteExpr>>,
 
     pub table_index: usize,
+    pub scan_id: usize,
 }
 
 impl DataSourcePlan {

src/query/catalog/src/runtime_filter_info.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use databend_common_base::base::tokio::sync::watch;
+use databend_common_base::base::tokio::sync::watch::Receiver;
+use databend_common_base::base::tokio::sync::watch::Sender;
 use databend_common_expression::Expr;
 use xorf::BinaryFuse16;
 
@@ -62,4 +65,24 @@ impl RuntimeFilterInfo {
     pub fn is_empty(&self) -> bool {
         self.inlist.is_empty() && self.bloom.is_empty() && self.min_max.is_empty()
     }
+
+    pub fn is_blooms_empty(&self) -> bool {
+        self.bloom.is_empty()
+    }
+}
+
+pub struct RuntimeFilterReady {
+    pub runtime_filter_watcher: Sender<Option<bool>>,
+    /// A dummy receiver to make runtime_filter_watcher channel open.
+    pub _runtime_filter_dummy_receiver: Receiver<Option<bool>>,
+}
+
+impl Default for RuntimeFilterReady {
+    fn default() -> Self {
+        let (watcher, dummy_receiver) = watch::channel(None);
+        Self {
+            runtime_filter_watcher: watcher,
+            _runtime_filter_dummy_receiver: dummy_receiver,
+        }
+    }
 }

src/query/catalog/src/table_context.rs

@@ -74,6 +74,7 @@ use crate::plan::PartInfoPtr;
 use crate::plan::Partitions;
 use crate::query_kind::QueryKind;
 use crate::runtime_filter_info::RuntimeFilterInfo;
+use crate::runtime_filter_info::RuntimeFilterReady;
 use crate::statistics::data_cache_statistics::DataCacheMetrics;
 use crate::table::Table;
 
@@ -317,6 +318,14 @@ pub trait TableContext: Send + Sync {
 
     fn set_runtime_filter(&self, filters: (usize, RuntimeFilterInfo));
 
+    fn set_runtime_filter_ready(&self, table_index: usize, ready: Arc<RuntimeFilterReady>);
+
+    fn get_runtime_filter_ready(&self, table_index: usize) -> Vec<Arc<RuntimeFilterReady>>;
+
+    fn set_wait_runtime_filter(&self, table_index: usize, need_to_wait: bool);
+
+    fn get_wait_runtime_filter(&self, table_index: usize) -> bool;
+
     fn clear_runtime_filter(&self);
 
     fn set_merge_into_join(&self, join: MergeIntoJoin);

src/query/service/src/interpreters/interpreter_copy_into_table.rs

@@ -137,6 +137,7 @@ impl CopyIntoTableInterpreter {
             (
                 CopyIntoTableSource::Stage(Box::new(PhysicalPlan::TableScan(TableScan {
                     plan_id: 0,
+                    scan_id: 0,
                     name_mapping,
                     stat_info: None,
                     table_index: None,

src/query/service/src/pipelines/builders/builder_aggregate.rs

@@ -97,6 +97,7 @@ impl PipelineBuilder {
     }
 
     pub(crate) fn build_aggregate_partial(&mut self, aggregate: &AggregatePartial) -> Result<()> {
+        self.contain_sink_processor = true;
         self.build_pipeline(&aggregate.input)?;
 
         let max_block_size = self.settings.get_max_block_size()?;

src/query/service/src/pipelines/builders/builder_join.rs

@@ -163,6 +163,7 @@ impl PipelineBuilder {
             join_state.clone(),
             output_len,
         )?;
+        build_state.add_runtime_filter_ready();
 
         let create_sink_processor = |input| {
             Ok(ProcessorPtr::create(TransformHashJoinBuild::try_create(

src/query/service/src/pipelines/builders/builder_recluster.rs

@@ -68,6 +68,7 @@ impl PipelineBuilder {
                     update_stream_columns: table.change_tracking_enabled(),
                     data_mask_policy: None,
                     table_index: usize::MAX,
+                    scan_id: usize::MAX,
                 };
 
                 {

src/query/service/src/pipelines/builders/builder_scan.rs

@@ -41,6 +41,8 @@ impl PipelineBuilder {
     pub(crate) fn build_table_scan(&mut self, scan: &TableScan) -> Result<()> {
         let table = self.ctx.build_table_from_source_plan(&scan.source)?;
         self.ctx.set_partitions(scan.source.parts.clone())?;
+        self.ctx
+            .set_wait_runtime_filter(scan.scan_id, self.contain_sink_processor);
         table.read_data(
             self.ctx.clone(),
             &scan.source,

src/query/service/src/pipelines/pipeline_builder.rs

@@ -60,6 +60,8 @@ pub struct PipelineBuilder {
 
     pub r_cte_scan_interpreters: Vec<CreateTableInterpreter>,
     pub(crate) is_exchange_neighbor: bool,
+
+    pub contain_sink_processor: bool,
 }
 
 impl PipelineBuilder {
@@ -83,6 +85,7 @@ impl PipelineBuilder {
             hash_join_states: HashMap::new(),
             r_cte_scan_interpreters: vec![],
             is_exchange_neighbor: false,
+            contain_sink_processor: false,
         }
     }
 

src/query/service/src/pipelines/processors/transforms/hash_join/build_state.rs

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
+use databend_common_catalog::runtime_filter_info::RuntimeFilterReady;
 use databend_common_expression::types::DataType;
 use databend_common_expression::ColumnVec;
 use databend_common_expression::DataBlock;
@@ -23,6 +26,7 @@ pub struct BuildState {
     pub(crate) outer_scan_map: Vec<Vec<bool>>,
     /// LeftMarkScan map, initialized at `HashJoinBuildState`, used in `HashJoinProbeState`
     pub(crate) mark_scan_map: Vec<Vec<u8>>,
+    pub(crate) runtime_filter_ready: Vec<Arc<RuntimeFilterReady>>,
 }
 
 impl BuildState {
@@ -31,6 +35,7 @@ impl BuildState {
             generation_state: BuildBlockGenerationState::new(),
             outer_scan_map: Vec::new(),
             mark_scan_map: Vec::new(),
+            runtime_filter_ready: Vec::new(),
         }
     }
 }