|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +use std::{ |
| 19 | + any::Any, |
| 20 | + fmt::Formatter, |
| 21 | + sync::Arc, |
| 22 | +}; |
| 23 | + |
| 24 | +use arrow::{ |
| 25 | + array::StringArray, |
| 26 | + datatypes::SchemaRef, |
| 27 | + record_batch::RecordBatch, |
| 28 | +}; |
| 29 | +use datafusion::{ |
| 30 | + execution::TaskContext, |
| 31 | + physical_plan::{ |
| 32 | + execution_plan::{Boundedness, EmissionType}, |
| 33 | + memory::MemoryStream, |
| 34 | + ExecutionPlan, Partitioning, |
| 35 | + PlanProperties, DisplayAs, DisplayFormatType, |
| 36 | + SendableRecordBatchStream, displayable, |
| 37 | + }, |
| 38 | + physical_expr::EquivalenceProperties, |
| 39 | +}; |
| 40 | + |
| 41 | +/// Custom distributed EXPLAIN execution plan that also returns distributed plan and stages |
| 42 | +#[derive(Debug)] |
| 43 | +pub struct DistributedExplainExec { |
| 44 | + schema: SchemaRef, |
| 45 | + logical_plan: String, |
| 46 | + physical_plan: String, |
| 47 | + distributed_plan: String, |
| 48 | + distributed_stages: String, |
| 49 | + properties: PlanProperties, |
| 50 | +} |
| 51 | + |
| 52 | +impl DistributedExplainExec { |
| 53 | + pub fn new( |
| 54 | + schema: SchemaRef, |
| 55 | + logical_plan: String, |
| 56 | + physical_plan: String, |
| 57 | + distributed_plan: String, |
| 58 | + distributed_stages: String, |
| 59 | + ) -> Self { |
| 60 | + // properties required by the ExecutionPlan trait |
| 61 | + let properties = PlanProperties::new( |
| 62 | + EquivalenceProperties::new(schema.clone()), |
| 63 | + Partitioning::UnknownPartitioning(1), |
| 64 | + EmissionType::Incremental, |
| 65 | + Boundedness::Bounded, |
| 66 | + ); |
| 67 | + |
| 68 | + Self { |
| 69 | + schema, |
| 70 | + logical_plan, |
| 71 | + physical_plan, |
| 72 | + distributed_plan, |
| 73 | + distributed_stages, |
| 74 | + properties, |
| 75 | + } |
| 76 | + } |
| 77 | + |
| 78 | + pub fn logical_plan(&self) -> &str { |
| 79 | + &self.logical_plan |
| 80 | + } |
| 81 | + |
| 82 | + pub fn physical_plan(&self) -> &str { |
| 83 | + &self.physical_plan |
| 84 | + } |
| 85 | + |
| 86 | + pub fn distributed_plan(&self) -> &str { |
| 87 | + &self.distributed_plan |
| 88 | + } |
| 89 | + |
| 90 | + pub fn distributed_stages(&self) -> &str { |
| 91 | + &self.distributed_stages |
| 92 | + } |
| 93 | + |
| 94 | + /// Format distributed stages for display |
| 95 | + pub fn format_distributed_stages(stages: &[crate::planning::DFRayStage]) -> String { |
| 96 | + let mut result = String::new(); |
| 97 | + for (i, stage) in stages.iter().enumerate() { |
| 98 | + result.push_str(&format!("Stage {}:\n", stage.stage_id)); |
| 99 | + result.push_str(&format!(" Partition Groups: {:?}\n", stage.partition_groups)); |
| 100 | + result.push_str(&format!(" Full Partitions: {}\n", stage.full_partitions)); |
| 101 | + result.push_str(" Plan:\n"); |
| 102 | + let plan_display = format!("{}", displayable(stage.plan.as_ref()).indent(true)); |
| 103 | + for line in plan_display.lines() { |
| 104 | + result.push_str(&format!(" {}\n", line)); |
| 105 | + } |
| 106 | + if i < stages.len() - 1 { |
| 107 | + result.push('\n'); |
| 108 | + } |
| 109 | + } |
| 110 | + if result.is_empty() { |
| 111 | + result.push_str("No distributed stages generated"); |
| 112 | + } |
| 113 | + result |
| 114 | + } |
| 115 | +} |
| 116 | + |
| 117 | +impl DisplayAs for DistributedExplainExec { |
| 118 | + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { |
| 119 | + write!(f, "DistributedExplainExec") |
| 120 | + } |
| 121 | +} |
| 122 | + |
| 123 | +impl ExecutionPlan for DistributedExplainExec { |
| 124 | + fn name(&self) -> &str { |
| 125 | + "DistributedExplainExec" |
| 126 | + } |
| 127 | + |
| 128 | + fn as_any(&self) -> &dyn Any { |
| 129 | + self |
| 130 | + } |
| 131 | + |
| 132 | + fn properties(&self) -> &PlanProperties { |
| 133 | + &self.properties |
| 134 | + } |
| 135 | + |
| 136 | + fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> { |
| 137 | + vec![] |
| 138 | + } |
| 139 | + |
| 140 | + fn with_new_children( |
| 141 | + self: Arc<Self>, |
| 142 | + _children: Vec<Arc<dyn ExecutionPlan>>, |
| 143 | + ) -> datafusion::error::Result<Arc<dyn ExecutionPlan>> { |
| 144 | + Ok(self) |
| 145 | + } |
| 146 | + |
| 147 | + fn execute( |
| 148 | + &self, |
| 149 | + _partition: usize, |
| 150 | + _context: Arc<TaskContext>, |
| 151 | + ) -> datafusion::error::Result<SendableRecordBatchStream> { |
| 152 | + let schema = self.schema.clone(); |
| 153 | + |
| 154 | + // Create the result data with our 4 plan types |
| 155 | + let plan_types = StringArray::from(vec![ |
| 156 | + "logical_plan", |
| 157 | + "physical_plan", |
| 158 | + "distributed_plan", |
| 159 | + "distributed_stages" |
| 160 | + ]); |
| 161 | + let plans = StringArray::from(vec![ |
| 162 | + self.logical_plan.as_str(), |
| 163 | + self.physical_plan.as_str(), |
| 164 | + self.distributed_plan.as_str(), |
| 165 | + self.distributed_stages.as_str(), |
| 166 | + ]); |
| 167 | + |
| 168 | + let batch = RecordBatch::try_new( |
| 169 | + schema.clone(), |
| 170 | + vec![Arc::new(plan_types), Arc::new(plans)], |
| 171 | + ).map_err(|e| datafusion::error::DataFusionError::ArrowError(e, None))?; |
| 172 | + |
| 173 | + // Use MemoryStream which is designed for DataFusion execution plans |
| 174 | + let stream = MemoryStream::try_new(vec![batch], schema, None)?; |
| 175 | + |
| 176 | + Ok(Box::pin(stream)) |
| 177 | + } |
| 178 | + |
| 179 | + fn schema(&self) -> SchemaRef { |
| 180 | + self.schema.clone() |
| 181 | + } |
| 182 | +} |
0 commit comments