docs: Improve documentation for FunctionFactory / CREATE FUNCTION (#17859)

* docs: Improve documentation for FunctionFactory / CREATE FUNCTION

* More docs

* Improve docs

* Update datafusion/core/src/execution/context/mod.rs

Co-authored-by: Marko Milenković <[email protected]>

---------

Co-authored-by: Marko Milenković <[email protected]>

Author: alamb

datafusion/core/src/execution/context/mod.rs

@@ -1786,28 +1786,63 @@ impl From<SessionContext> for SessionStateBuilder {
 /// A planner used to add extensions to DataFusion logical and physical plans.
 #[async_trait]
 pub trait QueryPlanner: Debug {
-    /// Given a `LogicalPlan`, create an [`ExecutionPlan`] suitable for execution
+    /// Given a [`LogicalPlan`], create an [`ExecutionPlan`] suitable for execution
     async fn create_physical_plan(
         &self,
         logical_plan: &LogicalPlan,
         session_state: &SessionState,
     ) -> Result<Arc<dyn ExecutionPlan>>;
 }
 
-/// A pluggable interface to handle `CREATE FUNCTION` statements
-/// and interact with [SessionState] to registers new udf, udaf or udwf.
+/// Interface for handling `CREATE FUNCTION` statements and interacting with
+/// [SessionState] to create and register functions ([`ScalarUDF`],
+/// [`AggregateUDF`], [`WindowUDF`], and [`TableFunctionImpl`]) dynamically.
+///
+/// Implement this trait to create user-defined functions in a custom way, such
+/// as loading from external libraries or defining them programmatically.
+/// DataFusion will parse `CREATE FUNCTION` statements into [`CreateFunction`]
+/// structs and pass them to the [`create`](Self::create) method.
+///
+/// Note there is no default implementation of this trait provided in DataFusion,
+/// because the implementation and requirements vary widely. Please see
+/// [function_factory example] for a reference implementation.
+///
+/// [function_factory example]: https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/function_factory.rs
+///
+/// # Examples of syntax that can be supported
+///
+/// ```sql
+/// CREATE FUNCTION f1(BIGINT)
+///   RETURNS BIGINT
+///   RETURN $1 + 1;
+/// ```
+/// or
+/// ```sql
+/// CREATE FUNCTION to_miles(DOUBLE)
+/// RETURNS DOUBLE
+/// LANGUAGE PYTHON
+/// AS '
+/// import pyarrow.compute as pc
+///
+/// conversation_rate_multiplier = 0.62137119
+///
+/// def to_miles(km_data):
+///     return pc.multiply(km_data, conversation_rate_multiplier)
+/// '
+/// ```
 
 #[async_trait]
 pub trait FunctionFactory: Debug + Sync + Send {
-    /// Handles creation of user defined function specified in [CreateFunction] statement
+    /// Creates a new dynamic function from the SQL in the [CreateFunction] statement
     async fn create(
         &self,
         state: &SessionState,
         statement: CreateFunction,
     ) -> Result<RegisterFunction>;
 }
 
-/// Type of function to create
+/// The result of processing a [`CreateFunction`] statement with [`FunctionFactory`].
+#[derive(Debug, Clone)]
 pub enum RegisterFunction {
     /// Scalar user defined function
     Scalar(Arc<ScalarUDF>),

datafusion/expr-common/src/signature.rs

@@ -49,7 +49,6 @@ pub const FIXED_SIZE_LIST_WILDCARD: i32 = i32::MIN;
 /// optimizations. You should always define your function to have the strictest
 /// possible volatility to maximize performance and avoid unexpected
 /// results.
-///
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
 pub enum Volatility {
     /// Always returns the same output when given the same input.

datafusion/expr/src/logical_plan/ddl.rs

@@ -458,14 +458,20 @@ impl PartialOrd for DropCatalogSchema {
     }
 }
 
-/// Arguments passed to `CREATE FUNCTION`
+/// Arguments passed to the `CREATE FUNCTION` statement
 ///
-/// Note this meant to be the same as from sqlparser's [`sqlparser::ast::Statement::CreateFunction`]
+/// These statements are turned into executable functions using [`FunctionFactory`]
+///
+/// # Notes
+///
+/// This structure purposely mirrors the structure in sqlparser's
+/// [`sqlparser::ast::Statement::CreateFunction`], but does not use it directly
+/// to avoid a dependency on sqlparser in the core crate.
+///
+///
+/// [`FunctionFactory`]: https://docs.rs/datafusion/latest/datafusion/execution/context/trait.FunctionFactory.html
 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
 pub struct CreateFunction {
-    // TODO: There is open question should we expose sqlparser types or redefine them here?
-    //       At the moment it make more sense to expose sqlparser types and leave
-    //       user to convert them as needed
     pub or_replace: bool,
     pub temporary: bool,
     pub name: String,
@@ -511,6 +517,9 @@ impl PartialOrd for CreateFunction {
     }
 }
 
+/// Part of the `CREATE FUNCTION` statement
+///
+/// See [`CreateFunction`] for details
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
 pub struct OperateFunctionArg {
     // TODO: figure out how to support mode
@@ -541,6 +550,9 @@ impl<'a> TreeNodeContainer<'a, Expr> for OperateFunctionArg {
     }
 }
 
+/// Part of the `CREATE FUNCTION` statement
+///
+/// See [`CreateFunction`] for details
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
 pub struct CreateFunctionBody {
     /// LANGUAGE lang_name