branch-4.0: [fix](ann range search) range search prepare failed on NULL literal #60564 (#60821)

cherry pick from #60564

Author: zhiqiang-hhhh

be/src/vec/exprs/vectorized_fn_call.cpp

@@ -391,31 +391,7 @@ void VectorizedFnCall::prepare_ann_range_search(
     auto left_child = get_child(0);
     auto right_child = get_child(1);
 
-    auto right_literal = std::dynamic_pointer_cast<VLiteral>(right_child);
-    if (right_literal == nullptr) {
-        suitable_for_ann_index = false;
-        return;
-    }
-
-    auto right_col = right_literal->get_column_ptr()->convert_to_full_column_if_const();
-    auto right_type = right_literal->get_data_type();
-
-    PrimitiveType right_primitive = right_type->get_primitive_type();
-    const bool float32_literal = right_primitive == PrimitiveType::TYPE_FLOAT;
-    const bool float64_literal = right_primitive == PrimitiveType::TYPE_DOUBLE;
-    if (!float32_literal && !float64_literal) {
-        mark_unsuitable("Right child is not a Float32Literal or Float64Literal.");
-        return;
-    }
-
-    if (float32_literal) {
-        const ColumnFloat32* cf32_right = assert_cast<const ColumnFloat32*>(right_col.get());
-        range_search_runtime.radius = cf32_right->get_data()[0];
-    } else if (float64_literal) {
-        const ColumnFloat64* cf64_right = assert_cast<const ColumnFloat64*>(right_col.get());
-        range_search_runtime.radius = static_cast<float>(cf64_right->get_data()[0]);
-    }
-
+    // ========== Step 1: Check left child - must be a distance function ==========
     auto get_virtual_expr = [&](const VExprSPtr& expr,
                                 std::shared_ptr<VirtualSlotRef>& slot_ref) -> VExprSPtr {
         auto virtual_ref = std::dynamic_pointer_cast<VirtualSlotRef>(expr);
@@ -430,40 +406,40 @@ void VectorizedFnCall::prepare_ann_range_search(
     std::shared_ptr<VirtualSlotRef> vir_slot_ref;
     auto normalized_left = get_virtual_expr(left_child, vir_slot_ref);
 
-    std::shared_ptr<VectorizedFnCall> function_call;
-    if (float32_literal) {
-        function_call = std::dynamic_pointer_cast<VectorizedFnCall>(normalized_left);
-        if (function_call == nullptr) {
-            mark_unsuitable("Left child is not a function call.");
-            return;
-        }
-    } else {
-        auto cast_float_to_double = std::dynamic_pointer_cast<VCastExpr>(normalized_left);
-        if (cast_float_to_double == nullptr) {
-            mark_unsuitable("Left child is not a cast expression.");
+    // Try to find the distance function call, it may be wrapped in a Cast(Float->Double)
+    std::shared_ptr<VectorizedFnCall> function_call =
+            std::dynamic_pointer_cast<VectorizedFnCall>(normalized_left);
+    bool has_float_to_double_cast = false;
+
+    if (function_call == nullptr) {
+        // Check if it's a Cast expression wrapping a function call
+        auto cast_expr = std::dynamic_pointer_cast<VCastExpr>(normalized_left);
+        if (cast_expr == nullptr) {
+            mark_unsuitable("Left child is neither a function call nor a cast expression.");
             return;
         }
-
-        auto normalized_cast_child =
-                get_virtual_expr(cast_float_to_double->get_child(0), vir_slot_ref);
+        has_float_to_double_cast = true;
+        auto normalized_cast_child = get_virtual_expr(cast_expr->get_child(0), vir_slot_ref);
         function_call = std::dynamic_pointer_cast<VectorizedFnCall>(normalized_cast_child);
         if (function_call == nullptr) {
             mark_unsuitable("Left child of cast is not a function call.");
             return;
         }
     }
 
+    // Check if it's a supported distance function
     if (DISTANCE_FUNCS.find(function_call->_function_name) == DISTANCE_FUNCS.end()) {
         mark_unsuitable(fmt::format("Left child is not a supported distance function: {}",
                                     function_call->_function_name));
         return;
-    } else {
-        // Strip the _approximate suffix.
-        std::string metric_name = function_call->_function_name;
-        metric_name = metric_name.substr(0, metric_name.size() - 12);
-        range_search_runtime.metric_type = segment_v2::string_to_metric(metric_name);
     }
 
+    // Strip the _approximate suffix to get metric type
+    std::string metric_name = function_call->_function_name;
+    metric_name = metric_name.substr(0, metric_name.size() - 12);
+    range_search_runtime.metric_type = segment_v2::string_to_metric(metric_name);
+
+    // ========== Step 2: Validate distance function arguments ==========
     // Identify the slot ref child and the constant query array child (ArrayLiteral or CAST to array)
     Int32 idx_of_slot_ref = -1;
     Int32 idx_of_array_expr = -1;
@@ -502,6 +478,47 @@ void VectorizedFnCall::prepare_ann_range_search(
     }
     range_search_runtime.query_value = extract_result.value();
     range_search_runtime.dim = range_search_runtime.query_value->size();
+
+    // ========== Step 3: Check right child - must be a float/double literal ==========
+    auto right_literal = std::dynamic_pointer_cast<VLiteral>(right_child);
+    if (right_literal == nullptr) {
+        mark_unsuitable("Right child is not a literal.");
+        return;
+    }
+
+    // Handle nullable literal gracefully - just mark as unsuitable instead of crash
+    if (right_literal->is_nullable()) {
+        mark_unsuitable("Right literal is nullable, not supported for ANN range search.");
+        return;
+    }
+
+    auto right_type = right_literal->get_data_type();
+    PrimitiveType right_primitive = right_type->get_primitive_type();
+    const bool float32_literal = right_primitive == PrimitiveType::TYPE_FLOAT;
+    const bool float64_literal = right_primitive == PrimitiveType::TYPE_DOUBLE;
+
+    if (!float32_literal && !float64_literal) {
+        mark_unsuitable("Right child is not a Float32Literal or Float64Literal.");
+        return;
+    }
+
+    // Validate consistency: if we have Cast(Float->Double), right must be double literal
+    if (has_float_to_double_cast && !float64_literal) {
+        mark_unsuitable("Cast expression expects double literal on right side.");
+        return;
+    }
+
+    // Extract radius value
+    auto right_col = right_literal->get_column_ptr()->convert_to_full_column_if_const();
+    if (float32_literal) {
+        const ColumnFloat32* cf32_right = assert_cast<const ColumnFloat32*>(right_col.get());
+        range_search_runtime.radius = cf32_right->get_data()[0];
+    } else {
+        const ColumnFloat64* cf64_right = assert_cast<const ColumnFloat64*>(right_col.get());
+        range_search_runtime.radius = static_cast<float>(cf64_right->get_data()[0]);
+    }
+
+    // ========== Done: Mark as suitable for ANN range search ==========
     range_search_runtime.is_ann_range_search = true;
     range_search_runtime.user_params = user_params;
     VLOG_DEBUG << fmt::format("Ann range search params: {}", range_search_runtime.to_string());

regression-test/data/ann_index_p0/ann_range_search_nullable_literal.out

@@ -0,0 +1,54 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !nullable_subquery_empty --
+
+-- !nullable_subquery_empty_ge --
+
+-- !nullable_subquery_all_null --
+
+-- !nullable_subquery_normal --
+0	[1, 2, 3, 4]
+1	[2, 3, 4, 5]
+2	[3, 4, 5, 6]
+
+-- !nullable_subquery_normal_max --
+0	[1, 2, 3, 4]
+1	[2, 3, 4, 5]
+2	[3, 4, 5, 6]
+3	[4, 5, 6, 7]
+4	[5, 6, 7, 8]
+
+-- !coalesce_with_null --
+0	[1, 2, 3, 4]
+1	[2, 3, 4, 5]
+2	[3, 4, 5, 6]
+
+-- !case_nullable --
+0	[1, 2, 3, 4]
+1	[2, 3, 4, 5]
+2	[3, 4, 5, 6]
+
+-- !normal_literal --
+0	[1, 2, 3, 4]
+1	[2, 3, 4, 5]
+2	[3, 4, 5, 6]
+
+-- !ip_nullable_subquery --
+
+-- !non_dist_nullable_empty --
+
+-- !non_dist_nullable_all_null --
+
+-- !non_dist_nullable_normal --
+0	[1, 2, 3, 4]
+1	[2, 3, 4, 5]
+
+-- !non_dist_func_nullable --
+
+-- !arithmetic_nullable --
+
+-- !mixed_dist_and_regular_nullable --
+
+-- !dist_normal_regular_nullable --
+
+-- !or_condition_nullable --
+