fix: Only use null-aware anti join for NOT IN, not NOT EXISTS

The previous implementation incorrectly applied null-aware semantics to ALL
LeftAnti joins, including NOT EXISTS subqueries. This was wrong because:

- **NOT IN**: Uses three-valued logic (TRUE/FALSE/UNKNOWN), requires null-aware
- **NOT EXISTS**: Uses two-valued logic (TRUE/FALSE), should NOT be null-aware

```sql
-- Setup: customers has (1, 2, 3, NULL), banned has (2, NULL)

-- NOT IN - Correctly returns empty (null-aware)
SELECT * FROM customers WHERE id NOT IN (SELECT id FROM banned);
-- Result: Empty (correct - NULL in subquery makes all comparisons UNKNOWN)

-- NOT EXISTS - Was incorrectly returning empty (bug)
SELECT * FROM customers c
WHERE NOT EXISTS (SELECT 1 FROM banned b WHERE c.id = b.id);
-- Expected: (1, 3, NULL) - NULL=NULL is FALSE, so no matches for these rows
-- Actual (buggy): Empty - incorrectly using null-aware semantics
```

In `decorrelate_predicate_subquery.rs`, line 424:
```rust
let null_aware = matches!(join_type, JoinType::LeftAnti);
```

This set `null_aware=true` for ALL LeftAnti joins, but it should only be
true for NOT IN (InSubquery), not NOT EXISTS (Exists).

The `SubqueryInfo` struct already distinguishes between them:
- **NOT IN**: Created with `new_with_in_expr()` → `in_predicate_opt` is `Some(...)`
- **NOT EXISTS**: Created with `new()` → `in_predicate_opt` is `None`

Fixed by checking both conditions:
```rust
let null_aware = matches!(join_type, JoinType::LeftAnti)
    && in_predicate_opt.is_some();  // Only NOT IN, not NOT EXISTS
```

**File**: `datafusion/optimizer/src/decorrelate_predicate_subquery.rs`

- Updated null_aware detection to only apply to NOT IN (lines 420-426)
- Added comprehensive comments explaining the distinction
- Check `in_predicate_opt.is_some()` to distinguish NOT IN from NOT EXISTS

**File**: `datafusion/sqllogictest/test_files/null_aware_anti_join.slt`

Added 5 new test scenarios (Tests 14-18):

**Test 14**: Direct comparison of NOT IN vs NOT EXISTS with NULLs
- NOT IN with NULL → empty result (null-aware)
- NOT EXISTS with NULL → returns non-matching rows (NOT null-aware)
- EXPLAIN verification

**Test 15**: NOT EXISTS with no NULLs

**Test 16**: NOT EXISTS with correlated subquery

**Test 17**: NOT EXISTS with all-NULL subquery
- Shows that NOT EXISTS returns all rows (NULL=NULL is FALSE)
- Compares with NOT IN which correctly returns empty

**Test 18**: Nested NOT EXISTS and NOT IN
- Verifies correct interaction between the two

```bash
cargo test -p datafusion-sqllogictest --test sqllogictests -- null_aware_anti_join

cargo test -p datafusion-sqllogictest --test sqllogictests subquery.slt

cargo test -p datafusion-optimizer --lib

cargo test -p datafusion-physical-plan --lib hash_join
```

This fix ensures DataFusion correctly implements SQL semantics:
- NOT IN subqueries now correctly use null-aware anti join (three-valued logic)
- NOT EXISTS subqueries now correctly use regular anti join (two-valued logic)

Users can now reliably use both NOT IN and NOT EXISTS with confidence that
NULL handling follows SQL standards.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: viirya

datafusion/optimizer/src/decorrelate_predicate_subquery.rs

@@ -418,10 +418,12 @@ fn build_join(
     }
 
     // Determine if this should be a null-aware anti join
-    // For LeftAnti joins (NOT IN), we need null-aware semantics if:
-    // 1. The join type is LeftAnti
-    // 2. The join predicate involves nullable columns (conservative: assume nullable)
-    let null_aware = matches!(join_type, JoinType::LeftAnti);
+    // Null-aware semantics are only needed for NOT IN subqueries, not NOT EXISTS:
+    // - NOT IN: Uses three-valued logic, requires null-aware handling
+    // - NOT EXISTS: Uses two-valued logic, regular anti join is correct
+    // We can distinguish them: NOT IN has in_predicate_opt, NOT EXISTS does not
+    let null_aware = matches!(join_type, JoinType::LeftAnti)
+        && in_predicate_opt.is_some();
 
     // join our sub query into the main plan
     let new_plan = if null_aware {

datafusion/sqllogictest/test_files/null_aware_anti_join.slt

@@ -229,6 +229,141 @@ SELECT * FROM outer_table
 WHERE id NOT IN (SELECT DISTINCT id FROM duplicates_with_null);
 ----
 
+#############
+## Test 14: NOT EXISTS vs NOT IN - Demonstrating the difference
+#############
+
+# NOT EXISTS should NOT use null-aware semantics
+# It uses two-valued logic (TRUE/FALSE), not three-valued logic (TRUE/FALSE/UNKNOWN)
+
+# Setup tables for comparison
+statement ok
+CREATE TABLE customers(id INT, name TEXT) AS VALUES
+(1, 'Alice'),
+(2, 'Bob'),
+(3, 'Charlie'),
+(NULL, 'Dave');
+
+statement ok
+CREATE TABLE banned(id INT) AS VALUES
+(2),
+(NULL);
+
+# Test 14a: NOT IN with NULL in subquery - Returns EMPTY (null-aware)
+query IT rowsort
+SELECT * FROM customers WHERE id NOT IN (SELECT id FROM banned);
+----
+
+# Test 14b: NOT EXISTS with NULL in subquery - Returns rows (NOT null-aware)
+# This should return (1, 'Alice'), (3, 'Charlie'), (NULL, 'Dave')
+# Because NOT EXISTS uses two-valued logic: NULL = NULL is FALSE, so no match found
+query IT rowsort
+SELECT * FROM customers c
+WHERE NOT EXISTS (SELECT 1 FROM banned b WHERE c.id = b.id);
+----
+1 Alice
+3 Charlie
+NULL Dave
+
+# Test 14c: Verify with EXPLAIN that NOT EXISTS doesn't use null-aware
+query TT
+EXPLAIN SELECT * FROM customers c
+WHERE NOT EXISTS (SELECT 1 FROM banned b WHERE c.id = b.id);
+----
+logical_plan
+01)LeftAnti Join: c.id = __correlated_sq_1.id
+02)--SubqueryAlias: c
+03)----TableScan: customers projection=[id, name]
+04)--SubqueryAlias: __correlated_sq_1
+05)----SubqueryAlias: b
+06)------TableScan: banned projection=[id]
+physical_plan
+01)HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(id@0, id@0)]
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+03)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+#############
+## Test 15: NOT EXISTS - No NULLs
+#############
+
+statement ok
+CREATE TABLE active_customers(id INT) AS VALUES (1), (3);
+
+# Should return only Bob (id=2) and Dave (id=NULL)
+query IT rowsort
+SELECT * FROM customers c
+WHERE NOT EXISTS (SELECT 1 FROM active_customers a WHERE c.id = a.id);
+----
+2 Bob
+NULL Dave
+
+#############
+## Test 16: NOT EXISTS - Correlated subquery
+#############
+
+statement ok
+CREATE TABLE orders_test(order_id INT, customer_id INT) AS VALUES
+(1, 100),
+(2, 200),
+(3, NULL);
+
+statement ok
+CREATE TABLE customers_test(customer_id INT, name TEXT) AS VALUES
+(100, 'Alice'),
+(200, 'Bob'),
+(300, 'Charlie'),
+(NULL, 'Unknown');
+
+# Find customers with no orders
+# Should return Charlie (300) and Unknown (NULL)
+query IT rowsort
+SELECT * FROM customers_test c
+WHERE NOT EXISTS (
+    SELECT 1 FROM orders_test o WHERE o.customer_id = c.customer_id
+);
+----
+300 Charlie
+NULL Unknown
+
+#############
+## Test 17: NOT EXISTS with all NULL subquery
+#############
+
+statement ok
+CREATE TABLE all_null_banned(id INT) AS VALUES (NULL), (NULL);
+
+# NOT EXISTS should return all rows because NULL = NULL is FALSE (no matches)
+query IT rowsort
+SELECT * FROM customers c
+WHERE NOT EXISTS (SELECT 1 FROM all_null_banned b WHERE c.id = b.id);
+----
+1 Alice
+2 Bob
+3 Charlie
+NULL Dave
+
+# Compare with NOT IN which returns empty
+query IT rowsort
+SELECT * FROM customers WHERE id NOT IN (SELECT id FROM all_null_banned);
+----
+
+#############
+## Test 18: Nested NOT EXISTS and NOT IN
+#############
+
+# NOT EXISTS outside, NOT IN inside - should work correctly
+query IT rowsort
+SELECT * FROM customers c
+WHERE NOT EXISTS (
+    SELECT 1 FROM banned b
+    WHERE c.id = b.id
+    AND b.id NOT IN (SELECT id FROM active_customers)
+);
+----
+1 Alice
+3 Charlie
+NULL Dave
+
 #############
 ## Cleanup
 #############
@@ -259,3 +394,21 @@ DROP TABLE payments;
 
 statement ok
 DROP TABLE duplicates_with_null;
+
+statement ok
+DROP TABLE customers;
+
+statement ok
+DROP TABLE banned;
+
+statement ok
+DROP TABLE active_customers;
+
+statement ok
+DROP TABLE orders_test;
+
+statement ok
+DROP TABLE customers_test;
+
+statement ok
+DROP TABLE all_null_banned;