build: Upgrade Velox DuckDB from 0.8.1 to 1.4.4 (facebookincubator#16650)

Summary:

Upgrade DuckDB dependency for Velox from v0.8.1 to v1.4.4

Updates needed:

- Update default duckdb alias in third-party/duckdb/BUCK to point to 1.4.4
- Replace manually-copied DuckDB class definitions in DuckLogicalOperator.h with direct includes of v1.4.4 headers, since this was to work around DuckDB being so old
- Fix API compatibility in QueryPlanner.cpp:
  - LogicalGet::column_ids is now private; use GetColumnIds()/ColumnIndex
  - LogicalLimit uses BoundLimitNode instead of int64_t
  - TableFunction::to_string signature changed to use TableFunctionToStringInput
  - Aggregate function callback signatures gained AggregateFunction& parameter
  - Handle new LOGICAL_UNNEST operator type (split from LogicalGet)
- Fix DuckParser.cpp:
  - Handle new OPERATOR_TRY expression type (was parsed as function before)
  - Handle new WindowBoundary GROUPS variants
  - Support INTERVAL constants parsed directly by DuckDB 1.4.4
  - Unwrap cast(trunc(cast(...))) chains in interval parsing
- Fix QueryAssertions.cpp: Value::EMPTYLIST removed; use Value::LIST
- Update CMake/CI: version, commit hash, and ccache patch for v1.4.4

Differential Revision: D95309313

Author: hdikeman

.github/workflows/linux-build-base.yml

@@ -90,6 +90,7 @@ jobs:
       VELOX_DEPENDENCY_SOURCE: SYSTEM
       cudf_SOURCE: BUNDLED
       CUDA_VERSION: '12.9'
+      DuckDB_SOURCE: BUNDLED
       faiss_SOURCE: BUNDLED
       USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}"
     steps:
@@ -384,6 +385,7 @@ jobs:
       - name: Make Debug Build
         env:
           VELOX_DEPENDENCY_SOURCE: SYSTEM
+          DuckDB_SOURCE: BUNDLED
           faiss_SOURCE: BUNDLED
           fmt_SOURCE: BUNDLED
           simdjson_SOURCE: BUNDLED

.github/workflows/scheduled.yml

@@ -216,6 +216,7 @@ jobs:
 
       - name: Build
         env:
+          DuckDB_SOURCE: BUNDLED
           EXTRA_CMAKE_FLAGS: >
             -DVELOX_ENABLE_ARROW=ON
             -DVELOX_ENABLE_GEO=ON

CMake/resolve_dependency_modules/duckdb.cmake

@@ -13,10 +13,10 @@
 # limitations under the License.
 include_guard(GLOBAL)
 
-set(VELOX_DUCKDB_VERSION 0.8.1)
+set(VELOX_DUCKDB_VERSION 1.4.4)
 set(
   VELOX_DUCKDB_BUILD_SHA256_CHECKSUM
-  a0674f7e320dc7ebcf51990d7fc1c0e7f7b2c335c08f5953702b5285e6c30694
+  43645e15419c6539bae6915ba397de6569e4a7ca0d502be95d653a78fdb0bece
 )
 set(
   VELOX_DUCKDB_SOURCE_URL
@@ -35,14 +35,15 @@ FetchContent_Declare(
   URL_HASH ${VELOX_DUCKDB_BUILD_SHA256_CHECKSUM}
   PATCH_COMMAND
     git apply ${CMAKE_CURRENT_LIST_DIR}/duckdb/remove-ccache.patch && git apply
-    ${CMAKE_CURRENT_LIST_DIR}/duckdb/re2.patch
+    ${CMAKE_CURRENT_LIST_DIR}/duckdb/re2.patch && git apply
+    ${CMAKE_CURRENT_LIST_DIR}/duckdb/zstd.patch
 )
 
 # DuckDB uses git commands to retrieve version information during the build,
 # which works with git clone. To prevent incorrectly using the parent project's
 # git version when building from a tarball, we define GIT_COMMIT_HASH to skip
 # that.
-set(GIT_COMMIT_HASH "6536a77")
+set(GIT_COMMIT_HASH "6ddac80")
 set(BUILD_UNITTESTS OFF)
 set(BUILD_TESTING OFF)
 set(ENABLE_SANITIZER OFF)

CMake/resolve_dependency_modules/duckdb/remove-ccache.patch

@@ -1,19 +1,27 @@
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -32,16 +32,6 @@ set(CMAKE_VERBOSE_MAKEFILE OFF)
+@@ -37,24 +37,4 @@
  set(CMAKE_POSITION_INDEPENDENT_CODE ON)
  set(CMAKE_MACOSX_RPATH 1)
 
--find_program(CCACHE_PROGRAM ccache)
--if(CCACHE_PROGRAM)
--  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
--else()
--  find_program(CCACHE_PROGRAM sccache)
--  if(CCACHE_PROGRAM)
--    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
--  endif()
+-if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER)
+-    find_program(COMPILER_LAUNCHER NAMES ccache sccache)
+-    if(COMPILER_LAUNCHER)
+-        message(STATUS "Using ${COMPILER_LAUNCHER} as C compiler launcher")
+-        set(CMAKE_C_COMPILER_LAUNCHER
+-                "${COMPILER_LAUNCHER}"
+-                CACHE STRING "" FORCE)
+-    endif()
+-endif()
+-
+-if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER)
+-    find_program(COMPILER_LAUNCHER NAMES ccache sccache)
+-    if(COMPILER_LAUNCHER)
+-        message(STATUS "Using ${COMPILER_LAUNCHER} as C++ compiler launcher")
+-        set(CMAKE_CXX_COMPILER_LAUNCHER
+-                "${COMPILER_LAUNCHER}"
+-                CACHE STRING "" FORCE)
+-    endif()
 -endif()
 -
  # Determine install paths
- set(INSTALL_LIB_DIR
-     lib

CMake/resolve_dependency_modules/duckdb/zstd.patch

@@ -0,0 +1,11 @@
+--- a/third_party/zstd/CMakeLists.txt
++++ b/third_party/zstd/CMakeLists.txt
+@@ -40,7 +40,7 @@ add_library(duckdb_zstd STATIC ${ZSTD_FILES})
+
+ target_include_directories(
+ 		duckdb_zstd
+-		PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
++		PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>)
+ set_target_properties(duckdb_zstd PROPERTIES EXPORT_NAME duckdb_duckdb_zstd)
+
+ install(TARGETS duckdb_zstd

scripts/setup-common.sh

@@ -90,7 +90,7 @@ function install_duckdb {
     # git version when building from a tarball, we define GIT_COMMIT_HASH to skip
     # that.
     cmake_install_dir duckdb \
-      -DGIT_COMMIT_HASH="6536a77" -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF \
+      -DGIT_COMMIT_HASH="6ddac80" -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF \
       -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
   fi
 }

scripts/setup-versions.sh

@@ -29,7 +29,7 @@ FB_OS_VERSION="v2026.01.05.00"
 FMT_VERSION="11.2.0"
 BOOST_VERSION="boost-1.84.0"
 ARROW_VERSION="18.0.0"
-DUCKDB_VERSION="v0.8.1"
+DUCKDB_VERSION="v1.4.4"
 PROTOBUF_VERSION="21.8"
 XSIMD_VERSION="10.0.0"
 SIMDJSON_VERSION="4.1.0"

velox/duckdb/conversion/DuckParser.cpp

@@ -104,6 +104,8 @@ std::string duckOperatorToVelox(ExpressionType type) {
       return "in";
     case ExpressionType::OPERATOR_NOT:
       return "not";
+    case ExpressionType::OPERATOR_TRY:
+      return "try";
     default:
       return normalizeFuncName(ExpressionTypeToOperator(type));
   }
@@ -181,6 +183,19 @@ core::ExprPtr parseConstantExpr(
     value = Value::DOUBLE(value.GetValue<double>());
   }
 
+  if (value.type().id() == LogicalTypeId::INTERVAL) {
+    auto interval = value.GetValue<::duckdb::interval_t>();
+    if (interval.months != 0) {
+      int32_t totalMonths = interval.months;
+      return std::make_shared<core::ConstantExpr>(
+          INTERVAL_YEAR_MONTH(), Variant(totalMonths), getAlias(expr));
+    }
+    int64_t totalMillis =
+        interval.days * 24LL * 60 * 60 * 1'000 + interval.micros / 1'000;
+    return std::make_shared<core::ConstantExpr>(
+        INTERVAL_DAY_TIME(), Variant(totalMillis), getAlias(expr));
+  }
+
   return std::make_shared<const core::ConstantExpr>(
       toVeloxType(value.type()), duckValueToVariant(value), getAlias(expr));
 }
@@ -217,21 +232,26 @@ std::optional<int64_t> extractInteger(const core::ConstantExpr& constInput) {
 
 } // namespace
 
+std::optional<int64_t> extractIntegerRecursive(const core::IExpr* expr) {
+  if (auto constInput = dynamic_cast<const core::ConstantExpr*>(expr)) {
+    return extractInteger(*constInput);
+  }
+  if (auto castInput = dynamic_cast<const core::CastExpr*>(expr)) {
+    return extractIntegerRecursive(castInput->input().get());
+  }
+  if (auto callInput = dynamic_cast<const core::CallExpr*>(expr)) {
+    if (callInput->name() == "trunc" && callInput->inputs().size() == 1) {
+      return extractIntegerRecursive(callInput->inputs()[0].get());
+    }
+  }
+  return std::nullopt;
+}
+
 std::shared_ptr<const core::ConstantExpr> tryParseInterval(
     const std::string& functionName,
     const core::ExprPtr& input,
     std::optional<std::string> alias) {
-  std::optional<int64_t> value;
-
-  if (auto constInput = dynamic_cast<const core::ConstantExpr*>(input.get())) {
-    value = extractInteger(*constInput);
-  } else if (
-      auto castInput = dynamic_cast<const core::CastExpr*>(input.get())) {
-    if (auto constInput =
-            dynamic_cast<const core::ConstantExpr*>(castInput->input().get())) {
-      value = extractInteger(*constInput);
-    }
-  }
+  auto value = extractIntegerRecursive(input.get());
 
   if (!value.has_value()) {
     return nullptr;
@@ -825,6 +845,11 @@ parse::WindowType parseWindowType(const WindowExpression& expr) {
         boundary == WindowBoundary::EXPR_PRECEDING_ROWS) {
       return parse::WindowType::kRows;
     }
+    if (boundary == WindowBoundary::CURRENT_ROW_GROUPS ||
+        boundary == WindowBoundary::EXPR_FOLLOWING_GROUPS ||
+        boundary == WindowBoundary::EXPR_PRECEDING_GROUPS) {
+      return parse::WindowType::kRange;
+    }
     return parse::WindowType::kRange;
   };
 
@@ -839,12 +864,15 @@ parse::BoundType parseBoundType(WindowBoundary boundary) {
   switch (boundary) {
     case WindowBoundary::CURRENT_ROW_RANGE:
     case WindowBoundary::CURRENT_ROW_ROWS:
+    case WindowBoundary::CURRENT_ROW_GROUPS:
       return parse::BoundType::kCurrentRow;
     case WindowBoundary::EXPR_PRECEDING_ROWS:
     case WindowBoundary::EXPR_PRECEDING_RANGE:
+    case WindowBoundary::EXPR_PRECEDING_GROUPS:
       return parse::BoundType::kPreceding;
     case WindowBoundary::EXPR_FOLLOWING_ROWS:
     case WindowBoundary::EXPR_FOLLOWING_RANGE:
+    case WindowBoundary::EXPR_FOLLOWING_GROUPS:
       return parse::BoundType::kFollowing;
     case WindowBoundary::UNBOUNDED_FOLLOWING:
       return parse::BoundType::kUnboundedFollowing;

velox/duckdb/conversion/tests/DuckParserTest.cpp

@@ -718,8 +718,7 @@ TEST(DuckParserTest, parseScalarOrWindowExpr) {
 
 TEST(DuckParserTest, invalidExpression) {
   VELOX_ASSERT_THROW(
-      parseExpr("func(a b)"),
-      "Cannot parse expression: func(a b). Parser Error: syntax error at or near \"b\"");
+      parseExpr("func(a b)"), "Cannot parse expression: func(a b).");
 }
 
 TEST(DuckParserTest, parseDecimalConstant) {

velox/exec/fuzzer/MemoryArbitrationFuzzer.cpp

@@ -24,8 +24,8 @@
 #include "velox/common/fuzzer/Utils.h"
 #include "velox/common/testutil/TempDirectoryPath.h"
 #include "velox/connectors/hive/HiveConnector.h"
-#include "velox/dwio/dwrf/RegisterDwrfReader.h" // @manual
-#include "velox/dwio/dwrf/RegisterDwrfWriter.h" // @manual
+#include "velox/dwio/dwrf/RegisterDwrfReader.h"
+#include "velox/dwio/dwrf/RegisterDwrfWriter.h"
 #include "velox/exec/MemoryReclaimer.h"
 #include "velox/exec/fuzzer/FuzzerUtil.h"
 #include "velox/exec/tests/utils/ArbitratorTestUtil.h"