fix(build): Harden the merge to main builds (#16424)

kgpai · meta-codesync[bot] · commit 1f4d5ff44247 · 2026-03-12T15:00:40.000-07:00
Summary: The merge to main job with resolve_dependencies fails occasionaly since it tries to get all dependencies from the internet . As we are trying to make the builds more robust and green - this change sets the job to use SYSTEM dependency. We will still test the resolve dependency module out but run it via our scheduled jobs. Pull Request resolved: #16424 Reviewed By: pratikpugalia Differential Revision: D93500381 Pulled By: kgpai fbshipit-source-id: 8ac5ccba068fef01f8455dacd925a2993c481a0d
diff --git a/.github/workflows/linux-build-base.yml b/.github/workflows/linux-build-base.yml
@@ -223,9 +223,10 @@ jobs:
 
   ubuntu-debug:
     runs-on: 16-core-ubuntu
+    container: ghcr.io/facebookincubator/velox-dev:ubuntu-22.04
     # prevent errors when forks ff their main branch
     if: ${{ github.repository == 'facebookincubator/velox' }}
-    name: Ubuntu debug with resolve_dependency
+    name: Ubuntu debug with system dependencies
     env:
       CCACHE_DIR: ${{ github.workspace }}/ccache
       USE_CLANG: ${{ inputs.use-clang && 'true' || 'false' }}
@@ -234,6 +235,34 @@ jobs:
         shell: bash
         working-directory: velox
     steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 2
+          path: velox
+          persist-credentials: false
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Install Dependencies
+        env:
+          VELOX_BUILD_SHARED: "OFF"
+          VELOX_ARROW_CMAKE_PATCH: ${{ github.workspace }}/velox/CMake/resolve_dependency_modules/arrow/cmake-compatibility.patch
+        run: |
+          if git diff --name-only HEAD^1 HEAD | grep -q "scripts/setup-"; then
+            # Overwrite old setup scripts with changed versions
+            cp scripts/setup-* /
+
+            mkdir /tmp/build
+            cd /tmp/build
+
+            USE_CLANG=false bash /setup-ubuntu.sh
+
+            cd /
+            rm -rf /tmp/build # cleanup to avoid issues with disk space
+          fi
 
       - name: Get Ccache Stash
         uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
@@ -246,24 +275,23 @@ jobs:
         run: |
           mkdir -p "$CCACHE_DIR"
 
-      - uses: actions/checkout@v5
-        with:
-          path: velox
-          persist-credentials: false
-
-      - name: Install Dependencies
-        run: |
-          source scripts/setup-ubuntu.sh && install_apt_deps && install_faiss_deps
-
       - name: Clear CCache Statistics
         run: |
           ccache -sz
 
       - name: Make Debug Build
         env:
-          VELOX_DEPENDENCY_SOURCE: BUNDLED
+          VELOX_DEPENDENCY_SOURCE: SYSTEM
           ICU_SOURCE: SYSTEM
-          MAKEFLAGS: NUM_THREADS=16 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=2
+          # Use BUNDLED gflags to provide PIC static gflags for .so plugins.
+          # The container's folly is built with -DGFLAGS_SHARED=FALSE so its
+          # exported config references gflags_static which BUNDLED gflags provides.
+          gflags_SOURCE: BUNDLED
+          # Keep system glog (container's glog is built against the same gflags
+          # version). Without this, BUNDLED gflags cascades to BUNDLED glog
+          # which conflicts with system glog loaded transitively.
+          glog_SOURCE: SYSTEM
+          MAKEFLAGS: NUM_THREADS=16 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3
         run: |
           EXTRA_CMAKE_FLAGS=(
             "-DCMAKE_LINK_LIBRARIES_STRATEGY=REORDER_FREELY"
@@ -273,15 +301,11 @@ jobs:
             "-DVELOX_ENABLE_GEO=ON"
             "-DVELOX_ENABLE_PARQUET=ON"
             "-DVELOX_MONO_LIBRARY=ON"
-            "-DVELOX_BUILD_SHARED=ON"
           )
           if [[ "${USE_CLANG}" = "true" ]]; then
              export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15;
           else
-            # Faiss (link issue when using Clang) is excluded for Clang compilation
-            # and need to be added back when using GCC.
             EXTRA_CMAKE_FLAGS+=("-DVELOX_ENABLE_FAISS=ON")
-            # Investigate issues with remote function service: Issue #13897
             EXTRA_CMAKE_FLAGS+=("-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON")
           fi
           make debug
diff --git a/.github/workflows/ubuntu-bundled-deps.yml b/.github/workflows/ubuntu-bundled-deps.yml
@@ -0,0 +1,110 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tests the BUNDLED dependency resolution path on a plain Ubuntu runner
+# (no container). Triggered on PRs that change dependency scripts or
+# CMake modules, and on a nightly schedule.
+
+name: Ubuntu Bundled Dependencies
+
+on:
+  schedule:
+    - cron: 0 5 * * *
+  workflow_dispatch: {}
+  pull_request:
+    paths:
+      - scripts/setup-*.sh
+      - scripts/setup-common.sh
+      - scripts/setup-versions.sh
+      - scripts/setup-helper-functions.sh
+      - CMake/**
+      - CMakeLists.txt
+      - .github/workflows/ubuntu-bundled-deps.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.repository }}-${{ github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  ubuntu-debug-bundled-deps:
+    runs-on: 16-core-ubuntu
+    if: ${{ github.repository == 'facebookincubator/velox' }}
+    name: Ubuntu debug with bundled dependencies
+    env:
+      CCACHE_DIR: ${{ github.workspace }}/ccache
+    defaults:
+      run:
+        shell: bash
+        working-directory: velox
+    steps:
+
+      - name: Get Ccache Stash
+        uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-ubuntu-debug-bundled-deps
+
+      - name: Ensure Stash Dirs Exists
+        working-directory: ${{ github.workspace }}
+        run: |
+          mkdir -p "$CCACHE_DIR"
+
+      - uses: actions/checkout@v5
+        with:
+          path: velox
+          persist-credentials: false
+
+      - name: Install Dependencies
+        run: |
+          source scripts/setup-ubuntu.sh && install_apt_deps && install_faiss_deps
+
+      - name: Clear CCache Statistics
+        run: |
+          ccache -sz
+
+      - name: Make Debug Build
+        env:
+          VELOX_DEPENDENCY_SOURCE: BUNDLED
+          ICU_SOURCE: SYSTEM
+          MAKEFLAGS: NUM_THREADS=16 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=2
+        run: |
+          EXTRA_CMAKE_FLAGS=(
+            "-DCMAKE_LINK_LIBRARIES_STRATEGY=REORDER_FREELY"
+            "-DVELOX_ENABLE_BENCHMARKS=ON"
+            "-DVELOX_ENABLE_EXAMPLES=ON"
+            "-DVELOX_ENABLE_ARROW=ON"
+            "-DVELOX_ENABLE_GEO=ON"
+            "-DVELOX_ENABLE_PARQUET=ON"
+            "-DVELOX_MONO_LIBRARY=ON"
+            "-DVELOX_BUILD_SHARED=ON"
+            "-DVELOX_ENABLE_FAISS=ON"
+            "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
+          )
+          make debug
+
+      - name: CCache after
+        run: |
+          ccache -vs
+
+      - uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: ccache-ubuntu-debug-bundled-deps
+
+      - name: Run Tests
+        run: |
+          cd _build/debug && ctest -j 8 --output-on-failure --no-tests=error
diff --git a/CMake/Findc-ares.cmake b/CMake/Findc-ares.cmake
@@ -20,7 +20,7 @@ if(c-ares_FOUND)
 endif()
 
 find_path(C_ARES_INCLUDE_DIR NAMES ares.h PATH_SUFFIXES c-ares)
-find_library(C_ARES_LIBRARY NAMES c-ares)
+find_library(C_ARES_LIBRARY NAMES c-ares cares)
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(c-ares DEFAULT_MSG C_ARES_LIBRARY C_ARES_INCLUDE_DIR)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -574,9 +574,13 @@ if(NOT TARGET gflags::gflags)
 endif()
 
 if(${gflags_SOURCE} STREQUAL "BUNDLED")
-  # we force glog from source to avoid issues with a system version built
-  # against another gflags version (which is likely)
-  set(glog_SOURCE BUNDLED)
+  # Allow explicit glog_SOURCE override (e.g. when system glog is built
+  # against the same gflags version as BUNDLED).
+  if(DEFINED ENV{glog_SOURCE})
+    set(glog_SOURCE $ENV{glog_SOURCE})
+  else()
+    set(glog_SOURCE BUNDLED)
+  endif()
 else()
   set(glog_SOURCE SYSTEM)
 endif()
diff --git a/scripts/docker/centos-multi.dockerfile b/scripts/docker/centos-multi.dockerfile
@@ -47,6 +47,11 @@ ENV UV_TOOL_BIN_DIR=/usr/local/bin \
 ENV CMAKE_POLICY_VERSION_MINIMUM="3.5" \
     VELOX_ARROW_CMAKE_PATCH=/cmake-compatibility.patch
 
+# Ensure libraries installed to INSTALL_PREFIX are found at runtime (e.g.
+# thrift1 needs libgflags.so.2.2 when folly links gflags statically but
+# other tools still use shared gflags).
+ENV LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${INSTALL_PREFIX}/lib64"
+
 # Some CMake configs contain the hard coded prefix '/deps', we need to replace that with
 # the future location to avoid build errors in the base-image
 RUN bash /setup-centos9.sh && \
diff --git a/scripts/setup-common.sh b/scripts/setup-common.sh
@@ -47,7 +47,14 @@ function install_fmt {
 
 function install_folly {
   wget_and_untar https://github.com/facebook/folly/archive/refs/tags/"${FB_OS_VERSION}".tar.gz folly
-  cmake_install_dir folly -DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
+  local FOLLY_FLAGS=(-DBUILD_SHARED_LIBS="$VELOX_BUILD_SHARED" -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON)
+  # When folly is static, use static gflags to avoid dual gflags flag
+  # registration when .so plugins are dlopen'd (both the binary and plugin
+  # would register the same flags in a shared gflags registry).
+  if [[ ${VELOX_BUILD_SHARED} != "ON" ]]; then
+    FOLLY_FLAGS+=(-DGFLAGS_SHARED=FALSE)
+  fi
+  cmake_install_dir folly "${FOLLY_FLAGS[@]}"
 }
 
 function install_fizz {
