build: Optimize CI with test splitting and 32-core runners

Split 5 monolithic test targets into 139 individual test binaries for
better ctest parallelism, and upgrade both ubuntu-debug and adapters
jobs to 32-core runners with tuned build settings.

Test target splits (CMakeLists.txt changes only):
- velox_exec_test -> 76 individual tests
- velox_exec_util_test -> 9 individual tests
- velox_aggregates_test -> 43 individual tests
- velox_cache_test -> 5 individual tests
- velox_serializer_test -> 5 individual tests

CI workflow changes (ubuntu-debug + adapters):
- Runner: 16-core/8-core -> 32-core (128 GB RAM)
- NUM_THREADS: 16/8 -> 32
- MAX_HIGH_MEM_JOBS: 4 -> 12 (128 GB / 6.8 GB per heavy TU)
- MAX_LINK_JOBS: 2/4 -> 12 (shared links use ~1 GB each)
- ctest: -j 8 -> -j 24 --timeout 900
- ulimit -n 65536 to prevent "Too many open files"
- Adapters: add -DVELOX_BUILD_SHARED=ON for faster linking

The test splits add ~135 new link targets. On the previous 8-core
runner (32 GB RAM), these would OOM during concurrent compilation
of heavy TUs (up to 6.8 GB each) + parallel linking.

Validated on 48-core test machine (throttled to 32 threads):
- Build: 26m 34s (cold, no ccache)
- Test: 17m 45s (631 tests, 625 passed)
- Total: 44m 28s vs estimated ~70 min current CI

Author: kgpai

.github/workflows/linux-build-base.yml

@@ -80,7 +80,7 @@ jobs:
     needs: get-changes
     # prevent errors when forks ff their main branch
     if: ${{ github.repository == 'facebookincubator/velox' }}
-    runs-on: 8-core-ubuntu-22.04
+    runs-on: 32-core-ubuntu
     container: ghcr.io/facebookincubator/velox-dev:adapters
     defaults:
       run:
@@ -138,7 +138,7 @@ jobs:
 
       - name: Make Release Build
         env:
-          MAKEFLAGS: NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=4
+          MAKEFLAGS: NUM_THREADS=32 MAX_HIGH_MEM_JOBS=12 MAX_LINK_JOBS=12
           CUDA_ARCHITECTURES: 70
           CUDA_COMPILER: /usr/local/cuda-${CUDA_VERSION}/bin/nvcc
           # Set compiler to GCC 14
@@ -156,6 +156,7 @@ jobs:
             "-DVELOX_ENABLE_ABFS=ON"
             "-DVELOX_ENABLE_WAVE=ON"
             "-DVELOX_MONO_LIBRARY=ON"
+            "-DVELOX_BUILD_SHARED=ON"
           )
           if [[ "${USE_CLANG}" = "true" ]]; then
              scripts/setup-centos9.sh install_clang15; export CC=/usr/bin/clang-15; export CXX=/usr/bin/clang++-15; CUDA_FLAGS="-ccbin /usr/lib64/llvm15/bin/clang++-15";
@@ -195,7 +196,8 @@ jobs:
           wget https://repo1.maven.org/maven2/org/mockito/mockito-core/2.23.4/mockito-core-2.23.4.jar -O /usr/local/hadoop/share/hadoop/mapreduce/mockito-core-2.23.4.jar
 
           export CLASSPATH=`/usr/local/hadoop/bin/hdfs classpath --glob`
-          ctest -j 8 --label-exclude cuda_driver --output-on-failure --no-tests=error
+          ulimit -n 65536
+          ctest -j 24 --timeout 900 --label-exclude cuda_driver --output-on-failure --no-tests=error
 
       # Clang-tidy needs a complete build because some files are only generated during the build
       # that clang tidy will not find and report errors otherwise.
@@ -222,7 +224,7 @@ jobs:
           python ./scripts/checks/run-clang-tidy.py -p $BUILD_PATH --commit $RANGE $FILES
 
   ubuntu-debug:
-    runs-on: 16-core-ubuntu
+    runs-on: 32-core-ubuntu
     container: ghcr.io/facebookincubator/velox-dev:ubuntu-22.04
     # prevent errors when forks ff their main branch
     if: ${{ github.repository == 'facebookincubator/velox' }}
@@ -291,7 +293,8 @@ jobs:
           # version). Without this, BUNDLED gflags cascades to BUNDLED glog
           # which conflicts with system glog loaded transitively.
           glog_SOURCE: SYSTEM
-          MAKEFLAGS: NUM_THREADS=16 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3
+          VELOX_GFLAGS_URL: file:///velox/deps-sources/gflags-v2.2.2.tar.gz
+          MAKEFLAGS: NUM_THREADS=32 MAX_HIGH_MEM_JOBS=12 MAX_LINK_JOBS=12
         run: |
           EXTRA_CMAKE_FLAGS=(
             "-DCMAKE_LINK_LIBRARIES_STRATEGY=REORDER_FREELY"
@@ -321,7 +324,8 @@ jobs:
 
       - name: Run Tests
         run: |
-          cd _build/debug && ctest -j 8 --output-on-failure --no-tests=error
+          ulimit -n 65536
+          cd _build/debug && ctest -j 24 --timeout 900 --output-on-failure --no-tests=error
 
   fedora-debug:
     runs-on: 16-core-ubuntu

velox/common/caching/tests/CMakeLists.txt

@@ -19,30 +19,38 @@ target_link_libraries(
   PRIVATE velox_common_base Folly::folly velox_time glog::glog GTest::gtest GTest::gtest_main
 )
 
-add_executable(
-  velox_cache_test
+# Split velox_cache_test into individual test binaries for parallel execution.
+set(
+  VELOX_CACHE_TEST_SOURCES
   AsyncDataCacheTest.cpp
   CacheTTLControllerTest.cpp
   SsdFileTest.cpp
   SsdFileTrackerTest.cpp
   StringIdMapTest.cpp
 )
-add_test(velox_cache_test velox_cache_test)
-target_link_libraries(
-  velox_cache_test
-  PRIVATE
-    velox_caching
-    velox_file
-    velox_file_test_utils
-    velox_memory
-    velox_test_util
-    velox_flag_definitions
-    Folly::folly
-    glog::glog
-    GTest::gtest
-    GTest::gtest_main
+
+set(
+  VELOX_CACHE_TEST_DEPS
+  velox_caching
+  velox_file
+  velox_file_test_utils
+  velox_memory
+  velox_test_util
+  velox_flag_definitions
+  Folly::folly
+  glog::glog
+  GTest::gtest
+  GTest::gtest_main
 )
 
+foreach(TEST_SOURCE IN LISTS VELOX_CACHE_TEST_SOURCES)
+  get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE)
+  set(TEST_TARGET "velox_cache_${TEST_NAME}")
+  add_executable(${TEST_TARGET} ${TEST_SOURCE})
+  add_test(${TEST_TARGET} ${TEST_TARGET})
+  target_link_libraries(${TEST_TARGET} PRIVATE ${VELOX_CACHE_TEST_DEPS})
+endforeach()
+
 add_executable(cached_factory_test CachedFactoryTest.cpp)
 add_test(cached_factory_test cached_factory_test)
 target_link_libraries(

velox/exec/tests/CMakeLists.txt

@@ -36,8 +36,9 @@ target_link_libraries(
   GTest::gtest_main
 )
 
-add_executable(
-  velox_exec_test
+# Split velox_exec_test into individual test binaries for parallel execution.
+set(
+  VELOX_EXEC_TEST_SOURCES
   AddressableNonNullValueListTest.cpp
   AggregationTest.cpp
   AggregateFunctionRegistryTest.cpp
@@ -62,7 +63,6 @@ add_executable(
   IndexLookupJoinTest.cpp
   LimitTest.cpp
   LocalPartitionTest.cpp
-  Main.cpp
   MarkDistinctTest.cpp
   EnforceDistinctTest.cpp
   MemoryReclaimerTest.cpp
@@ -117,39 +117,8 @@ add_executable(
   WriterFuzzerUtilTest.cpp
 )
 
-if(VELOX_ENABLE_GEO)
-  target_sources(velox_exec_test PRIVATE SpatialJoinTest.cpp)
-endif()
-
-add_executable(
-  velox_exec_infra_test
-  AssertQueryBuilderTest.cpp
-  DriverTest.cpp
-  FunctionSignatureBuilderTest.cpp
-  GroupedExecutionTest.cpp
-  Main.cpp
-  OperatorUtilsTest.cpp
-  PlanBuilderTest.cpp
-  PrestoQueryRunnerTest.cpp
-  QueryAssertionsTest.cpp
-  TaskTest.cpp
-  TreeOfLosersTest.cpp
-)
-
-add_test(NAME velox_exec_test COMMAND velox_exec_test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-
-# TODO: Revert back to 3000 once is fixed.
-# https://github.com/facebookincubator/velox/issues/13879
-set_tests_properties(velox_exec_test PROPERTIES TIMEOUT 6000)
-
-add_test(
-  NAME velox_exec_infra_test
-  COMMAND velox_exec_infra_test
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)
-
-target_link_libraries(
-  velox_exec_test
+set(
+  VELOX_EXEC_TEST_DEPS
   velox_aggregates
   velox_dwio_common
   velox_dwio_common_exception
@@ -191,6 +160,45 @@ target_link_libraries(
   fmt::fmt
 )
 
+foreach(TEST_SOURCE IN LISTS VELOX_EXEC_TEST_SOURCES)
+  get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE)
+  set(TEST_TARGET "velox_exec_${TEST_NAME}")
+  add_executable(${TEST_TARGET} ${TEST_SOURCE} Main.cpp)
+  add_test(NAME ${TEST_TARGET} COMMAND ${TEST_TARGET} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  target_link_libraries(${TEST_TARGET} ${VELOX_EXEC_TEST_DEPS})
+endforeach()
+
+if(VELOX_ENABLE_GEO)
+  add_executable(velox_exec_SpatialJoinTest SpatialJoinTest.cpp Main.cpp)
+  add_test(
+    NAME velox_exec_SpatialJoinTest
+    COMMAND velox_exec_SpatialJoinTest
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  )
+  target_link_libraries(velox_exec_SpatialJoinTest ${VELOX_EXEC_TEST_DEPS})
+endif()
+
+add_executable(
+  velox_exec_infra_test
+  AssertQueryBuilderTest.cpp
+  DriverTest.cpp
+  FunctionSignatureBuilderTest.cpp
+  GroupedExecutionTest.cpp
+  Main.cpp
+  OperatorUtilsTest.cpp
+  PlanBuilderTest.cpp
+  PrestoQueryRunnerTest.cpp
+  QueryAssertionsTest.cpp
+  TaskTest.cpp
+  TreeOfLosersTest.cpp
+)
+
+add_test(
+  NAME velox_exec_infra_test
+  COMMAND velox_exec_infra_test
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
 target_link_libraries(
   velox_exec_infra_test
   velox_dwio_common
@@ -228,11 +236,11 @@ target_link_libraries(
   fmt::fmt
 )
 
-add_executable(
-  velox_exec_util_test
-  Main.cpp
+# Split velox_exec_util_test into individual test binaries for parallel execution.
+# Each test depends on the shared base class PrestoQueryRunnerIntermediateTypeTransformTestBase.cpp.
+set(
+  VELOX_EXEC_UTIL_TEST_SOURCES
   PrestoQueryRunnerHyperLogLogTransformTest.cpp
-  PrestoQueryRunnerIntermediateTypeTransformTestBase.cpp
   PrestoQueryRunnerTDigestTransformTest.cpp
   PrestoQueryRunnerQDigestTransformTest.cpp
   PrestoQueryRunnerKHyperLogLogTransformTest.cpp
@@ -243,10 +251,8 @@ add_executable(
   PrestoQueryRunnerTimestampWithTimeZoneTransformTest.cpp
 )
 
-add_test(velox_exec_util_test velox_exec_util_test)
-
-target_link_libraries(
-  velox_exec_util_test
+set(
+  VELOX_EXEC_UTIL_TEST_DEPS
   velox_fuzzer_util
   velox_exec_test_lib
   velox_functions_test_lib
@@ -256,6 +262,19 @@ target_link_libraries(
   GTest::gtest_main
 )
 
+foreach(TEST_SOURCE IN LISTS VELOX_EXEC_UTIL_TEST_SOURCES)
+  get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE)
+  set(TEST_TARGET "velox_exec_${TEST_NAME}")
+  add_executable(
+    ${TEST_TARGET}
+    ${TEST_SOURCE}
+    PrestoQueryRunnerIntermediateTypeTransformTestBase.cpp
+    Main.cpp
+  )
+  add_test(NAME ${TEST_TARGET} COMMAND ${TEST_TARGET})
+  target_link_libraries(${TEST_TARGET} ${VELOX_EXEC_UTIL_TEST_DEPS})
+endforeach()
+
 add_executable(velox_in_10_min_demo VeloxIn10MinDemo.cpp)
 
 target_link_libraries(

velox/functions/prestosql/aggregates/tests/CMakeLists.txt

@@ -12,8 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-add_executable(
-  velox_aggregates_test
+# Split velox_aggregates_test into individual test binaries for parallel execution.
+set(
+  VELOX_AGGREGATES_TEST_SOURCES
   AggregationFunctionRegTest.cpp
   ApproxDistinctTest.cpp
   ApproxMostFrequentTest.cpp
@@ -34,7 +35,6 @@ add_executable(
   GeometricMeanTest.cpp
   HistogramTest.cpp
   KHyperLogLogAggregateTest.cpp
-  Main.cpp
   MapAccumulatorTest.cpp
   MapAggTest.cpp
   MapUnionAggregationTest.cpp
@@ -60,15 +60,8 @@ add_executable(
   VarianceAggregationTest.cpp
 )
 
-if(${VELOX_ENABLE_GEO})
-  target_sources(velox_aggregates_test PRIVATE GeometryAggregateTest.cpp)
-  target_link_libraries(velox_aggregates_test velox_functions_geo)
-endif()
-
-add_test(NAME velox_aggregates_test COMMAND velox_aggregates_test WORKING_DIRECTORY .)
-
-target_link_libraries(
-  velox_aggregates_test
+set(
+  VELOX_AGGREGATES_TEST_DEPS
   velox_aggregates
   velox_core
   velox_dwio_common_test_utils
@@ -90,3 +83,25 @@ target_link_libraries(
   GTest::gtest
   GTest::gtest_main
 )
+
+foreach(TEST_SOURCE IN LISTS VELOX_AGGREGATES_TEST_SOURCES)
+  get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE)
+  set(TEST_TARGET "velox_aggregates_${TEST_NAME}")
+  add_executable(${TEST_TARGET} ${TEST_SOURCE} Main.cpp)
+  add_test(NAME ${TEST_TARGET} COMMAND ${TEST_TARGET} WORKING_DIRECTORY .)
+  target_link_libraries(${TEST_TARGET} ${VELOX_AGGREGATES_TEST_DEPS})
+endforeach()
+
+if(${VELOX_ENABLE_GEO})
+  add_executable(velox_aggregates_GeometryAggregateTest GeometryAggregateTest.cpp Main.cpp)
+  add_test(
+    NAME velox_aggregates_GeometryAggregateTest
+    COMMAND velox_aggregates_GeometryAggregateTest
+    WORKING_DIRECTORY .
+  )
+  target_link_libraries(
+    velox_aggregates_GeometryAggregateTest
+    ${VELOX_AGGREGATES_TEST_DEPS}
+    velox_functions_geo
+  )
+endif()

velox/serializers/tests/CMakeLists.txt

@@ -32,19 +32,18 @@ target_link_libraries(
   GTest::gmock
 )
 
-add_executable(
-  velox_serializer_test
+# Split velox_serializer_test into individual test binaries for parallel execution.
+set(
+  VELOX_SERIALIZER_TEST_SOURCES
   CompactRowSerializerTest.cpp
   PrestoOutputStreamListenerTest.cpp
   PrestoSerializerTest.cpp
   SerializedPageFileTest.cpp
   UnsafeRowSerializerTest.cpp
 )
 
-add_test(velox_serializer_test velox_serializer_test)
-
-target_link_libraries(
-  velox_serializer_test
+set(
+  VELOX_SERIALIZER_TEST_DEPS
   velox_expression_test_utility
   velox_presto_serializer
   velox_vector_test_lib
@@ -55,6 +54,14 @@ target_link_libraries(
   glog::glog
 )
 
+foreach(TEST_SOURCE IN LISTS VELOX_SERIALIZER_TEST_SOURCES)
+  get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE)
+  set(TEST_TARGET "velox_serializer_${TEST_NAME}")
+  add_executable(${TEST_TARGET} ${TEST_SOURCE})
+  add_test(${TEST_TARGET} ${TEST_TARGET})
+  target_link_libraries(${TEST_TARGET} ${VELOX_SERIALIZER_TEST_DEPS})
+endforeach()
+
 if(VELOX_ENABLE_BENCHMARKS)
   add_executable(velox_serializer_benchmark SerializerBenchmark.cpp)