Adding e2e tests for the native Arrow Flight connector

This adds e2e tests using the ArrowFlightQueryRunner to run native workers
and test queries against a H2 Flight Producer data source. The CI workflow
has been updated to add additional steps to build a native presto server
with Arrow Flight connector enabled, and run the e2e Java tests with queries
taken from AbstractTestNativeGeneralQueries that are compatible with the
Flight data source.

Author: BryanCutler

.github/workflows/arrow-flight-tests.yml

@@ -14,7 +14,7 @@ env:
   RETRY: .github/bin/retry
 
 jobs:
-  test:
+  arrowflight-java-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -48,6 +48,161 @@ jobs:
           export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
           ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
 
-      # Run Maven tests for the target module
+      # Run Maven tests for the target module, excluding native tests
       - name: Maven Tests
-        run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }}
+        run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries"
+
+  prestocpp-linux-build-for-test:
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84
+    env:
+      CCACHE_DIR: "${{ github.workspace }}/ccache"
+      DEPENDENCY_DIR: "${{ github.workspace }}/adapter-deps/download"
+      INSTALL_PREFIX: "${{ github.workspace }}/adapter-deps/install"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Update velox
+        run: |
+          cd presto-native-execution
+          make velox-submodule
+
+      - name: Install Arrow Flight
+        run: |
+          mkdir -p ${DEPENDENCY_DIR}/adapter-deps/download
+          mkdir -p ${INSTALL_PREFIX}/adapter-deps/install
+          source /opt/rh/gcc-toolset-12/enable
+          set -xu
+          cd presto-native-execution
+          PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh arrow_flight
+
+      - name: Install Github CLI for using apache/infrastructure-actions/stash
+        run: |
+          curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm
+          rpm -iv gh_2.63.2_linux_amd64.rpm
+
+      - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-prestocpp-linux-build-for-test
+
+      - name: Zero ccache statistics
+        run: ccache -sz
+
+      - name: Build engine
+        run: |
+          source /opt/rh/gcc-toolset-12/enable
+          cd presto-native-execution
+          cmake \
+            -B _build/release \
+            -GNinja \
+            -DTREAT_WARNINGS_AS_ERRORS=1 \
+            -DENABLE_ALL_WARNINGS=1 \
+            -DVELOX_ENABLE_ARROW=OFF \
+            -DVELOX_ENABLE_PARQUET=OFF \
+            -DPRESTO_ENABLE_ARROW_FLIGHT_CONNECTOR=ON \
+            -DCMAKE_PREFIX_PATH=/usr/local \
+            -DThrift_ROOT=/usr/local \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DMAX_LINK_JOBS=4
+          ninja -C _build/release -j 4
+
+      - name: Ccache after
+        run: ccache -s
+
+      - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
+        with:
+          path: '${{ env.CCACHE_DIR }}'
+          key: ccache-prestocpp-linux-build-for-test
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: arrow-flight-presto-native-build
+          path: presto-native-execution/_build/release/presto_cpp/main/presto_server
+
+      - name: Upload Arrow Flight install artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: arrow-flight-install
+          path: ${{ env.INSTALL_PREFIX }}/lib64/libarrow_flight*
+
+  arrowflight-native-e2e-tests:
+    needs: prestocpp-linux-build-for-test
+    runs-on: ubuntu-22.04
+    container:
+      image: prestodb/presto-native-dependency:0.292-20250204112033-cf8ba84
+    env:
+      INSTALL_PREFIX: "${{ github.workspace }}/adapter-deps/install"
+    strategy:
+      fail-fast: false
+      matrix:
+        modules:
+          - ":presto-base-arrow-flight"  # Only run tests for the `presto-base-arrow-flight` module
+
+    timeout-minutes: 80
+    concurrency:
+      group: ${{ github.workflow }}-test-${{ matrix.modules }}-${{ github.event.pull_request.number }}
+      cancel-in-progress: true
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: arrow-flight-presto-native-build
+          path: presto-native-execution/_build/release/presto_cpp/main
+
+      - name: Download Arrow Flight install artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: arrow-flight-install
+          path: ${{ env.INSTALL_PREFIX }}/lib64
+
+      # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
+      - name: Restore execute permissions and library path
+        run: |
+          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
+          # Ensure transitive dependency libboost-iostreams is found.
+          ldconfig /usr/local/lib
+
+      - name: Install OpenJDK8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 8
+          cache: 'maven'
+      - name: Download nodejs to maven cache
+        run: .github/bin/download_nodejs
+
+      - name: Maven install
+        env:
+          # Use different Maven options to install.
+          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+        run: |
+          export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
+          ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl ${{ matrix.modules }}
+
+      - name: Run arrowflight native e2e tests
+        run: |
+          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
+          mvn test \
+            ${MAVEN_TEST} \
+            -pl ${{ matrix.modules }} \
+            -Dtest="TestArrowFlightNativeQueries" \
+            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
+            -DDATA_DIR=${RUNNER_TEMP} \
+            -Duser.timezone=America/Bahia_Banderas \
+            -T1C

presto-base-arrow-flight/src/test/java/com/facebook/plugin/arrow/ArrowFlightQueryRunner.java

@@ -26,9 +26,13 @@
 import org.apache.arrow.memory.RootAllocator;
 
 import java.io.File;
+import java.net.URI;
 import java.util.Map;
 import java.util.Optional;
+import java.util.function.BiFunction;
 
+import static com.facebook.plugin.arrow.testingConnector.TestingArrowFlightPlugin.ARROW_FLIGHT_CATALOG;
+import static com.facebook.plugin.arrow.testingConnector.TestingArrowFlightPlugin.ARROW_FLIGHT_CONNECTOR;
 import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
 
 public class ArrowFlightQueryRunner
@@ -40,49 +44,60 @@ private ArrowFlightQueryRunner()
 
     public static DistributedQueryRunner createQueryRunner(int flightServerPort) throws Exception
     {
-        return createQueryRunner(ImmutableMap.of("arrow-flight.server.port", String.valueOf(flightServerPort)));
+        return createQueryRunner(flightServerPort, ImmutableMap.of(), ImmutableMap.of(), Optional.empty());
     }
 
-    private static DistributedQueryRunner createQueryRunner(Map<String, String> catalogProperties) throws Exception
+    public static DistributedQueryRunner createQueryRunner(
+            int flightServerPort,
+            Map<String, String> extraProperties,
+            Map<String, String> coordinatorProperties,
+            Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
+            throws Exception
     {
-        return createQueryRunner(ImmutableMap.of(), catalogProperties);
+        return createQueryRunner(extraProperties, ImmutableMap.of("arrow-flight.server.port", String.valueOf(flightServerPort)), coordinatorProperties, externalWorkerLauncher);
     }
 
     private static DistributedQueryRunner createQueryRunner(
             Map<String, String> extraProperties,
-            Map<String, String> catalogProperties)
+            Map<String, String> catalogProperties,
+            Map<String, String> coordinatorProperties,
+            Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
             throws Exception
     {
         Session session = testSessionBuilder()
-                .setCatalog("arrowflight")
+                .setCatalog(ARROW_FLIGHT_CATALOG)
                 .setSchema("tpch")
                 .build();
 
         DistributedQueryRunner.Builder queryRunnerBuilder = DistributedQueryRunner.builder(session);
         Optional<Integer> workerCount = getProperty("WORKER_COUNT").map(Integer::parseInt);
         workerCount.ifPresent(queryRunnerBuilder::setNodeCount);
 
-        DistributedQueryRunner queryRunner = queryRunnerBuilder.setExtraProperties(extraProperties).build();
+        DistributedQueryRunner queryRunner = queryRunnerBuilder
+                .setExtraProperties(extraProperties)
+                .setCoordinatorProperties(coordinatorProperties)
+                .setExternalWorkerLauncher(externalWorkerLauncher).build();
 
         try {
-            queryRunner.installPlugin(new TestingArrowFlightPlugin());
+            boolean nativeExecution = externalWorkerLauncher.isPresent();
+            queryRunner.installPlugin(new TestingArrowFlightPlugin(nativeExecution));
 
             ImmutableMap.Builder<String, String> properties = ImmutableMap.<String, String>builder()
                     .putAll(catalogProperties)
                     .put("arrow-flight.server", "localhost")
                     .put("arrow-flight.server-ssl-enabled", "true")
                     .put("arrow-flight.server-ssl-certificate", "src/test/resources/server.crt");
 
-            queryRunner.createCatalog("arrowflight", "arrow-flight", properties.build());
+            queryRunner.createCatalog(ARROW_FLIGHT_CATALOG, ARROW_FLIGHT_CONNECTOR, properties.build());
 
             return queryRunner;
         }
         catch (Exception e) {
-            throw new RuntimeException("Failed to create ArrowQueryRunner", e);
+            throw new RuntimeException("Failed to create ArrowFlightQueryRunner", e);
         }
     }
 
-    private static Optional<String> getProperty(String name)
+    public static Optional<String> getProperty(String name)
     {
         String systemPropertyValue = System.getProperty(name);
         if (systemPropertyValue != null) {
@@ -116,7 +131,9 @@ public static void main(String[] args)
 
         DistributedQueryRunner queryRunner = createQueryRunner(
                 ImmutableMap.of("http-server.http.port", "8080"),
-                ImmutableMap.of("arrow-flight.server.port", String.valueOf(9443)));
+                ImmutableMap.of("arrow-flight.server.port", String.valueOf(9443)),
+                ImmutableMap.of(),
+                Optional.empty());
         Thread.sleep(10);
         log.info("======== SERVER STARTED ========");
         log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl());