From 4f23c7bc2e33f51eb42f5b783677bdd6cedd785c Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Fri, 4 Jul 2025 10:56:26 -0400 Subject: [PATCH 01/22] Init --- .../actions/yscope-image-build/action.yaml | 66 +++++++++++++++++++ ...resto-coordinator-runtime-image-build.yaml | 42 ++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 .github/actions/yscope-image-build/action.yaml create mode 100644 .github/workflows/presto-coordinator-runtime-image-build.yaml diff --git a/.github/actions/yscope-image-build/action.yaml b/.github/actions/yscope-image-build/action.yaml new file mode 100644 index 0000000000000..a05b9499ac725 --- /dev/null +++ b/.github/actions/yscope-image-build/action.yaml @@ -0,0 +1,66 @@ +name: "yscope-image-build" + +inputs: + dockerfile_path: + description: "Dockerfile relative path from the root of the project" + required: true + image_name_prefix: + description: "Container image name prefix" + required: true + image_registry: + default: "ghcr.io" + description: "Container image registry" + required: false + image_registry_username: + default: "${{github.actor}}" + description: "Container image registry username" + required: false + image_registry_password: + default: "" + description: "Container image registry password" + required: false + platform_id: + description: "Platform ID of the container (e.g. ubuntu)" + required: true + platform_version_id: + description: "Platform VERSION_ID / VERSION_CODENAME of the container + (e.g. jammy, focal, etc.)" + required: true + +runs: + using: "composite" + steps: + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" + + - name: "Login to Image Registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "${{inputs.image_registry}}" + username: "${{inputs.image_registry_username}}" + password: "${{inputs.image_registry_password}}" + + - name: "Sanitize Repository Name" + id: "sanitization" + shell: "bash" + run: | + # Docker doesn't support repository names with uppercase characters, so we convert the + # name to lowercase here. + echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ + >> "$GITHUB_OUTPUT" + + - name: "Update Metadata" + id: "meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ + /${{inputs.image_name_prefix}}-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" + + - name: "Build and Push" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "./" + file: ${{dockerfile_path}} + push: true + tags: "${{steps.meta.outputs.tags}}" + labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/presto-coordinator-runtime-image-build.yaml b/.github/workflows/presto-coordinator-runtime-image-build.yaml new file mode 100644 index 0000000000000..a872acd67d132 --- /dev/null +++ b/.github/workflows/presto-coordinator-runtime-image-build.yaml @@ -0,0 +1,42 @@ +name: "presto-coordinator-runtime-image-build" + +on: + pull_request: + paths: + - ".github/workflows/presto-coordinator-runtime-image-build.yaml" + - "docker/**/*" + push: + paths: + - ".github/workflows/presto-coordinator-runtime-image-build.yaml" + - "docker/**/*" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so + # that each commit to release-0.293-clp-connector is checked completely. + cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" + +jobs: + presto-coordinator-runtime-image: + name: presto-coordinator-runtime-image + runs-on: ubuntu-22.04 + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/yscope-image-build" + with: + dockerfile_path: "./docker/Dockerfile" + image_name_prefix: "yscope-presto-with-clp-connector-coordinator" + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "centos" + platform_version_id: "stream9" From 48dff1382792f9312c64137e880f748b218e8130 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Fri, 4 Jul 2025 11:01:21 -0400 Subject: [PATCH 02/22] Temporaryily remove all other workflows for faster testing --- .../actions/yscope-image-build/action.yaml | 2 +- .github/workflows/docs.yml | 54 --- .github/workflows/maven-checks.yml | 50 --- .../prestocpp-format-and-header-check.yml | 44 --- .../prestocpp-linux-build-and-unit-test.yml | 314 ------------------ .github/workflows/tests.yml | 84 ----- 6 files changed, 1 insertion(+), 547 deletions(-) delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/maven-checks.yml delete mode 100644 .github/workflows/prestocpp-format-and-header-check.yml delete mode 100644 .github/workflows/prestocpp-linux-build-and-unit-test.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/actions/yscope-image-build/action.yaml b/.github/actions/yscope-image-build/action.yaml index a05b9499ac725..f4c98a6ef892e 100644 --- a/.github/actions/yscope-image-build/action.yaml +++ b/.github/actions/yscope-image-build/action.yaml @@ -60,7 +60,7 @@ runs: uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" - file: ${{dockerfile_path}} + file: ${{inputs.dockerfile_path}} push: true tags: "${{steps.meta.outputs.tags}}" labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 46565491c504b..0000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: docs - -on: - pull_request: - paths: - - 'presto-docs/**' - push: - paths: - - 'presto-docs/**' - -env: - # An envar that signals to tests we are executing in the CI environment - CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" - RETRY: .github/bin/retry - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so - # that each commit to release-0.293-clp-connector is checked completely. - cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - modules: - - ":presto-docs" - timeout-minutes: 80 - steps: - - uses: actions/checkout@v4 - with: - show-progress: false - - name: Install LaTeX dependencies - run: | - sudo apt-get update - sudo apt-get install -y texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra latexmk tex-gyre texlive-xetex fonts-freefont-otf xindy - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 8.0.442 - cache: 'maven' - - name: Maven Install - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) - - name: Maven Tests - run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml deleted file mode 100644 index e2df97491d862..0000000000000 --- a/.github/workflows/maven-checks.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: maven checks - -on: - pull_request: - push: - -env: - # An envar that signals to tests we are executing in the CI environment - CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - RETRY: .github/bin/retry - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so - # that each commit to release-0.293-clp-connector is checked completely. - cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" - -jobs: - maven-checks: - strategy: - fail-fast: false - matrix: - java: [ 8.0.442, 17.0.13 ] - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - name: Free Disk Space - run: | - df -h - sudo apt-get clean - df -h - - uses: actions/checkout@v4 - with: - show-progress: false - - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - cache: 'maven' - - name: Download nodejs to maven cache - run: .github/bin/download_nodejs - - name: Maven Checks - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - ./mvnw install -B -V -T 1C -DskipTests -Dmaven.javadoc.skip=true --no-transfer-progress -P ci -pl '!presto-test-coverage,!:presto-docs' - - name: Clean Maven Output - run: ./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage' diff --git a/.github/workflows/prestocpp-format-and-header-check.yml b/.github/workflows/prestocpp-format-and-header-check.yml deleted file mode 100644 index c554ee8785786..0000000000000 --- a/.github/workflows/prestocpp-format-and-header-check.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: prestocpp-format-and-header-check - -on: - workflow_dispatch: - pull_request: - paths: - - 'presto-native-execution/**' - - '.github/workflows/prestocpp-format-and-header-check.yml' - push: - paths: - - 'presto-native-execution/**' - - '.github/workflows/prestocpp-format-and-header-check.yml' - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so - # that each commit to release-0.293-clp-connector is checked completely. - cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" - -jobs: - prestocpp-format-and-header-check: - runs-on: ubuntu-latest - container: - image: public.ecr.aws/oss-presto/velox-dev:check - steps: - - uses: actions/checkout@v4 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Check formatting - run: | - git fetch origin master - cd presto-native-execution - make format-check - - - name: Check license headers - run: | - git fetch origin master - cd presto-native-execution - make header-check diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml deleted file mode 100644 index e26e330403ec2..0000000000000 --- a/.github/workflows/prestocpp-linux-build-and-unit-test.yml +++ /dev/null @@ -1,314 +0,0 @@ -name: prestocpp-linux-build-and-unit-test - -on: - workflow_dispatch: - pull_request: - paths: - - 'presto-native-execution/**' - - 'presto-native-sidecar-plugin/**' - - '.github/workflows/prestocpp-linux-build-and-unit-test.yml' - push: - paths-ignore: - - 'presto-docs/**' - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so - # that each commit to release-0.293-clp-connector is checked completely. - cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" - -jobs: - prestocpp-linux-build-for-test: - runs-on: ubuntu-22.04 - container: - image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e - env: - CCACHE_DIR: "${{ github.workspace }}/ccache" - steps: - - uses: actions/checkout@v4 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Update velox - run: | - cd presto-native-execution - make velox-submodule - - - name: Install Github CLI for using apache/infrastructure-actions/stash - run: | - curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm - rpm -iv gh_2.63.2_linux_amd64.rpm - - - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-prestocpp-linux-build-for-test - - - name: Zero ccache statistics - run: ccache -sz - - - name: Build engine - run: | - source /opt/rh/gcc-toolset-12/enable - cd presto-native-execution - cmake \ - -B _build/release \ - -GNinja \ - -DTREAT_WARNINGS_AS_ERRORS=1 \ - -DENABLE_ALL_WARNINGS=1 \ - -DCMAKE_BUILD_TYPE=Release \ - -DPRESTO_ENABLE_PARQUET=ON \ - -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ - -DPRESTO_ENABLE_JWT=ON \ - -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ - -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DThrift_ROOT=/usr/local \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DMAX_LINK_JOBS=4 - ninja -C _build/release -j $(getconf _NPROCESSORS_ONLN) - - - name: Ccache after - run: ccache -s - - - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 - with: - path: '${{ env.CCACHE_DIR }}' - key: ccache-prestocpp-linux-build-for-test - - - name: Run Unit Tests - run: | - # Ensure transitive dependency libboost-iostreams is found. - ldconfig /usr/local/lib - cd presto-native-execution/_build/release - ctest -j $(getconf _NPROCESSORS_ONLN) -VV --output-on-failure --exclude-regex velox.* - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: presto-native-build - path: | - presto-native-execution/_build/release/presto_cpp/main/presto_server - presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main - - prestocpp-linux-presto-e2e-tests: - needs: prestocpp-linux-build-for-test - runs-on: ubuntu-22.04 - container: - image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e - env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" - steps: - - uses: actions/checkout@v4 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: presto-native-build - path: presto-native-execution/_build/release - - # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 - - name: Restore execute permissions and library path - run: | - chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server - chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main - # Ensure transitive dependency libboost-iostreams is found. - ldconfig /usr/local/lib - - - name: Install OpenJDK8 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 8.0.442 - cache: 'maven' - - name: Download nodejs to maven cache - run: .github/bin/download_nodejs - - - name: Maven install - env: - # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - run: | - for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) - - - name: Run presto-native e2e tests - run: | - export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" - export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'` - # Convert file paths to comma separated class names - export TESTCLASSES= - for test_file in $TESTFILES - do - tmp=${test_file##*/} - test_class=${tmp%%\.*} - export TESTCLASSES="${TESTCLASSES},$test_class" - done - export TESTCLASSES=${TESTCLASSES#,} - echo "TESTCLASSES = $TESTCLASSES" - # TODO: neeed to enable remote function tests with - # "-Ppresto-native-execution-remote-functions" once - # > https://github.com/facebookincubator/velox/discussions/6163 - # is fixed. - - mvn test \ - ${MAVEN_TEST} \ - -pl 'presto-native-execution' \ - -Dtest="${TESTCLASSES}" \ - -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ - -DDATA_DIR=${RUNNER_TEMP} \ - -Duser.timezone=America/Bahia_Banderas \ - -T1C - - prestocpp-linux-presto-native-tests: - needs: prestocpp-linux-build-for-test - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - matrix: - storage-format: [ "PARQUET", "DWRF" ] - container: - image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e - env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" - steps: - - uses: actions/checkout@v4 - - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: presto-native-build - path: presto-native-execution/_build/release - - # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 - - name: Restore execute permissions and library path - run: | - chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server - chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main - # Ensure transitive dependency libboost-iostreams is found. - ldconfig /usr/local/lib - - - name: Install OpenJDK8 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '8.0.442' - cache: 'maven' - - name: Download nodejs to maven cache - run: .github/bin/download_nodejs - - - name: Maven install - env: - # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - run: | - for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) - - - name: Run presto-native tests - run: | - export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" - export TESTFILES=`find ./presto-native-tests/src/test -type f -name 'Test*.java'` - # Convert file paths to comma separated class names - export TESTCLASSES= - for test_file in $TESTFILES - do - tmp=${test_file##*/} - test_class=${tmp%%\.*} - export TESTCLASSES="${TESTCLASSES},$test_class" - done - export TESTCLASSES=${TESTCLASSES#,} - echo "TESTCLASSES = $TESTCLASSES" - - mvn test \ - ${MAVEN_TEST} \ - -pl 'presto-native-tests' \ - -DstorageFormat=${{ matrix.storage-format }} \ - -Dtest="${TESTCLASSES}" \ - -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ - -DDATA_DIR=${RUNNER_TEMP} \ - -Duser.timezone=America/Bahia_Banderas \ - -T1C - - prestocpp-linux-presto-sidecar-tests: - needs: prestocpp-linux-build-for-test - runs-on: ubuntu-22.04 - container: - image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e - env: - MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" - steps: - - uses: actions/checkout@v4 - - name: Fix git permissions - # Usually actions/checkout does this but as we run in a container - # it doesn't work - run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - name: presto-native-build - path: presto-native-execution/_build/release - - # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 - - name: Restore execute permissions and library path - run: | - chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server - chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main - # Ensure transitive dependency libboost-iostreams is found. - ldconfig /usr/local/lib - - name: Install OpenJDK8 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '8.0.442' - cache: 'maven' - - name: Download nodejs to maven cache - run: .github/bin/download_nodejs - - - name: Maven install - env: - # Use different Maven options to install. - MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - run: | - for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) - - name: Run presto-native sidecar tests - run: | - export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" - export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'` - # Convert file paths to comma separated class names - export TESTCLASSES= - for test_file in $TESTFILES - do - tmp=${test_file##*/} - test_class=${tmp%%\.*} - export TESTCLASSES="${TESTCLASSES},$test_class" - done - export TESTCLASSES=${TESTCLASSES#,} - echo "TESTCLASSES = $TESTCLASSES" - mvn test \ - ${MAVEN_TEST} \ - -pl 'presto-native-sidecar-plugin' \ - -Dtest="${TESTCLASSES}" \ - -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ - -DDATA_DIR=${RUNNER_TEMP} \ - -Duser.timezone=America/Bahia_Banderas \ - -T1C diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 6f829502fbbb8..0000000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: test - -on: - pull_request: - push: - -env: - # An envar that signals to tests we are executing in the CI environment - CONTINUOUS_INTEGRATION: true - MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" - MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" - MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" - RETRY: .github/bin/retry - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so - # that each commit to release-0.293-clp-connector is checked completely. - cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" - -jobs: - changes: - runs-on: ubuntu-latest - # Required permissions - permissions: - pull-requests: read - # Set job outputs to values from filter step - outputs: - codechange: ${{ steps.filter.outputs.codechange }} - steps: - - uses: "actions/checkout@v4" - with: - submodules: "recursive" - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - codechange: - - '!presto-docs/**' - - test: - runs-on: ubuntu-latest - needs: changes - strategy: - fail-fast: false - matrix: - java: [8.0.442, 17.0.13] - modules: - - ":presto-tests -P presto-tests-execution-memory" - - ":presto-tests -P presto-tests-general" - - ":presto-tests -P ci-only-distributed-non-hash-gen" - - ":presto-tests -P ci-only-tpch-distributed-queries" - - ":presto-tests -P ci-only-local-queries" - - ":presto-tests -P ci-only-distributed-queries" - - ":presto-tests -P ci-only-aggregation-queries" - - ":presto-tests -P ci-only-plan-determinism" - - ":presto-tests -P ci-only-resource-manager" - - ":presto-main-base" - - ":presto-main" - timeout-minutes: 80 - steps: - - uses: actions/checkout@v4 - if: needs.changes.outputs.codechange == 'true' - with: - show-progress: false - - uses: actions/setup-java@v4 - if: needs.changes.outputs.codechange == 'true' - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - cache: 'maven' - - name: Download nodejs to maven cache - if: needs.changes.outputs.codechange == 'true' - run: .github/bin/download_nodejs - - name: Maven Install - if: needs.changes.outputs.codechange == 'true' - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) - - name: Maven Tests - if: needs.changes.outputs.codechange == 'true' - run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} From 9bd9207a5bdb4e8867cd8363657723d1de052a73 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Fri, 4 Jul 2025 11:04:52 -0400 Subject: [PATCH 03/22] Try fix --- .../actions/yscope-image-build/action.yaml | 14 ++- .github/workflows/maven-checks.yaml | 107 ++++++++++++++++++ ...resto-coordinator-runtime-image-build.yaml | 42 ------- docker/build.sh | 23 ++-- .../presto/plugin/clp/TestClpQueryBase.java | 2 +- 5 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/maven-checks.yaml delete mode 100644 .github/workflows/presto-coordinator-runtime-image-build.yaml diff --git a/.github/actions/yscope-image-build/action.yaml b/.github/actions/yscope-image-build/action.yaml index f4c98a6ef892e..cad718ba47ea3 100644 --- a/.github/actions/yscope-image-build/action.yaml +++ b/.github/actions/yscope-image-build/action.yaml @@ -1,6 +1,15 @@ name: "yscope-image-build" inputs: + build_args: + default: | + PRESTO_VERSION=0.293 + description: "The build args for building the image" + required: false + context_dir: + default: "./" + description: "Which directory we are in when building the image" + required: false dockerfile_path: description: "Dockerfile relative path from the root of the project" required: true @@ -59,8 +68,11 @@ runs: - name: "Build and Push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: - context: "./" + context: ${{inputs.context_dir}} file: ${{inputs.dockerfile_path}} push: true tags: "${{steps.meta.outputs.tags}}" labels: "${{steps.meta.outputs.labels}}" + build-args: | + ${{inputs.build_args}} + JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0 diff --git a/.github/workflows/maven-checks.yaml b/.github/workflows/maven-checks.yaml new file mode 100644 index 0000000000000..4c0649b526522 --- /dev/null +++ b/.github/workflows/maven-checks.yaml @@ -0,0 +1,107 @@ +name: maven checks + +on: + pull_request: + push: + +env: + # An envar that signals to tests we are executing in the CI environment + CONTINUOUS_INTEGRATION: true + MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + RETRY: .github/bin/retry + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so + # that each commit to release-0.293-clp-connector is checked completely. + cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" + +jobs: + maven-checks: + strategy: + fail-fast: false + matrix: + java: [ 8.0.442, 17.0.13 ] + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Free Disk Space + run: | + df -h + sudo apt-get clean + df -h + - uses: actions/checkout@v4 + with: + show-progress: false + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + cache: 'maven' + - name: Download nodejs to maven cache + run: .github/bin/download_nodejs + - name: Maven Checks + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + ./mvnw install -B -V -T 1C -DskipTests -Dmaven.javadoc.skip=true --no-transfer-progress -P ci -pl '!presto-test-coverage,!:presto-docs' + - name: Upload presto-server tarball + if: matrix.java == '8.0.442' + uses: actions/upload-artifact@v4 + with: + name: presto-server + path: presto-server/target/presto-server-0.293.tar.gz + - name: Upload presto-cli executable + if: matrix.java == '8.0.442' + uses: actions/upload-artifact@v4 + with: + name: presto-cli + path: presto-cli/target/presto-cli-0.293-executable.jar + - name: Clean Maven Output + run: ./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage' + + presto-coordinator-runtime-image: + name: presto-coordinator-runtime-image + needs: maven-checks + runs-on: ubuntu-22.04 + if: ${{ always() && success() }} + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Login to Image Registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + + - name: Download presto-server + uses: actions/download-artifact@v4 + with: + name: presto-server + path: /tmp + + - name: Download presto-cli + uses: actions/download-artifact@v4 + with: + name: presto-cli + path: /tmp + + - name: "Run build.sh" + working-directory: ./docker + run: | + mv /tmp/presto-server-0.293.tar.gz ./ + mv /tmp/presto-cli-0.293-executable.jar ./ + + docker buildx create --name container --driver docker-container --use + + IMAGE_NAME=yscope-presto-with-clp-connector-coordinator \ + REG_ORG=ghcr.io/${{github.actor}} \ + PUBLISH=true \ + ./build.sh 0.293 + + rm presto-server-0.293.tar.gz + rm presto-cli-0.293-executable.jar diff --git a/.github/workflows/presto-coordinator-runtime-image-build.yaml b/.github/workflows/presto-coordinator-runtime-image-build.yaml deleted file mode 100644 index a872acd67d132..0000000000000 --- a/.github/workflows/presto-coordinator-runtime-image-build.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "presto-coordinator-runtime-image-build" - -on: - pull_request: - paths: - - ".github/workflows/presto-coordinator-runtime-image-build.yaml" - - "docker/**/*" - push: - paths: - - ".github/workflows/presto-coordinator-runtime-image-build.yaml" - - "docker/**/*" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so - # that each commit to release-0.293-clp-connector is checked completely. - cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" - -jobs: - presto-coordinator-runtime-image: - name: presto-coordinator-runtime-image - runs-on: ubuntu-22.04 - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/yscope-image-build" - with: - dockerfile_path: "./docker/Dockerfile" - image_name_prefix: "yscope-presto-with-clp-connector-coordinator" - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "centos" - platform_version_id: "stream9" diff --git a/docker/build.sh b/docker/build.sh index f561b5edc76da..2edd7646eec19 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -17,22 +17,19 @@ VERSION=$1; shift TAG="${TAG:-latest}" IMAGE_NAME="${IMAGE_NAME:-presto}" REG_ORG="${REG_ORG:-docker.io/prestodb}" -PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64,linux/ppc64le}" -TMP_IIDFILE=$(mktemp) -IIDFILE="${IIDFILE:-$TMP_IIDFILE}" PUBLISH="${PUBLISH:-false}" -BUILDER="${BUILDER:-container}" -export BUILDX_NO_DEFAULT_ATTESTATIONS="" -# If PUBLISH=false, images only stores in local cache, otherwise they are pushed to th container registry -docker buildx build --builder="${BUILDER}" --iidfile "${IIDFILE}" \ +# Compose full image name with tag +FULL_IMAGE_NAME="${REG_ORG}/${IMAGE_NAME}:${TAG}" + +# Build image using regular docker build +docker build \ --build-arg="PRESTO_VERSION=${VERSION}" \ --build-arg="JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0" \ - --output "type=image,name=${REG_ORG}/${IMAGE_NAME},push-by-digest=true,name-canonical=true,push=${PUBLISH}" \ - --platform "${PLATFORMS}" -f Dockerfile . + -t "${FULL_IMAGE_NAME}" \ + -f Dockerfile . -if [[ "$PUBLISH" = "true" ]]; then - # This only happens when push=true, since push-by-digest=true in the above build step, need to tag the images explicitly - docker buildx imagetools create --builder="${BUILDER}" \ - -t "${REG_ORG}/${IMAGE_NAME}:${TAG}" "${REG_ORG}/${IMAGE_NAME}@$(cat "$IIDFILE")" +# Optionally push the image +if [[ "$PUBLISH" == "true" ]]; then + docker push "${FULL_IMAGE_NAME}" fi diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java index 1b759ab3e1282..e476c439eaa1f 100644 --- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java +++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java @@ -75,7 +75,7 @@ public class TestClpQueryBase new AnalyzePropertyManager(), createTestTransactionManager(new CatalogManager())); - protected static final ClpTableHandle table = new ClpTableHandle(new SchemaTableName("default", "test"), "", ClpTableHandle.StorageType.FS); + protected static final ClpTableHandle table = new ClpTableHandle(new SchemaTableName("default", "test"), ""); protected static final ClpColumnHandle city = new ClpColumnHandle( "city", RowType.from(ImmutableList.of( From d74b8856d241b5fc635c364aa2a95c2f495bb8cf Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 7 Jul 2025 13:17:01 -0400 Subject: [PATCH 04/22] Fix --- .github/workflows/maven-checks.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maven-checks.yaml b/.github/workflows/maven-checks.yaml index 4c0649b526522..33b381356b7bb 100644 --- a/.github/workflows/maven-checks.yaml +++ b/.github/workflows/maven-checks.yaml @@ -65,7 +65,13 @@ jobs: name: presto-coordinator-runtime-image needs: maven-checks runs-on: ubuntu-22.04 - if: ${{ always() && success() }} + if: ${{ + always() && + success() && + github.event_name == 'push' && + github.repository == 'y-scope/presto' && + github.ref == 'refs/heads/release-0.293-clp-connector' + }} steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: @@ -98,8 +104,8 @@ jobs: docker buildx create --name container --driver docker-container --use - IMAGE_NAME=yscope-presto-with-clp-connector-coordinator \ - REG_ORG=ghcr.io/${{github.actor}} \ + IMAGE_NAME=coordinator-with-clp-connector \ + REG_ORG=ghcr.io/${{github.actor}}/presto \ PUBLISH=true \ ./build.sh 0.293 From 481535111370f6994d90382a6b11b77db0962f0d Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 7 Jul 2025 15:31:31 -0400 Subject: [PATCH 05/22] Fix --- .github/workflows/maven-checks.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/maven-checks.yaml b/.github/workflows/maven-checks.yaml index 33b381356b7bb..5cb18cdaea349 100644 --- a/.github/workflows/maven-checks.yaml +++ b/.github/workflows/maven-checks.yaml @@ -65,13 +65,7 @@ jobs: name: presto-coordinator-runtime-image needs: maven-checks runs-on: ubuntu-22.04 - if: ${{ - always() && - success() && - github.event_name == 'push' && - github.repository == 'y-scope/presto' && - github.ref == 'refs/heads/release-0.293-clp-connector' - }} + if: ${{ always() && success() && github.event_name == 'push' && github.repository == 'y-scope/presto' && github.ref == 'refs/heads/release-0.293-clp-connector' }} steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: From e7fe20c9e57bdc1bb2f16b63d1be7457784410b4 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 7 Jul 2025 17:04:49 -0400 Subject: [PATCH 06/22] Still build the runtime image when updating the PR, but only publish the image when pushing to our main branch --- .github/workflows/maven-checks.yaml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven-checks.yaml b/.github/workflows/maven-checks.yaml index 5cb18cdaea349..306d2c1fc22ad 100644 --- a/.github/workflows/maven-checks.yaml +++ b/.github/workflows/maven-checks.yaml @@ -61,11 +61,11 @@ jobs: - name: Clean Maven Output run: ./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage' - presto-coordinator-runtime-image: + presto-coordinator-with-clp-connector-runtime-image: name: presto-coordinator-runtime-image needs: maven-checks runs-on: ubuntu-22.04 - if: ${{ always() && success() && github.event_name == 'push' && github.repository == 'y-scope/presto' && github.ref == 'refs/heads/release-0.293-clp-connector' }} + if: ${{ always() && success() }} steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: @@ -96,11 +96,21 @@ jobs: mv /tmp/presto-server-0.293.tar.gz ./ mv /tmp/presto-cli-0.293-executable.jar ./ - docker buildx create --name container --driver docker-container --use + if [ "${{ github.event_name }}" = "push" ] && \ + [ "${{ github.repository }}" = "y-scope/presto" ] && \ + [ "${{ github.ref }}" = "refs/heads/release-0.293-clp-connector" ]; then + export PUBLISH=true + else + echo "github.event_name = ${{ github.event_name }}" + echo "github.repository = ${{ github.repository }}" + echo "github.ref = ${{ github.ref }}" + echo "Skip publish the image" + export PUBLISH=false + fi - IMAGE_NAME=coordinator-with-clp-connector \ + IMAGE_NAME=coordinator-with-clp-connector-runtime \ REG_ORG=ghcr.io/${{github.actor}}/presto \ - PUBLISH=true \ + PUBLISH=$PUBLISH \ ./build.sh 0.293 rm presto-server-0.293.tar.gz From fc2cc1872631ba6cbde3aabbeb7ab50905359f1b Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 7 Jul 2025 21:15:46 -0400 Subject: [PATCH 07/22] Build worker image --- ...with-clp-connector-runtime-image-build.yml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml new file mode 100644 index 0000000000000..b768862252da6 --- /dev/null +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -0,0 +1,40 @@ +name: prestocpp-worker-with-clp-connector-runtime-image-build + +on: + # TODO: specifiy the branch to the release-0.293 when finalize the PR + pull_request: + push: + +jobs: + prestocpp-worker-with-clp-connector-runtime-image: + name: prestocpp-worker-with-clp-connector-runtime-image + runs-on: ubuntu-22.04 + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Login to Image Registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + + - name: "Build worker image" + working-directory: ./presto-native-execution + run: | + sed -i 's|image: presto/prestissimo-runtime:ubuntu-22.04|image: ghcr.io/${{github.actor}}/presto/prestissimo-with-clp-connector-runtime:ubuntu-22.04|' docker-compose.yml + docker compose build ubuntu-native-dependency + docker compose build ubuntu-native-runtime + + if [ "${{ github.event_name }}" = "push" ] && \ + [ "${{ github.repository }}" = "y-scope/presto" ] && \ + [ "${{ github.ref }}" = "refs/heads/release-0.293-clp-connector" ]; then + docker push "ghcr.io/${{github.actor}}/presto/prestissimo-with-clp-connector-runtime:ubuntu-22.04" + else + echo "github.event_name = ${{ github.event_name }}" + echo "github.repository = ${{ github.repository }}" + echo "github.ref = ${{ github.ref }}" + echo "Skip publish the image" + fi From 60dd19d9ea6d289cb91ac9a3b34a6c4c1eeca6b2 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Wed, 9 Jul 2025 10:15:11 -0400 Subject: [PATCH 08/22] Restore workflows --- .github/workflows/docs.yml | 54 +++ .../{maven-checks.yaml => maven-checks.yml} | 0 .../prestocpp-format-and-header-check.yml | 44 +++ .../prestocpp-linux-build-and-unit-test.yml | 314 ++++++++++++++++++ .github/workflows/tests.yml | 84 +++++ 5 files changed, 496 insertions(+) create mode 100644 .github/workflows/docs.yml rename .github/workflows/{maven-checks.yaml => maven-checks.yml} (100%) create mode 100644 .github/workflows/prestocpp-format-and-header-check.yml create mode 100644 .github/workflows/prestocpp-linux-build-and-unit-test.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000000..46565491c504b --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,54 @@ +name: docs + +on: + pull_request: + paths: + - 'presto-docs/**' + push: + paths: + - 'presto-docs/**' + +env: + # An envar that signals to tests we are executing in the CI environment + CONTINUOUS_INTEGRATION: true + MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + RETRY: .github/bin/retry + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so + # that each commit to release-0.293-clp-connector is checked completely. + cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + modules: + - ":presto-docs" + timeout-minutes: 80 + steps: + - uses: actions/checkout@v4 + with: + show-progress: false + - name: Install LaTeX dependencies + run: | + sudo apt-get update + sudo apt-get install -y texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra latexmk tex-gyre texlive-xetex fonts-freefont-otf xindy + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 8.0.442 + cache: 'maven' + - name: Maven Install + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) + - name: Maven Tests + run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} diff --git a/.github/workflows/maven-checks.yaml b/.github/workflows/maven-checks.yml similarity index 100% rename from .github/workflows/maven-checks.yaml rename to .github/workflows/maven-checks.yml diff --git a/.github/workflows/prestocpp-format-and-header-check.yml b/.github/workflows/prestocpp-format-and-header-check.yml new file mode 100644 index 0000000000000..c554ee8785786 --- /dev/null +++ b/.github/workflows/prestocpp-format-and-header-check.yml @@ -0,0 +1,44 @@ +name: prestocpp-format-and-header-check + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + - '.github/workflows/prestocpp-format-and-header-check.yml' + push: + paths: + - 'presto-native-execution/**' + - '.github/workflows/prestocpp-format-and-header-check.yml' + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so + # that each commit to release-0.293-clp-connector is checked completely. + cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" + +jobs: + prestocpp-format-and-header-check: + runs-on: ubuntu-latest + container: + image: public.ecr.aws/oss-presto/velox-dev:check + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Check formatting + run: | + git fetch origin master + cd presto-native-execution + make format-check + + - name: Check license headers + run: | + git fetch origin master + cd presto-native-execution + make header-check diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml new file mode 100644 index 0000000000000..e26e330403ec2 --- /dev/null +++ b/.github/workflows/prestocpp-linux-build-and-unit-test.yml @@ -0,0 +1,314 @@ +name: prestocpp-linux-build-and-unit-test + +on: + workflow_dispatch: + pull_request: + paths: + - 'presto-native-execution/**' + - 'presto-native-sidecar-plugin/**' + - '.github/workflows/prestocpp-linux-build-and-unit-test.yml' + push: + paths-ignore: + - 'presto-docs/**' + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so + # that each commit to release-0.293-clp-connector is checked completely. + cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" + +jobs: + prestocpp-linux-build-for-test: + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e + env: + CCACHE_DIR: "${{ github.workspace }}/ccache" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Update velox + run: | + cd presto-native-execution + make velox-submodule + + - name: Install Github CLI for using apache/infrastructure-actions/stash + run: | + curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm + rpm -iv gh_2.63.2_linux_amd64.rpm + + - uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-build-for-test + + - name: Zero ccache statistics + run: ccache -sz + + - name: Build engine + run: | + source /opt/rh/gcc-toolset-12/enable + cd presto-native-execution + cmake \ + -B _build/release \ + -GNinja \ + -DTREAT_WARNINGS_AS_ERRORS=1 \ + -DENABLE_ALL_WARNINGS=1 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \ + -DPRESTO_ENABLE_JWT=ON \ + -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \ + -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DThrift_ROOT=/usr/local \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DMAX_LINK_JOBS=4 + ninja -C _build/release -j $(getconf _NPROCESSORS_ONLN) + + - name: Ccache after + run: ccache -s + + - uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3 + with: + path: '${{ env.CCACHE_DIR }}' + key: ccache-prestocpp-linux-build-for-test + + - name: Run Unit Tests + run: | + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + cd presto-native-execution/_build/release + ctest -j $(getconf _NPROCESSORS_ONLN) -VV --output-on-failure --exclude-regex velox.* + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: presto-native-build + path: | + presto-native-execution/_build/release/presto_cpp/main/presto_server + presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + + prestocpp-linux-presto-e2e-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 8.0.442 + cache: 'maven' + - name: Download nodejs to maven cache + run: .github/bin/download_nodejs + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run presto-native e2e tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + # TODO: neeed to enable remote function tests with + # "-Ppresto-native-execution-remote-functions" once + # > https://github.com/facebookincubator/velox/discussions/6163 + # is fixed. + + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-execution' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + + prestocpp-linux-presto-native-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + storage-format: [ "PARQUET", "DWRF" ] + container: + image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8.0.442' + cache: 'maven' + - name: Download nodejs to maven cache + run: .github/bin/download_nodejs + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + + - name: Run presto-native tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-tests/src/test -type f -name 'Test*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-tests' \ + -DstorageFormat=${{ matrix.storage-format }} \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C + + prestocpp-linux-presto-sidecar-tests: + needs: prestocpp-linux-build-for-test + runs-on: ubuntu-22.04 + container: + image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e + env: + MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end" + steps: + - uses: actions/checkout@v4 + - name: Fix git permissions + # Usually actions/checkout does this but as we run in a container + # it doesn't work + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: presto-native-build + path: presto-native-execution/_build/release + + # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38 + - name: Restore execute permissions and library path + run: | + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server + chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main + # Ensure transitive dependency libboost-iostreams is found. + ldconfig /usr/local/lib + - name: Install OpenJDK8 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '8.0.442' + cache: 'maven' + - name: Download nodejs to maven cache + run: .github/bin/download_nodejs + + - name: Maven install + env: + # Use different Maven options to install. + MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + run: | + for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s) + - name: Run presto-native sidecar tests + run: | + export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server" + export TESTFILES=`find ./presto-native-sidecar-plugin/src/test -type f -name 'Test*.java'` + # Convert file paths to comma separated class names + export TESTCLASSES= + for test_file in $TESTFILES + do + tmp=${test_file##*/} + test_class=${tmp%%\.*} + export TESTCLASSES="${TESTCLASSES},$test_class" + done + export TESTCLASSES=${TESTCLASSES#,} + echo "TESTCLASSES = $TESTCLASSES" + mvn test \ + ${MAVEN_TEST} \ + -pl 'presto-native-sidecar-plugin' \ + -Dtest="${TESTCLASSES}" \ + -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \ + -DDATA_DIR=${RUNNER_TEMP} \ + -Duser.timezone=America/Bahia_Banderas \ + -T1C diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000000..6f829502fbbb8 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,84 @@ +name: test + +on: + pull_request: + push: + +env: + # An envar that signals to tests we are executing in the CI environment + CONTINUOUS_INTEGRATION: true + MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError" + MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError" + MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all --no-transfer-progress -Dmaven.javadoc.skip=true" + MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --no-transfer-progress --fail-at-end" + RETRY: .github/bin/retry + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `release-0.293-clp-connector` branch so + # that each commit to release-0.293-clp-connector is checked completely. + cancel-in-progress: "${{github.ref != 'refs/heads/release-0.293-clp-connector'}}" + +jobs: + changes: + runs-on: ubuntu-latest + # Required permissions + permissions: + pull-requests: read + # Set job outputs to values from filter step + outputs: + codechange: ${{ steps.filter.outputs.codechange }} + steps: + - uses: "actions/checkout@v4" + with: + submodules: "recursive" + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + codechange: + - '!presto-docs/**' + + test: + runs-on: ubuntu-latest + needs: changes + strategy: + fail-fast: false + matrix: + java: [8.0.442, 17.0.13] + modules: + - ":presto-tests -P presto-tests-execution-memory" + - ":presto-tests -P presto-tests-general" + - ":presto-tests -P ci-only-distributed-non-hash-gen" + - ":presto-tests -P ci-only-tpch-distributed-queries" + - ":presto-tests -P ci-only-local-queries" + - ":presto-tests -P ci-only-distributed-queries" + - ":presto-tests -P ci-only-aggregation-queries" + - ":presto-tests -P ci-only-plan-determinism" + - ":presto-tests -P ci-only-resource-manager" + - ":presto-main-base" + - ":presto-main" + timeout-minutes: 80 + steps: + - uses: actions/checkout@v4 + if: needs.changes.outputs.codechange == 'true' + with: + show-progress: false + - uses: actions/setup-java@v4 + if: needs.changes.outputs.codechange == 'true' + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + cache: 'maven' + - name: Download nodejs to maven cache + if: needs.changes.outputs.codechange == 'true' + run: .github/bin/download_nodejs + - name: Maven Install + if: needs.changes.outputs.codechange == 'true' + run: | + export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" + ./mvnw install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) + - name: Maven Tests + if: needs.changes.outputs.codechange == 'true' + run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} From a534bec866ce3d914fb6cfb35c1c8d9d89705ffc Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Thu, 10 Jul 2025 15:54:48 -0400 Subject: [PATCH 09/22] Remove the uncessary action --- .../actions/yscope-image-build/action.yaml | 78 ------------------- 1 file changed, 78 deletions(-) delete mode 100644 .github/actions/yscope-image-build/action.yaml diff --git a/.github/actions/yscope-image-build/action.yaml b/.github/actions/yscope-image-build/action.yaml deleted file mode 100644 index cad718ba47ea3..0000000000000 --- a/.github/actions/yscope-image-build/action.yaml +++ /dev/null @@ -1,78 +0,0 @@ -name: "yscope-image-build" - -inputs: - build_args: - default: | - PRESTO_VERSION=0.293 - description: "The build args for building the image" - required: false - context_dir: - default: "./" - description: "Which directory we are in when building the image" - required: false - dockerfile_path: - description: "Dockerfile relative path from the root of the project" - required: true - image_name_prefix: - description: "Container image name prefix" - required: true - image_registry: - default: "ghcr.io" - description: "Container image registry" - required: false - image_registry_username: - default: "${{github.actor}}" - description: "Container image registry username" - required: false - image_registry_password: - default: "" - description: "Container image registry password" - required: false - platform_id: - description: "Platform ID of the container (e.g. ubuntu)" - required: true - platform_version_id: - description: "Platform VERSION_ID / VERSION_CODENAME of the container - (e.g. jammy, focal, etc.)" - required: true - -runs: - using: "composite" - steps: - - name: "Set up Docker Buildx" - uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" - - - name: "Login to Image Registry" - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "${{inputs.image_registry}}" - username: "${{inputs.image_registry_username}}" - password: "${{inputs.image_registry_password}}" - - - name: "Sanitize Repository Name" - id: "sanitization" - shell: "bash" - run: | - # Docker doesn't support repository names with uppercase characters, so we convert the - # name to lowercase here. - echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ - >> "$GITHUB_OUTPUT" - - - name: "Update Metadata" - id: "meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /${{inputs.image_name_prefix}}-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" - - - name: "Build and Push" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: ${{inputs.context_dir}} - file: ${{inputs.dockerfile_path}} - push: true - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" - build-args: | - ${{inputs.build_args}} - JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0 From 9e17bc09954b79b51cab01756ec5be89949cb8c4 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 14 Jul 2025 08:58:17 -0400 Subject: [PATCH 10/22] Use docker build-and-push action --- .github/workflows/maven-checks.yml | 59 +++++++++--------- ...with-clp-connector-runtime-image-build.yml | 60 +++++++++++++------ 2 files changed, 69 insertions(+), 50 deletions(-) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index 306d2c1fc22ad..4bab92e04f847 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -71,47 +71,42 @@ jobs: with: submodules: "recursive" - - name: "Login to Image Registry" - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: ghcr.io - username: ${{github.actor}} - password: ${{secrets.GITHUB_TOKEN}} - - name: Download presto-server uses: actions/download-artifact@v4 with: name: presto-server - path: /tmp + path: ./docker - name: Download presto-cli uses: actions/download-artifact@v4 with: name: presto-cli - path: /tmp + path: ./docker - - name: "Run build.sh" - working-directory: ./docker - run: | - mv /tmp/presto-server-0.293.tar.gz ./ - mv /tmp/presto-cli-0.293-executable.jar ./ - - if [ "${{ github.event_name }}" = "push" ] && \ - [ "${{ github.repository }}" = "y-scope/presto" ] && \ - [ "${{ github.ref }}" = "refs/heads/release-0.293-clp-connector" ]; then - export PUBLISH=true - else - echo "github.event_name = ${{ github.event_name }}" - echo "github.repository = ${{ github.repository }}" - echo "github.ref = ${{ github.ref }}" - echo "Skip publish the image" - export PUBLISH=false - fi + - name: "Login to image registry" + uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: ghcr.io + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} - IMAGE_NAME=coordinator-with-clp-connector-runtime \ - REG_ORG=ghcr.io/${{github.actor}}/presto \ - PUBLISH=$PUBLISH \ - ./build.sh 0.293 + - name: "Update Metadata" + id: "meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{github.repository}}/coordinator-with-clp-connector-runtime - rm presto-server-0.293.tar.gz - rm presto-cli-0.293-executable.jar + - name: "Build and push" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + build-args: |- + PRESTO_VERSION=0.293 + JMX_PROMETHEUS_JAVA_AGENT_VERSION=0.20.0 + context: "./docker/" + file: "./docker/Dockerfile" + push: >- + ${{github.event_name != 'pull_request' + && github.ref == 'refs/heads/release-0.293-clp-connector'}} + tags: "${{steps.meta.outputs.tags}}" + labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index b768862252da6..6e606cb8455f9 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -20,21 +20,45 @@ jobs: registry: ghcr.io username: ${{github.actor}} password: ${{secrets.GITHUB_TOKEN}} - - - name: "Build worker image" - working-directory: ./presto-native-execution - run: | - sed -i 's|image: presto/prestissimo-runtime:ubuntu-22.04|image: ghcr.io/${{github.actor}}/presto/prestissimo-with-clp-connector-runtime:ubuntu-22.04|' docker-compose.yml - docker compose build ubuntu-native-dependency - docker compose build ubuntu-native-runtime - - if [ "${{ github.event_name }}" = "push" ] && \ - [ "${{ github.repository }}" = "y-scope/presto" ] && \ - [ "${{ github.ref }}" = "refs/heads/release-0.293-clp-connector" ]; then - docker push "ghcr.io/${{github.actor}}/presto/prestissimo-with-clp-connector-runtime:ubuntu-22.04" - else - echo "github.event_name = ${{ github.event_name }}" - echo "github.repository = ${{ github.repository }}" - echo "github.ref = ${{ github.ref }}" - echo "Skip publish the image" - fi + + - name: "Update Metadata for Dependency Image" + id: "meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-dependency + + - name: "Build and push" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "./presto-native-execution/" + file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile" + push: >- + ${{github.event_name != 'pull_request' + && github.ref == 'refs/heads/release-0.293-clp-connector'}} + tags: "${{steps.meta.outputs.tags}}" + labels: "${{steps.meta.outputs.labels}}" + + - name: "Update Metadata for Runtime Image" + id: "meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-runtime + + - name: "Build and push" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + build-args: |- + NUM_THREADS=2 + DEPENDENCY_IMAGE=ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-dependency + BASE_IMAGE=ubuntu:22.04 + OSNAME=ubuntu + EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON + context: "./presto-native-execution/" + file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" + push: >- + ${{github.event_name != 'pull_request' + && github.ref == 'refs/heads/release-0.293-clp-connector'}} + tags: "${{steps.meta.outputs.tags}}" + labels: "${{steps.meta.outputs.labels}}" From dbb28fa5921407592de9c4251ba824d0ec722764 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 14 Jul 2025 09:11:30 -0400 Subject: [PATCH 11/22] Fix --- ...-with-clp-connector-runtime-image-build.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index 6e606cb8455f9..ca95239a4624f 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -22,11 +22,13 @@ jobs: password: ${{secrets.GITHUB_TOKEN}} - name: "Update Metadata for Dependency Image" - id: "meta" + id: "meta-dependency" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: >- ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-dependency + tags: | + type=raw,value=ubuntu-22.04 - name: "Build and push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" @@ -36,22 +38,24 @@ jobs: push: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" + tags: "${{steps.meta-dependency.outputs.tags}}" + labels: "${{steps.meta-dependency.outputs.labels}}" - name: "Update Metadata for Runtime Image" - id: "meta" + id: "meta-runtime" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: >- ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-runtime + tags: | + type=raw,value=ubuntu-22.04 - name: "Build and push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: build-args: |- NUM_THREADS=2 - DEPENDENCY_IMAGE=ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-dependency + DEPENDENCY_IMAGE=${{ steps.meta-dependency.outputs.tags }} BASE_IMAGE=ubuntu:22.04 OSNAME=ubuntu EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON @@ -60,5 +64,5 @@ jobs: push: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" + tags: "${{steps.meta-runtime.outputs.tags}}" + labels: "${{steps.meta-runtime.outputs.labels}}" From 1e6968170d3f6ab9e47db7ba12c8f971683e963c Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 14 Jul 2025 09:27:21 -0400 Subject: [PATCH 12/22] Fix --- .github/workflows/maven-checks.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index 4bab92e04f847..05f5b7c8e8c5e 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -96,6 +96,13 @@ jobs: with: images: >- ghcr.io/${{github.repository}}/coordinator-with-clp-connector-runtime + tags: | + type=raw,value=centos9 + + - name: "Validate" + run: | + echo ${{ github.event_name }} + echo ${{ github.ref }} - name: "Build and push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" From 4ca014c6b3118e9376243c832151e09cc96939bf Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 14 Jul 2025 10:03:06 -0400 Subject: [PATCH 13/22] Fix --- .github/workflows/maven-checks.yml | 5 ---- ...with-clp-connector-runtime-image-build.yml | 2 +- docker/build.sh | 23 +++++++++++-------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index 05f5b7c8e8c5e..a5fef5696c477 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -99,11 +99,6 @@ jobs: tags: | type=raw,value=centos9 - - name: "Validate" - run: | - echo ${{ github.event_name }} - echo ${{ github.ref }} - - name: "Build and push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index ca95239a4624f..ebe68e3f9e1a1 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -54,7 +54,7 @@ jobs: uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: build-args: |- - NUM_THREADS=2 + NUM_THREADS=4 DEPENDENCY_IMAGE=${{ steps.meta-dependency.outputs.tags }} BASE_IMAGE=ubuntu:22.04 OSNAME=ubuntu diff --git a/docker/build.sh b/docker/build.sh index 2edd7646eec19..f561b5edc76da 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -17,19 +17,22 @@ VERSION=$1; shift TAG="${TAG:-latest}" IMAGE_NAME="${IMAGE_NAME:-presto}" REG_ORG="${REG_ORG:-docker.io/prestodb}" +PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64,linux/ppc64le}" +TMP_IIDFILE=$(mktemp) +IIDFILE="${IIDFILE:-$TMP_IIDFILE}" PUBLISH="${PUBLISH:-false}" +BUILDER="${BUILDER:-container}" -# Compose full image name with tag -FULL_IMAGE_NAME="${REG_ORG}/${IMAGE_NAME}:${TAG}" - -# Build image using regular docker build -docker build \ +export BUILDX_NO_DEFAULT_ATTESTATIONS="" +# If PUBLISH=false, images only stores in local cache, otherwise they are pushed to th container registry +docker buildx build --builder="${BUILDER}" --iidfile "${IIDFILE}" \ --build-arg="PRESTO_VERSION=${VERSION}" \ --build-arg="JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0" \ - -t "${FULL_IMAGE_NAME}" \ - -f Dockerfile . + --output "type=image,name=${REG_ORG}/${IMAGE_NAME},push-by-digest=true,name-canonical=true,push=${PUBLISH}" \ + --platform "${PLATFORMS}" -f Dockerfile . -# Optionally push the image -if [[ "$PUBLISH" == "true" ]]; then - docker push "${FULL_IMAGE_NAME}" +if [[ "$PUBLISH" = "true" ]]; then + # This only happens when push=true, since push-by-digest=true in the above build step, need to tag the images explicitly + docker buildx imagetools create --builder="${BUILDER}" \ + -t "${REG_ORG}/${IMAGE_NAME}:${TAG}" "${REG_ORG}/${IMAGE_NAME}@$(cat "$IIDFILE")" fi From fec665a250fa647f73a963efad375be324f89ac3 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 14 Jul 2025 12:31:01 -0400 Subject: [PATCH 14/22] Fix --- ...with-clp-connector-runtime-image-build.yml | 42 ++++++++++++++----- .../prestissimo-runtime.dockerfile | 9 ++++ 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index ebe68e3f9e1a1..a9db907a645a8 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -30,7 +30,31 @@ jobs: tags: | type=raw,value=ubuntu-22.04 - - name: "Build and push" + - name: "Update Metadata for Runtime Image" + id: "meta-runtime" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-runtime + tags: | + type=raw,value=ubuntu-22.04 + + - name: Check for dependency changes in presto-native-execution + id: dependency-changes + run: | + git fetch origin ${{ github.event.before }} --depth=1 || true + dep_changed=false + if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q '^presto-native-execution/scripts'; then + dep_changed=true + fi + cd presto-native-execution/velox + if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q '^scripts'; then + dep_changed=true + fi + echo "dep-changed=${dep_changed}" >> $GITHUB_OUTPUT + + - name: "Build and push dependency image" + if: steps.dependency-changes.outputs.dep-changed == 'true' uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./presto-native-execution/" @@ -41,20 +65,16 @@ jobs: tags: "${{steps.meta-dependency.outputs.tags}}" labels: "${{steps.meta-dependency.outputs.labels}}" - - name: "Update Metadata for Runtime Image" - id: "meta-runtime" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-runtime - tags: | - type=raw,value=ubuntu-22.04 + - name: Get number of CPU + id: get-cores + run: | + echo "num-threads=$(nproc)" >> $GITHUB_OUTPUT - - name: "Build and push" + - name: "Build and push runtime image" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: build-args: |- - NUM_THREADS=4 + NUM_THREADS=${{ steps.get-cores.outputs.num-threads }} DEPENDENCY_IMAGE=${{ steps.meta-dependency.outputs.tags }} BASE_IMAGE=ubuntu:22.04 OSNAME=ubuntu diff --git a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile index 7ae303f21f27d..84cea848fb734 100644 --- a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile @@ -39,6 +39,15 @@ FROM ${BASE_IMAGE} ENV BUILD_BASE_DIR=_build ENV BUILD_DIR="" +# TODO: This is a temporary fix for issue: https://github.com/prestodb/presto/issues/25531 +# When there is a proper fix, update this piece of code. +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata && \ + ln -snf /usr/share/zoneinfo/America/Toronto /etc/localtime && \ + echo "America/New_York" > /etc/timezone && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + COPY --chmod=0775 --from=prestissimo-image /prestissimo/${BUILD_BASE_DIR}/${BUILD_DIR}/presto_cpp/main/presto_server /usr/bin/ COPY --chmod=0775 --from=prestissimo-image /runtime-libraries/* /usr/lib64/prestissimo-libs/ COPY --chmod=0755 ./etc /opt/presto-server/etc From 283b7514331b920fae4a3e7134ecee71185a2b29 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Tue, 15 Jul 2025 14:11:31 -0400 Subject: [PATCH 15/22] Fix --- ...p-worker-with-clp-connector-runtime-image-build.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index a9db907a645a8..2271e3207c030 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -39,7 +39,7 @@ jobs: tags: | type=raw,value=ubuntu-22.04 - - name: Check for dependency changes in presto-native-execution + - name: Check for dependency changes or missing dependency image id: dependency-changes run: | git fetch origin ${{ github.event.before }} --depth=1 || true @@ -53,8 +53,14 @@ jobs: fi echo "dep-changed=${dep_changed}" >> $GITHUB_OUTPUT + image_exists=true + if ! docker manifest inspect "${{ steps.meta-dependency.outputs.tags }}" > /dev/null 2>&1; then + image_exists=false + fi + echo "image-exists=${image_exists}" >> $GITHUB_OUTPUT + - name: "Build and push dependency image" - if: steps.dependency-changes.outputs.dep-changed == 'true' + if: steps.dependency-changes.outputs.dep-changed == 'true' || steps.dependency-changes.outputs.image-exists == 'false' uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./presto-native-execution/" From a6ebbaa41fe3f052fabc93eab709c8e310133ca1 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Tue, 15 Jul 2025 14:25:25 -0400 Subject: [PATCH 16/22] Fix --- ...estocpp-worker-with-clp-connector-runtime-image-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index 2271e3207c030..10ac73e3458f3 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -47,8 +47,11 @@ jobs: if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q '^presto-native-execution/scripts'; then dep_changed=true fi + + velox_old_sha=$(git ls-tree ${{ github.event.before }} presto-native-execution/velox | awk '{print $3}') + velox_new_sha=$(git ls-tree ${{ github.sha }} presto-native-execution/velox | awk '{print $3}') cd presto-native-execution/velox - if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q '^scripts'; then + if git diff --name-only ${velox_old_sha} ${velox_new_sha} | grep -q '^scripts'; then dep_changed=true fi echo "dep-changed=${dep_changed}" >> $GITHUB_OUTPUT From 5ca0420ab8ea7b7db979fb76f89d967ae6096b4b Mon Sep 17 00:00:00 2001 From: "Xiaochong(Eddy) Wei" <40865608+anlowee@users.noreply.github.com> Date: Fri, 18 Jul 2025 16:18:06 -0400 Subject: [PATCH 17/22] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .github/workflows/maven-checks.yml | 27 ++++++++++--------- ...with-clp-connector-runtime-image-build.yml | 26 +++++++++--------- .../prestissimo-runtime.dockerfile | 4 +-- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index a5fef5696c477..d8eab21a639a6 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -46,28 +46,31 @@ jobs: run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install -B -V -T 1C -DskipTests -Dmaven.javadoc.skip=true --no-transfer-progress -P ci -pl '!presto-test-coverage,!:presto-docs' - - name: Upload presto-server tarball + - name: Upload presto-server if: matrix.java == '8.0.442' uses: actions/upload-artifact@v4 with: name: presto-server path: presto-server/target/presto-server-0.293.tar.gz - - name: Upload presto-cli executable + if-no-files-found: "error" + retention-days: 1 + - name: Upload presto-cli if: matrix.java == '8.0.442' uses: actions/upload-artifact@v4 with: name: presto-cli path: presto-cli/target/presto-cli-0.293-executable.jar + if-no-files-found: "error" + retention-days: 1 - name: Clean Maven Output run: ./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage' - presto-coordinator-with-clp-connector-runtime-image: name: presto-coordinator-runtime-image needs: maven-checks runs-on: ubuntu-22.04 if: ${{ always() && success() }} steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + - uses: "actions/checkout@v4" with: submodules: "recursive" @@ -84,27 +87,27 @@ jobs: path: ./docker - name: "Login to image registry" - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + uses: "docker/login-action@v3" with: registry: ghcr.io username: ${{github.actor}} password: ${{secrets.GITHUB_TOKEN}} - - name: "Update Metadata" + - name: "Set up container image metadata" id: "meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + uses: "docker/metadata-action@v5" with: images: >- - ghcr.io/${{github.repository}}/coordinator-with-clp-connector-runtime - tags: | - type=raw,value=centos9 + ghcr.io/${{github.repository}}/coordinator + tags: |- + type=raw,value=dev - name: "Build and push" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + uses: "docker/build-push-action@v6" with: build-args: |- - PRESTO_VERSION=0.293 JMX_PROMETHEUS_JAVA_AGENT_VERSION=0.20.0 + PRESTO_VERSION=0.293 context: "./docker/" file: "./docker/Dockerfile" push: >- diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml index 10ac73e3458f3..81fec1cd9d1e1 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml @@ -14,30 +14,30 @@ jobs: with: submodules: "recursive" - - name: "Login to Image Registry" + - name: "Login to image registry" uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" with: registry: ghcr.io username: ${{github.actor}} password: ${{secrets.GITHUB_TOKEN}} - - name: "Update Metadata for Dependency Image" + - name: "Set up metadata for dependency image" id: "meta-dependency" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: >- - ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-dependency - tags: | - type=raw,value=ubuntu-22.04 + ghcr.io/${{github.repository}}/prestissimo-worker-dev-env + tags: |- + type=raw,value=dev - - name: "Update Metadata for Runtime Image" + - name: "Set up metadata for runtime image" id: "meta-runtime" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: >- - ghcr.io/${{github.repository}}/prestissimo-with-clp-connector-runtime - tags: | - type=raw,value=ubuntu-22.04 + ghcr.io/${{github.repository}}/prestissimo-worker + tags: |- + type=raw,value=dev - name: Check for dependency changes or missing dependency image id: dependency-changes @@ -74,17 +74,17 @@ jobs: tags: "${{steps.meta-dependency.outputs.tags}}" labels: "${{steps.meta-dependency.outputs.labels}}" - - name: Get number of CPU + - name: Get number of cores id: get-cores - run: | + run: |- echo "num-threads=$(nproc)" >> $GITHUB_OUTPUT - name: "Build and push runtime image" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: build-args: |- - NUM_THREADS=${{ steps.get-cores.outputs.num-threads }} - DEPENDENCY_IMAGE=${{ steps.meta-dependency.outputs.tags }} + NUM_THREADS=${{steps.get-cores.outputs.num-threads}} + DEPENDENCY_IMAGE=${{steps.meta-dependency.outputs.tags}} BASE_IMAGE=ubuntu:22.04 OSNAME=ubuntu EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON diff --git a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile index 84cea848fb734..88c5a5787e2db 100644 --- a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile @@ -39,8 +39,8 @@ FROM ${BASE_IMAGE} ENV BUILD_BASE_DIR=_build ENV BUILD_DIR="" -# TODO: This is a temporary fix for issue: https://github.com/prestodb/presto/issues/25531 -# When there is a proper fix, update this piece of code. +# Temporary fix for https://github.com/prestodb/presto/issues/25531 +# TODO: Update this code when there's a proper fix RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata && \ ln -snf /usr/share/zoneinfo/America/Toronto /etc/localtime && \ From fa06d23b4cd67176aea4f6361bc00c04949419c0 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Fri, 18 Jul 2025 16:28:01 -0400 Subject: [PATCH 18/22] Address comments --- .github/workflows/maven-checks.yml | 4 +- ...ml => prestissimo-worker-images-build.yml} | 66 ++++++------------- 2 files changed, 23 insertions(+), 47 deletions(-) rename .github/workflows/{prestocpp-worker-with-clp-connector-runtime-image-build.yml => prestissimo-worker-images-build.yml} (54%) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index d8eab21a639a6..4e79cbe55f89a 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -64,8 +64,8 @@ jobs: retention-days: 1 - name: Clean Maven Output run: ./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage' - presto-coordinator-with-clp-connector-runtime-image: - name: presto-coordinator-runtime-image + presto-coordinator-image: + name: presto-coordinator-image needs: maven-checks runs-on: ubuntu-22.04 if: ${{ always() && success() }} diff --git a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml b/.github/workflows/prestissimo-worker-images-build.yml similarity index 54% rename from .github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml rename to .github/workflows/prestissimo-worker-images-build.yml index 81fec1cd9d1e1..32a6ac5ce2c19 100644 --- a/.github/workflows/prestocpp-worker-with-clp-connector-runtime-image-build.yml +++ b/.github/workflows/prestissimo-worker-images-build.yml @@ -1,4 +1,4 @@ -name: prestocpp-worker-with-clp-connector-runtime-image-build +name: prestissimo-worker-images-build on: # TODO: specifiy the branch to the release-0.293 when finalize the PR @@ -6,8 +6,8 @@ on: push: jobs: - prestocpp-worker-with-clp-connector-runtime-image: - name: prestocpp-worker-with-clp-connector-runtime-image + prestissimo-worker-images-build: + name: prestissimo-worker-images-build runs-on: ubuntu-22.04 steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" @@ -22,7 +22,7 @@ jobs: password: ${{secrets.GITHUB_TOKEN}} - name: "Set up metadata for dependency image" - id: "meta-dependency" + id: "metadata-deps-image" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: >- @@ -30,40 +30,7 @@ jobs: tags: |- type=raw,value=dev - - name: "Set up metadata for runtime image" - id: "meta-runtime" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{github.repository}}/prestissimo-worker - tags: |- - type=raw,value=dev - - - name: Check for dependency changes or missing dependency image - id: dependency-changes - run: | - git fetch origin ${{ github.event.before }} --depth=1 || true - dep_changed=false - if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q '^presto-native-execution/scripts'; then - dep_changed=true - fi - - velox_old_sha=$(git ls-tree ${{ github.event.before }} presto-native-execution/velox | awk '{print $3}') - velox_new_sha=$(git ls-tree ${{ github.sha }} presto-native-execution/velox | awk '{print $3}') - cd presto-native-execution/velox - if git diff --name-only ${velox_old_sha} ${velox_new_sha} | grep -q '^scripts'; then - dep_changed=true - fi - echo "dep-changed=${dep_changed}" >> $GITHUB_OUTPUT - - image_exists=true - if ! docker manifest inspect "${{ steps.meta-dependency.outputs.tags }}" > /dev/null 2>&1; then - image_exists=false - fi - echo "image-exists=${image_exists}" >> $GITHUB_OUTPUT - - name: "Build and push dependency image" - if: steps.dependency-changes.outputs.dep-changed == 'true' || steps.dependency-changes.outputs.image-exists == 'false' uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./presto-native-execution/" @@ -71,27 +38,36 @@ jobs: push: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.meta-dependency.outputs.tags}}" - labels: "${{steps.meta-dependency.outputs.labels}}" + tags: "${{steps.metadata-deps-image.outputs.tags}}" + labels: "${{steps.metadata-deps-image.outputs.labels}}" + + - name: "Set up metadata for runtime image" + id: "metadata-runtime-image" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{github.repository}}/prestissimo-worker + tags: |- + type=raw,value=dev - name: Get number of cores id: get-cores run: |- - echo "num-threads=$(nproc)" >> $GITHUB_OUTPUT + echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT - name: "Build and push runtime image" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: build-args: |- - NUM_THREADS=${{steps.get-cores.outputs.num-threads}} - DEPENDENCY_IMAGE=${{steps.meta-dependency.outputs.tags}} BASE_IMAGE=ubuntu:22.04 - OSNAME=ubuntu + DEPENDENCY_IMAGE=${{steps.metadata-deps-image.outputs.tags}} EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON + NUM_THREADS=${{steps.get-cores.outputs.num_cores}} + OSNAME=ubuntu context: "./presto-native-execution/" file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" push: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/release-0.293-clp-connector'}} - tags: "${{steps.meta-runtime.outputs.tags}}" - labels: "${{steps.meta-runtime.outputs.labels}}" + tags: "${{steps.metadata-runtime-image.outputs.tags}}" + labels: "${{steps.metadata-runtime-image.outputs.labels}}" From 3b76c12b6c714d4694922b943df9c59a31958033 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 21 Jul 2025 14:27:25 -0400 Subject: [PATCH 19/22] Address comments --- .github/workflows/maven-checks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index 4e79cbe55f89a..ef84043820193 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -68,7 +68,6 @@ jobs: name: presto-coordinator-image needs: maven-checks runs-on: ubuntu-22.04 - if: ${{ always() && success() }} steps: - uses: "actions/checkout@v4" with: From 74787c767501cd6b25474b1a6d3c2b6998fd6fc9 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 21 Jul 2025 14:29:42 -0400 Subject: [PATCH 20/22] Remove TODO --- .github/workflows/prestissimo-worker-images-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/prestissimo-worker-images-build.yml b/.github/workflows/prestissimo-worker-images-build.yml index 32a6ac5ce2c19..a223ae896530f 100644 --- a/.github/workflows/prestissimo-worker-images-build.yml +++ b/.github/workflows/prestissimo-worker-images-build.yml @@ -1,7 +1,6 @@ name: prestissimo-worker-images-build on: - # TODO: specifiy the branch to the release-0.293 when finalize the PR pull_request: push: From 313a64834db400b741f9597c0a8cce641d0243a8 Mon Sep 17 00:00:00 2001 From: Xiaochong Wei Date: Mon, 21 Jul 2025 18:13:40 -0400 Subject: [PATCH 21/22] Address comments --- .../scripts/dockerfiles/prestissimo-runtime.dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile index 88c5a5787e2db..0cd18dc3ce498 100644 --- a/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile +++ b/presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile @@ -43,8 +43,6 @@ ENV BUILD_DIR="" # TODO: Update this code when there's a proper fix RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata && \ - ln -snf /usr/share/zoneinfo/America/Toronto /etc/localtime && \ - echo "America/New_York" > /etc/timezone && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From ceaa80935ca4020ba5d42698478c7746ddb494c2 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 21 Jul 2025 19:04:35 -0400 Subject: [PATCH 22/22] Minor refactoring. --- .github/workflows/maven-checks.yml | 57 +++++++++---------- .../prestissimo-worker-images-build.yml | 36 ++++++------ 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/.github/workflows/maven-checks.yml b/.github/workflows/maven-checks.yml index ef84043820193..2295c413bef7b 100644 --- a/.github/workflows/maven-checks.yml +++ b/.github/workflows/maven-checks.yml @@ -46,60 +46,59 @@ jobs: run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" ./mvnw install -B -V -T 1C -DskipTests -Dmaven.javadoc.skip=true --no-transfer-progress -P ci -pl '!presto-test-coverage,!:presto-docs' - - name: Upload presto-server + - name: "Upload presto-server" if: matrix.java == '8.0.442' - uses: actions/upload-artifact@v4 + uses: "actions/upload-artifact@v4" with: - name: presto-server - path: presto-server/target/presto-server-0.293.tar.gz + name: "presto-server" + path: "presto-server/target/presto-server-0.293.tar.gz" if-no-files-found: "error" retention-days: 1 - - name: Upload presto-cli + - name: "Upload presto-cli" if: matrix.java == '8.0.442' - uses: actions/upload-artifact@v4 + uses: "actions/upload-artifact@v4" with: - name: presto-cli - path: presto-cli/target/presto-cli-0.293-executable.jar + name: "presto-cli" + path: "presto-cli/target/presto-cli-0.293-executable.jar" if-no-files-found: "error" retention-days: 1 - - name: Clean Maven Output - run: ./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage' + - name: "Clean Maven output" + run: "./mvnw clean -pl '!:presto-server,!:presto-cli,!presto-test-coverage'" + presto-coordinator-image: - name: presto-coordinator-image - needs: maven-checks - runs-on: ubuntu-22.04 + name: "presto-coordinator-image" + needs: "maven-checks" + runs-on: "ubuntu-22.04" steps: - uses: "actions/checkout@v4" with: submodules: "recursive" - - name: Download presto-server - uses: actions/download-artifact@v4 + - name: "Download presto-server" + uses: "actions/download-artifact@v4" with: - name: presto-server - path: ./docker + name: "presto-server" + path: "./docker" - - name: Download presto-cli - uses: actions/download-artifact@v4 + - name: "Download presto-cli" + uses: "actions/download-artifact@v4" with: - name: presto-cli - path: ./docker + name: "presto-cli" + path: "./docker" - name: "Login to image registry" uses: "docker/login-action@v3" with: - registry: ghcr.io - username: ${{github.actor}} - password: ${{secrets.GITHUB_TOKEN}} + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" - name: "Set up container image metadata" id: "meta" uses: "docker/metadata-action@v5" with: - images: >- - ghcr.io/${{github.repository}}/coordinator - tags: |- - type=raw,value=dev + images: "ghcr.io/${{github.repository}}/coordinator" + tags: "type=raw,value=dev" - name: "Build and push" uses: "docker/build-push-action@v6" @@ -107,7 +106,7 @@ jobs: build-args: |- JMX_PROMETHEUS_JAVA_AGENT_VERSION=0.20.0 PRESTO_VERSION=0.293 - context: "./docker/" + context: "./docker" file: "./docker/Dockerfile" push: >- ${{github.event_name != 'pull_request' diff --git a/.github/workflows/prestissimo-worker-images-build.yml b/.github/workflows/prestissimo-worker-images-build.yml index a223ae896530f..b36dcb71949be 100644 --- a/.github/workflows/prestissimo-worker-images-build.yml +++ b/.github/workflows/prestissimo-worker-images-build.yml @@ -1,4 +1,4 @@ -name: prestissimo-worker-images-build +name: "prestissimo-worker-images-build" on: pull_request: @@ -6,8 +6,8 @@ on: jobs: prestissimo-worker-images-build: - name: prestissimo-worker-images-build - runs-on: ubuntu-22.04 + name: "prestissimo-worker-images-build" + runs-on: "ubuntu-22.04" steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: @@ -16,23 +16,21 @@ jobs: - name: "Login to image registry" uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" with: - registry: ghcr.io - username: ${{github.actor}} - password: ${{secrets.GITHUB_TOKEN}} + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" - name: "Set up metadata for dependency image" id: "metadata-deps-image" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: - images: >- - ghcr.io/${{github.repository}}/prestissimo-worker-dev-env - tags: |- - type=raw,value=dev + images: "ghcr.io/${{github.repository}}/prestissimo-worker-dev-env" + tags: "type=raw,value=dev" - name: "Build and push dependency image" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: - context: "./presto-native-execution/" + context: "./presto-native-execution" file: "./presto-native-execution/scripts/dockerfiles/ubuntu-22.04-dependency.dockerfile" push: >- ${{github.event_name != 'pull_request' @@ -44,13 +42,11 @@ jobs: id: "metadata-runtime-image" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: - images: >- - ghcr.io/${{github.repository}}/prestissimo-worker - tags: |- - type=raw,value=dev + images: "ghcr.io/${{github.repository}}/prestissimo-worker" + tags: "type=raw,value=dev" - - name: Get number of cores - id: get-cores + - name: "Get number of cores" + id: "get-cores" run: |- echo "num_cores=$(nproc)" >> $GITHUB_OUTPUT @@ -60,10 +56,12 @@ jobs: build-args: |- BASE_IMAGE=ubuntu:22.04 DEPENDENCY_IMAGE=${{steps.metadata-deps-image.outputs.tags}} - EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF -DPRESTO_ENABLE_PARQUET=ON -DPRESTO_ENABLE_S3=ON + EXTRA_CMAKE_FLAGS=-DPRESTO_ENABLE_TESTING=OFF \ + -DPRESTO_ENABLE_PARQUET=ON \ + -DPRESTO_ENABLE_S3=ON NUM_THREADS=${{steps.get-cores.outputs.num_cores}} OSNAME=ubuntu - context: "./presto-native-execution/" + context: "./presto-native-execution" file: "./presto-native-execution/scripts/dockerfiles/prestissimo-runtime.dockerfile" push: >- ${{github.event_name != 'pull_request'