diff --git a/.github/workflows/cloudberry-ci.yml b/.github/workflows/cloudberry-ci.yml new file mode 100644 index 0000000..9c7d123 --- /dev/null +++ b/.github/workflows/cloudberry-ci.yml @@ -0,0 +1,130 @@ +name: Cloudberry PL/Java CI + +on: + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened, edited] + push: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CLOUDBERRY_REPO: apache/cloudberry + CLOUDBERRY_REF: main + COMPOSE_FILE: pljava/concourse/docker/ubuntu22.04/docker-compose.yml + CONTAINER_NAME: cbdb-pljava + +jobs: + ci: + name: Build + Smoke (Docker) + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet || true + sudo rm -rf /opt/ghc || true + sudo rm -rf /usr/local/share/boost || true + sudo rm -rf /usr/local/lib/android || true + sudo rm -rf /opt/hostedtoolcache || true + sudo docker system prune -af || true + df -h + + - name: Checkout PL/Java (this repo) + uses: actions/checkout@v4 + with: + fetch-depth: 1 + path: pljava + + - name: Checkout Cloudberry source + uses: actions/checkout@v4 + with: + repository: ${{ env.CLOUDBERRY_REPO }} + ref: ${{ env.CLOUDBERRY_REF }} + path: cloudberry + submodules: true + + - name: Build and start dev container + run: | + docker compose -f ${{ env.COMPOSE_FILE }} down -v || true + docker compose -f ${{ env.COMPOSE_FILE }} up -d + docker exec ${{ env.CONTAINER_NAME }} sudo find /home/gpadmin/workspace/cloudberry -path /home/gpadmin/workspace/cloudberry/.git -prune -o -exec chown gpadmin:gpadmin {} + + docker exec ${{ env.CONTAINER_NAME }} sudo find /home/gpadmin/workspace/pljava -path /home/gpadmin/workspace/pljava/.git -prune -o -exec chown gpadmin:gpadmin {} + + + - name: Build Cloudberry + PL/Java and run tests (installcheck) + run: | + docker exec ${{ env.CONTAINER_NAME }} bash -lc "bash /home/gpadmin/workspace/pljava/concourse/docker/ubuntu22.04/scripts/entrypoint.sh" + + - name: Collect logs and results + if: always() + run: | + mkdir -p artifacts + docker logs ${{ env.CONTAINER_NAME }} > artifacts/cbdb-pljava.log 2>&1 || true + docker exec ${{ env.CONTAINER_NAME }} bash -lc "source /usr/local/cloudberry-db/cloudberry-env.sh && source /home/gpadmin/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh && gpstate -s" > artifacts/gpstate.txt 2>&1 || true + docker exec ${{ env.CONTAINER_NAME }} bash -lc "source /usr/local/cloudberry-db/cloudberry-env.sh && source /home/gpadmin/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh && psql -d template1 -c 'select version()'" > artifacts/version.txt 2>&1 || true + + # Regression outputs (if present) + if [ -d pljava/gpdb/tests/results ]; then + tar czf artifacts/pljava-regress-results.tar.gz -C pljava gpdb/tests/results || true + fi + ls -la pljava/gpdb/tests/regression.* 2>/dev/null || true + cp -v pljava/gpdb/tests/regression.* artifacts/ 2>/dev/null || true + + # Cloudberry demo cluster logs (if present) + if [ -d cloudberry/gpAux/gpdemo ]; then + find cloudberry/gpAux/gpdemo -maxdepth 4 -type d \( -name pg_log -o -name log \) -print 2>/dev/null || true + tar czf artifacts/cloudberry-demo-logs.tar.gz -C cloudberry gpAux/gpdemo 2>/dev/null || true + fi + + - name: Summarize installcheck + if: always() + run: | + LOG="pljava/gpdb/tests/results/installcheck.log" + { + echo "## PL/Java installcheck" + if [ -f "$LOG" ]; then + echo "" + echo "- Log: \`$LOG\`" + echo "- Test results dir: \`pljava/gpdb/tests/results/\`" + echo "" + TOTAL=$(grep -E "^[[:space:]]*test .* \\.\\.\\." -c "$LOG" || true) + OK=$(grep -E "^[[:space:]]*test .* \\.\\.\\. ok" -c "$LOG" || true) + FAILED=$(grep -E "^[[:space:]]*test .* \\.\\.\\. FAILED" -c "$LOG" || true) + echo "- Total: $TOTAL" + echo "- ok: $OK" + echo "- FAILED: $FAILED" + echo "" + echo "Last 50 lines:" + echo "" + echo '```' + tail -n 50 "$LOG" || true + echo '```' + else + echo "" + echo "- No \`installcheck.log\` found (build may have failed early)." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs-and-results + path: | + artifacts/** + pljava/gpdb/tests/results/** + pljava/gpdb/tests/regression.* + if-no-files-found: ignore + retention-days: 7 + + - name: Cleanup + if: always() + run: | + docker compose -f ${{ env.COMPOSE_FILE }} down -v || true diff --git a/.gitignore b/.gitignore index aabfe1f..9c1f645 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.classpath /.project /.settings +.idea/ target/site target/staging target/javadoc-bundle-options @@ -10,4 +11,9 @@ pljava-so/target/ target/ *.so *.zip -concourse/secrets/ \ No newline at end of file +concourse/secrets/ +nar-maven-plugin/ +nar-maven-plugin.tar.gz +gpdb/tests/results/ +gpdb/tests/regression.* +artifacts/ diff --git a/Makefile b/Makefile index 271eefd..d9b82dd 100755 --- a/Makefile +++ b/Makefile @@ -41,7 +41,8 @@ REGRESS_DIR = $(top_builddir) .PHONY: build installdirs install uninstall test localconfig targetconfig installcheck targetcheck release install-nar-snapshot: - curl -o nar-maven-plugin.tar.gz https://codeload.github.com/maven-nar/nar-maven-plugin/tar.gz/refs/tags/nar-maven-plugin-3.5.2 + rm -rf nar-maven-plugin nar-maven-plugin-nar-maven-plugin-3.5.2 nar-maven-plugin.tar.gz + curl -fsSL -o nar-maven-plugin.tar.gz https://codeload.github.com/maven-nar/nar-maven-plugin/tar.gz/refs/tags/nar-maven-plugin-3.5.2 tar xzf nar-maven-plugin.tar.gz mv nar-maven-plugin-nar-maven-plugin-3.5.2 nar-maven-plugin (cd nar-maven-plugin ; mvn) diff --git a/concourse/docker/ubuntu22.04/README.md b/concourse/docker/ubuntu22.04/README.md new file mode 100644 index 0000000..8722ba8 --- /dev/null +++ b/concourse/docker/ubuntu22.04/README.md @@ -0,0 +1,34 @@ +# Cloudberry PL/Java Docker (Ubuntu 22.04 / pxf-style) + +This directory uses the upstream image `apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest`: + +- **Sources are prepared externally** (CI checkout / local sibling directory mount), no `git clone` inside the container +- The container runs scripts to **build Cloudberry from source and create a demo cluster (with standby)** +- Then builds PL/Java and runs the built-in regression tests (`cbdb/tests`) + +## Directory layout + +``` + +/ + cloudberry/ + pljava/ + +``` + +## Run locally + +From `/pljava`: + +```sh +docker compose -f concourse/docker/ubuntu22.04/docker-compose.yml down -v || true +docker compose -f concourse/docker/ubuntu22.04/docker-compose.yml up -d + +docker exec cbdb-pljava bash -lc "bash /home/gpadmin/workspace/pljava/concourse/docker/ubuntu22.04/scripts/entrypoint.sh" +``` + +Run PL/Java regression only (assumes Cloudberry + cluster + PL/Java are ready): + +```sh +docker exec cbdb-pljava bash -lc "source /home/gpadmin/workspace/pljava/concourse/docker/ubuntu22.04/scripts/entrypoint.sh && run_pljava_test_only" +``` diff --git a/concourse/docker/ubuntu22.04/docker-compose.yml b/concourse/docker/ubuntu22.04/docker-compose.yml new file mode 100644 index 0000000..06c5172 --- /dev/null +++ b/concourse/docker/ubuntu22.04/docker-compose.yml @@ -0,0 +1,15 @@ +services: + cbdb-pljava: + platform: linux/amd64 + image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + container_name: cbdb-pljava + hostname: cdw + tty: true + stdin_open: true + volumes: + - ../../..:/home/gpadmin/workspace/pljava:rw + - ../../../../cloudberry:/home/gpadmin/workspace/cloudberry:rw + ports: + - "15432:7000" + - "12222:22" + command: ["tail", "-f", "/dev/null"] diff --git a/concourse/docker/ubuntu22.04/scripts/build_cloudberrry.sh b/concourse/docker/ubuntu22.04/scripts/build_cloudberrry.sh new file mode 100644 index 0000000..ba9894f --- /dev/null +++ b/concourse/docker/ubuntu22.04/scripts/build_cloudberrry.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Ubuntu 22.04 Cloudberry build script (pxf-style). +# - assumes sources are already mounted at ~/workspace/cloudberry +# - builds + installs into /usr/local/cloudberry-db +# - creates a demo cluster (with standby) for pg_regress health checks + +log() { echo "[build_cloudberrry][$(date '+%F %T')] $*"; } +die() { log "ERROR: $*"; exit 1; } + +WORKSPACE="${WORKSPACE:-/home/gpadmin/workspace}" +CLOUDBERRY_SRC="${CLOUDBERRY_SRC:-${WORKSPACE}/cloudberry}" +INSTALL_PREFIX="${INSTALL_PREFIX:-/usr/local/cloudberry-db}" + +if [ ! -d "${CLOUDBERRY_SRC}" ]; then + die "Cloudberry source not found at ${CLOUDBERRY_SRC} (did you mount/checkout cloudberry?)" +fi + +if [ "${FORCE_CLOUDBERRY_BUILD:-}" != "true" ] && \ + [ -f "${INSTALL_PREFIX}/cloudberry-env.sh" ] && \ + [ -f "${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh" ]; then + log "Cloudberry already installed and demo cluster exists; skipping build (set FORCE_CLOUDBERRY_BUILD=true to rebuild)" + exit 0 +fi + +log "install base packages" +sudo apt-get update +sudo apt-get install -y sudo git locales openssh-server iproute2 \ + bison bzip2 cmake curl flex gcc g++ make pkg-config rsync wget tar \ + libapr1-dev libbz2-dev libcurl4-gnutls-dev libevent-dev libkrb5-dev libipc-run-perl \ + libldap2-dev libpam0g-dev libprotobuf-dev libreadline-dev libssl-dev libuv1-dev \ + liblz4-dev libxerces-c-dev libxml2-dev libyaml-dev libzstd-dev libperl-dev \ + protobuf-compiler python3-dev python3-pip python3-setuptools libsnappy-dev + +sudo locale-gen en_US.UTF-8 +sudo update-locale LANG=en_US.UTF-8 + +log "setup ssh keys for gpadmin" +mkdir -p /home/gpadmin/.ssh +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 2048 -C 'apache-cloudberry-dev' -f /home/gpadmin/.ssh/id_rsa -N "" +fi +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 700 /home/gpadmin/.ssh +chmod 600 /home/gpadmin/.ssh/authorized_keys +chmod 644 /home/gpadmin/.ssh/id_rsa.pub + +log "configure resource limits" +sudo tee /etc/security/limits.d/90-db-limits.conf >/dev/null <<'EOF' +gpadmin soft core unlimited +gpadmin hard core unlimited +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 +EOF + +log "prepare install prefix ${INSTALL_PREFIX}" +sudo rm -rf "${INSTALL_PREFIX}" +sudo mkdir -p "${INSTALL_PREFIX}" +sudo chown -R gpadmin:gpadmin "${INSTALL_PREFIX}" + +log "configure Cloudberry" +cd "${CLOUDBERRY_SRC}" +./configure --prefix="${INSTALL_PREFIX}" \ + --disable-external-fts \ + --enable-debug \ + --enable-cassert \ + --enable-debug-extensions \ + --enable-gpcloud \ + --enable-ic-proxy \ + --enable-mapreduce \ + --enable-orafce \ + --enable-orca \ + --disable-pax \ + --enable-pxf \ + --enable-tap-tests \ + --with-gssapi \ + --with-ldap \ + --with-libxml \ + --with-lz4 \ + --with-pam \ + --with-perl \ + --with-pgport=5432 \ + --with-python \ + --with-pythonsrc-ext \ + --with-ssl=openssl \ + --with-uuid=e2fs \ + --with-includes=/usr/include/xercesc + +log "build + install Cloudberry" +make -j"$(nproc)" -C "${CLOUDBERRY_SRC}" +make -j"$(nproc)" -C "${CLOUDBERRY_SRC}/contrib" +make install -C "${CLOUDBERRY_SRC}" +make install -C "${CLOUDBERRY_SRC}/contrib" + +log "create demo cluster" +# shellcheck disable=SC1091 +source "${INSTALL_PREFIX}/cloudberry-env.sh" +make create-demo-cluster -C "${CLOUDBERRY_SRC}" +# shellcheck disable=SC1091 +source "${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh" + +psql -P pager=off template1 -c "select version()" +psql -P pager=off template1 -c "select * from gp_segment_configuration order by dbid" + +log "Cloudberry demo cluster ready (PGPORT=${PGPORT:-})" + diff --git a/concourse/docker/ubuntu22.04/scripts/entrypoint.sh b/concourse/docker/ubuntu22.04/scripts/entrypoint.sh new file mode 100644 index 0000000..83a1737 --- /dev/null +++ b/concourse/docker/ubuntu22.04/scripts/entrypoint.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail + +log() { echo "[entrypoint][$(date '+%F %T')] $*"; } +die() { log "ERROR: $*"; exit 1; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +WORKSPACE="${WORKSPACE:-/home/gpadmin/workspace}" +PLJAVA_SRC="${PLJAVA_SRC:-${WORKSPACE}/pljava}" +CLOUDBERRY_SRC="${CLOUDBERRY_SRC:-${WORKSPACE}/cloudberry}" +INSTALL_PREFIX="${INSTALL_PREFIX:-/usr/local/cloudberry-db}" + +export LANG="${LANG:-en_US.UTF-8}" +export LC_ALL="${LC_ALL:-en_US.UTF-8}" + +setup_ssh() { + log "start sshd" + sudo mkdir -p /var/run/sshd + sudo ssh-keygen -A >/dev/null 2>&1 || true + sudo /usr/sbin/sshd || true + mkdir -p /home/gpadmin/.ssh + touch /home/gpadmin/.ssh/known_hosts + ssh-keyscan -t rsa "$(hostname)" 2>/dev/null >> /home/gpadmin/.ssh/known_hosts || true + chmod 600 /home/gpadmin/.ssh/known_hosts || true +} + +build_cloudberry() { + bash "${SCRIPT_DIR}/build_cloudberrry.sh" +} + +source_cbdb_env() { + if [ -f "${INSTALL_PREFIX}/cloudberry-env.sh" ]; then + # shellcheck disable=SC1091 + source "${INSTALL_PREFIX}/cloudberry-env.sh" + elif [ -f "${INSTALL_PREFIX}/greenplum_path.sh" ]; then + # shellcheck disable=SC1091 + source "${INSTALL_PREFIX}/greenplum_path.sh" + fi + + if [ -f "${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh" ]; then + # shellcheck disable=SC1091 + source "${CLOUDBERRY_SRC}/gpAux/gpdemo/gpdemo-env.sh" + fi +} + +wait_for_cbdb() { + log "wait for Cloudberry to accept connections (PGPORT=${PGPORT:-unset})" + for _ in $(seq 1 120); do + if psql -d template1 -v ON_ERROR_STOP=1 -c "select 1" >/dev/null 2>&1; then + return 0 + fi + sleep 5 + done + die "Cloudberry did not become ready in time" +} + +install_pljava_build_deps() { + log "install PL/Java build deps" + sudo apt-get update + sudo apt-get install -y curl wget tar gcc g++ make libkrb5-dev openssl libssl-dev + + # PL/Java 1.5.x in this fork uses maven-compiler-plugin source/target=6, so build with JDK8. + sudo apt-get install -y openjdk-8-jdk +} + +setup_java8() { + local java8_home="" + for candidate in /usr/lib/jvm/java-8-openjdk-*; do + if [ -d "${candidate}" ]; then + java8_home="${candidate}" + break + fi + done + if [ -z "${java8_home}" ]; then + die "could not find Java 8 under /usr/lib/jvm (is openjdk-8-jdk installed?)" + fi + + export JAVA_HOME="${JAVA_HOME:-${java8_home}}" + export PATH="${JAVA_HOME}/bin:${PATH}" + java -version +} + +setup_maven() { + local maven_version="3.9.6" + if [ ! -x /usr/local/apache-maven/bin/mvn ]; then + wget -nv "https://archive.apache.org/dist/maven/maven-3/${maven_version}/binaries/apache-maven-${maven_version}-bin.tar.gz" \ + -O "/tmp/apache-maven-${maven_version}-bin.tar.gz" + tar xzf "/tmp/apache-maven-${maven_version}-bin.tar.gz" -C /tmp + sudo rm -rf /usr/local/apache-maven + sudo mv "/tmp/apache-maven-${maven_version}" /usr/local/apache-maven + fi + export PATH="/usr/local/apache-maven/bin:${PATH}" + + # Configure a Maven mirror/proxy to avoid network flakiness when reaching + # Maven Central from some environments. + local mirror_url="${MAVEN_MIRROR_URL:-https://maven.aliyun.com/repository/public}" + mkdir -p /home/gpadmin/.m2 + cat > /home/gpadmin/.m2/settings.xml < + + + ${MAVEN_MIRROR_ID:-mirror} + central + ${mirror_url} + + + +EOF + log "configured Maven mirror: ${mirror_url}" + + mvn -version +} + +configure_pljava_runtime() { + # Point Cloudberry at libjvm.so for runtime. + local libjvm + libjvm="$(find "${JAVA_HOME}" -type f -name libjvm.so -path '*server*' | head -n 1 || true)" + [ -n "${libjvm}" ] || die "Could not locate libjvm.so under JAVA_HOME=${JAVA_HOME}" + + local jvm_server_dir jvm_lib_dir jvm_jli_dir + jvm_server_dir="$(dirname "${libjvm}")" + jvm_lib_dir="$(dirname "${jvm_server_dir}")" + jvm_jli_dir="${jvm_lib_dir}/jli" + export LD_LIBRARY_PATH="${jvm_server_dir}:${jvm_lib_dir}:${jvm_jli_dir}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + + gpstop -arf +} + +test_pljava() { + log "run PL/Java built-in regression tests (make installcheck)" + cd "${PLJAVA_SRC}" + mkdir -p "${PLJAVA_SRC}/gpdb/tests/results" + local installcheck_log="${PLJAVA_SRC}/gpdb/tests/results/installcheck.log" + log "installcheck log: ${installcheck_log}" + make installcheck REGRESS_DIR="${CLOUDBERRY_SRC}" 2>&1 | tee "${installcheck_log}" +} + +build_pljava() { + log "build + install PL/Java" + cd "${PLJAVA_SRC}" + make clean + make + make install +} + +build_and_test_pljava() { + build_pljava + configure_pljava_runtime + test_pljava +} + +run_pljava_test_only() { + source_cbdb_env + wait_for_cbdb + install_pljava_build_deps + setup_java8 + setup_maven + configure_pljava_runtime + test_pljava +} + +run_pljava_build_only() { + source_cbdb_env + wait_for_cbdb + install_pljava_build_deps + setup_java8 + setup_maven + build_pljava +} + +main() { + setup_ssh + build_cloudberry + source_cbdb_env + wait_for_cbdb + install_pljava_build_deps + setup_java8 + setup_maven + build_and_test_pljava + log "done" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/gpdb/tests/expected/pljava_test_optimizer.out b/gpdb/tests/expected/pljava_test_optimizer.out index 78d04c4..ba505cd 100644 --- a/gpdb/tests/expected/pljava_test_optimizer.out +++ b/gpdb/tests/expected/pljava_test_optimizer.out @@ -649,8 +649,8 @@ SELECT id,name, salary FROM employees2 order by id; -- s/\(SOMEFILE\:SOMEFUNC\)// -- end_matchsubs SELECT javatest.transferPeople(1) FROM javatest.test; -- should error -ERROR: query plan with multiple segworker groups is not supported -HINT: likely caused by a function that reads or modifies data in a distributed table +ERROR: function cannot execute on a QE slice because it accesses relation "javatest.employees1" +CONTEXT: SQL statement "SELECT id, name, salary FROM employees1 WHERE salary > $1" select javatest.maxFromSetReturnExample(2,10); maxfromsetreturnexample -------------------------