CI: add binary swap CI for REL_2_STABLE #3

Workflow file for this run

.github/workflows/binary-swap-check.yml at c88bd13

	# --------------------------------------------------------------------
	#
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed
	# with this work for additional information regarding copyright
	# ownership. The ASF licenses this file to You under the Apache
	# License, Version 2.0 (the "License"); you may not use this file
	# except in compliance with the License. You may obtain a copy of the
	# License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
	# implied. See the License for the specific language governing
	# permissions and limitations under the License.
	#
	# --------------------------------------------------------------------
	# GitHub Actions Workflow: Apache Cloudberry Binary Swap Compatibility Check
	# --------------------------------------------------------------------
	#
	# This workflow detects binary/catalog incompatibilities by comparing the
	# current PR/push code against the latest release tag.
	#
	# Workflow:
	# 1. Resolve baseline: Find the latest release tag from REL_2_STABLE
	# 2. Build baseline: Build RPM from the baseline tag (parallel)
	# 3. Build current: Build RPM from current PR/push code (parallel)
	# 4. Binary swap test: Install baseline, create cluster, swap to current,
	# verify data integrity
	#
	# Triggers:
	# - Pull requests to `REL_2_STABLE`
	# - Push to `REL_2_STABLE`
	# - Manual workflow dispatch
	#
	# --------------------------------------------------------------------

	name: Binary Swap Compatibility Check

	on:
	push:
	branches: [REL_2_STABLE]
	pull_request:
	branches: [REL_2_STABLE]
	types: [opened, synchronize, reopened]
	workflow_dispatch:
	inputs:
	baseline_ref:
	description: 'Baseline ref to compare against (leave empty for auto-detect latest tag)'
	required: false
	default: ''
	type: string

	concurrency:
	group: binary-swap-${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	permissions:
	contents: read
	packages: read
	actions: read

	env:
	LOG_RETENTION_DAYS: 7
	CBDB_INSTALL_DIR: /usr/local/cloudberry-db
	BASELINE_INSTALL_DIR: /usr/local/cloudberry-db-baseline
	CURRENT_INSTALL_DIR: /usr/local/cloudberry-db-current

	jobs:

	## ======================================================================
	## Job: resolve-baseline
	## ======================================================================

	resolve-baseline:
	name: Resolve Baseline Version
	runs-on: ubuntu-22.04
	outputs:
	baseline_ref: ${{ steps.resolve.outputs.baseline_ref }}
	baseline_version: ${{ steps.resolve.outputs.baseline_version }}
	steps:
	- name: Checkout (for git ls-remote)
	uses: actions/checkout@v4
	with:
	fetch-depth: 1

	- name: Resolve latest release tag
	id: resolve
	run: \|
	set -eo pipefail

	# Use manual input if provided
	if [[ -n "${{ github.event.inputs.baseline_ref }}" ]]; then
	echo "Using manually specified baseline: ${{ github.event.inputs.baseline_ref }}"
	echo "baseline_ref=${{ github.event.inputs.baseline_ref }}" >> "$GITHUB_OUTPUT"
	echo "baseline_version=${{ github.event.inputs.baseline_ref }}" >> "$GITHUB_OUTPUT"
	exit 0
	fi

	# Find the latest release tag
	# Tag format: X.Y.Z-incubating (e.g., 2.0.0-incubating, 2.1.0-incubating)
	# Exclude RC versions: X.Y.Z-incubating-rcN
	echo "Finding latest release tag..."

	# Get all tags matching the incubating pattern, sort by version, filter out RC versions
	TAG_INFO=$(git ls-remote --tags --refs origin '*-incubating' 2>/dev/null \
	\| grep -v '\-rc[0-9]*$' \
	\| sort -t'/' -k3 -V \
	\| tail -1 \|\| echo "")

	if [[ -z "$TAG_INFO" ]]; then
	echo "::error::No release tags found matching pattern '*-incubating' (excluding RC versions)"
	echo "::error::Expected format: X.Y.Z-incubating (e.g., 2.0.0-incubating)"
	exit 1
	fi

	BASELINE_REF=$(echo "$TAG_INFO" \| awk '{print $1}')
	BASELINE_VERSION=$(echo "$TAG_INFO" \| awk '{print $2}' \| sed 's\|refs/tags/\|\|')

	echo "Found baseline: ${BASELINE_VERSION} (${BASELINE_REF})"
	echo "baseline_ref=${BASELINE_REF}" >> "$GITHUB_OUTPUT"
	echo "baseline_version=${BASELINE_VERSION}" >> "$GITHUB_OUTPUT"

	## ======================================================================
	## Job: build-baseline
	## ======================================================================

	build-baseline:
	name: Build Baseline RPM
	needs: [resolve-baseline]
	runs-on: ubuntu-22.04
	timeout-minutes: 120

	container:
	image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
	options: >-
	--user root
	-h cdw
	-v /usr/share:/host_usr_share
	-v /usr/local:/host_usr_local
	-v /opt:/host_opt

	steps:
	- name: Free Disk Space
	run: \|
	echo "=== Disk space before cleanup ==="
	df -h /
	rm -rf /host_opt/hostedtoolcache \|\| true
	rm -rf /host_usr_local/lib/android \|\| true
	rm -rf /host_usr_share/dotnet \|\| true
	rm -rf /host_opt/ghc \|\| true
	rm -rf /host_usr_local/.ghcup \|\| true
	rm -rf /host_usr_share/swift \|\| true
	rm -rf /host_usr_local/share/powershell \|\| true
	rm -rf /host_usr_local/share/chromium \|\| true
	rm -rf /host_usr_share/miniconda \|\| true
	rm -rf /host_opt/az \|\| true
	rm -rf /host_usr_share/sbt \|\| true
	echo "=== Disk space after cleanup ==="
	df -h /

	- name: Checkout Build Tools
	uses: actions/checkout@v4
	with:
	ref: main
	path: build_tools
	sparse-checkout: \|
	devops
	fetch-depth: 1

	- name: Checkout Baseline Code
	uses: actions/checkout@v4
	with:
	ref: ${{ needs.resolve-baseline.outputs.baseline_version }}
	fetch-depth: 1
	submodules: recursive
	path: baseline_src

	- name: Inject Build Tools
	run: \|
	# Copy devops scripts to baseline source if they don't exist
	if [ ! -d "baseline_src/devops" ]; then
	echo "Injecting devops scripts from main branch..."
	cp -r build_tools/devops baseline_src/
	fi

	- name: Environment Initialization
	run: \|
	set -eo pipefail
	if ! su - gpadmin -c "/tmp/init_system.sh"; then
	echo "::error::Container initialization failed"
	exit 1
	fi
	chown -R gpadmin:gpadmin .
	chmod -R 755 .
	rm -rf /__t/*
	echo "SRC_DIR=${GITHUB_WORKSPACE}/baseline_src" >> "$GITHUB_ENV"

	# Create log directory required by build scripts
	mkdir -p "${GITHUB_WORKSPACE}/baseline_src/build-logs"
	chown gpadmin:gpadmin "${GITHUB_WORKSPACE}/baseline_src/build-logs"

	- name: Configure
	env:
	SRC_DIR: ${{ github.workspace }}/baseline_src
	run: \|
	set -eo pipefail
	chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
	time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=false ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"

	- name: Build
	env:
	SRC_DIR: ${{ github.workspace }}/baseline_src
	run: \|
	set -eo pipefail
	chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh
	time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"

	- name: Create RPM
	env:
	CBDB_VERSION: ${{ needs.resolve-baseline.outputs.baseline_version }}
	SRC_DIR: ${{ github.workspace }}/baseline_src
	run: \|
	set -eo pipefail
	rpmdev-setuptree
	ln -s "${SRC_DIR}"/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec "${HOME}"/rpmbuild/SPECS/ \|\| true
	cp "${SRC_DIR}"/LICENSE /usr/local/cloudberry-db \|\| true
	"${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "${CBDB_VERSION}" --release "1"
	os_version=$(grep -oP '(?<=^VERSION_ID=")[0-9]' /etc/os-release)
	cp "${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-*.rpm "${GITHUB_WORKSPACE}/"

	- name: Upload Baseline RPM
	uses: actions/upload-artifact@v4
	with:
	name: baseline-rpm
	retention-days: ${{ env.LOG_RETENTION_DAYS }}
	if-no-files-found: error
	path: \|
	*.rpm
	!debuginfo.rpm

	## ======================================================================
	## Job: build-current
	## ======================================================================

	build-current:
	name: Build Current RPM
	runs-on: ubuntu-22.04
	timeout-minutes: 120

	container:
	image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
	options: >-
	--user root
	-h cdw
	-v /usr/share:/host_usr_share
	-v /usr/local:/host_usr_local
	-v /opt:/host_opt

	steps:
	- name: Free Disk Space
	run: \|
	echo "=== Disk space before cleanup ==="
	df -h /
	rm -rf /host_opt/hostedtoolcache \|\| true
	rm -rf /host_usr_local/lib/android \|\| true
	rm -rf /host_usr_share/dotnet \|\| true
	rm -rf /host_opt/ghc \|\| true
	rm -rf /host_usr_local/.ghcup \|\| true
	rm -rf /host_usr_share/swift \|\| true
	rm -rf /host_usr_local/share/powershell \|\| true
	rm -rf /host_usr_local/share/chromium \|\| true
	rm -rf /host_usr_share/miniconda \|\| true
	rm -rf /host_opt/az \|\| true
	rm -rf /host_usr_share/sbt \|\| true
	echo "=== Disk space after cleanup ==="
	df -h /

	- name: Checkout Current Code
	uses: actions/checkout@v4
	with:
	fetch-depth: 1
	submodules: recursive

	- name: Checkout Build Tools
	uses: actions/checkout@v4
	with:
	ref: main
	path: build_tools
	sparse-checkout: \|
	devops
	fetch-depth: 1

	- name: Inject Build Tools
	run: \|
	# Copy devops scripts to current source if they don't exist
	if [ ! -d "devops" ]; then
	echo "Injecting devops scripts from main branch..."
	cp -r build_tools/devops .
	fi

	- name: Environment Initialization
	run: \|
	set -eo pipefail
	if ! su - gpadmin -c "/tmp/init_system.sh"; then
	echo "::error::Container initialization failed"
	exit 1
	fi
	chown -R gpadmin:gpadmin .
	chmod -R 755 .
	rm -rf /__t/*
	echo "SRC_DIR=${GITHUB_WORKSPACE}" >> "$GITHUB_ENV"

	# Create log directory required by build scripts
	mkdir -p "${GITHUB_WORKSPACE}/build-logs"
	chown gpadmin:gpadmin "${GITHUB_WORKSPACE}/build-logs"

	- name: Configure
	env:
	SRC_DIR: ${{ github.workspace }}
	run: \|
	set -eo pipefail
	chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh
	time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=false ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"

	- name: Build
	env:
	SRC_DIR: ${{ github.workspace }}
	run: \|
	set -eo pipefail
	chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh
	time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"

	- name: Create RPM
	env:
	CBDB_VERSION: 99.0.0-current
	SRC_DIR: ${{ github.workspace }}
	run: \|
	set -eo pipefail
	rpmdev-setuptree
	ln -s "${SRC_DIR}"/devops/build/packaging/rpm/apache-cloudberry-db-incubating.spec "${HOME}"/rpmbuild/SPECS/ \|\| true
	cp "${SRC_DIR}"/LICENSE /usr/local/cloudberry-db \|\| true
	"${SRC_DIR}"/devops/build/packaging/rpm/build-rpm.sh --version "99.0.0" --release "current"
	cp "${HOME}"/rpmbuild/RPMS/x86_64/apache-cloudberry-db-incubating-*.rpm "${SRC_DIR}/"

	- name: Compile pg_regress
	run: \|
	# Use the environment where Cloudberry was built/installed
	source "${CBDB_INSTALL_DIR}/cloudberry-env.sh"

	# Compile pg_regress
	make -C "${SRC_DIR}/src/test/regress" pg_regress

	# Prepare artifact directory
	mkdir -p "${HOME}/regress_bin"
	cp "${SRC_DIR}/src/test/regress/pg_regress" "${HOME}/regress_bin/"
	# Copy any necessary shared libraries or init files if needed
	cp "${SRC_DIR}/src/test/regress/init_file" "${HOME}/regress_bin/" \|\| true

	- name: Upload pg_regress
	uses: actions/upload-artifact@v4
	with:
	name: pg-regress-bin
	retention-days: ${{ env.LOG_RETENTION_DAYS }}
	path: ${{ env.HOME }}/regress_bin/

	- name: Upload Current RPM
	uses: actions/upload-artifact@v4
	with:
	name: current-rpm
	retention-days: ${{ env.LOG_RETENTION_DAYS }}
	if-no-files-found: error
	path: \|
	*.rpm
	!debuginfo.rpm

	## ======================================================================
	## Job: binary-swap-test
	## ======================================================================

	binary-swap-test:
	name: Binary Swap Test
	needs: [resolve-baseline, build-baseline, build-current]
	runs-on: ubuntu-22.04
	timeout-minutes: 60

	container:
	image: apache/incubator-cloudberry:cbdb-test-rocky9-latest
	options: >-
	--privileged
	--user root
	--hostname cdw
	--shm-size=2gb
	--ulimit core=-1
	-v /usr/share:/host_usr_share
	-v /usr/local:/host_usr_local
	-v /opt:/host_opt

	steps:
	- name: Free Disk Space
	run: \|
	df -h /
	rm -rf /host_opt/hostedtoolcache \|\| true
	rm -rf /host_usr_local/lib/android \|\| true
	rm -rf /host_usr_share/dotnet \|\| true
	df -h /

	- name: Checkout (for test scripts)
	uses: actions/checkout@v4
	with:
	fetch-depth: 1

	- name: Environment Initialization
	run: \|
	set -eo pipefail
	if ! su - gpadmin -c "/tmp/init_system.sh"; then
	echo "::error::Container initialization failed"
	exit 1
	fi
	mkdir -p binary-swap-logs/details
	chown -R gpadmin:gpadmin .
	chmod -R 755 .
	echo "SRC_DIR=${GITHUB_WORKSPACE}" >> "$GITHUB_ENV"
	echo "LOGS_DIR=${GITHUB_WORKSPACE}/binary-swap-logs" >> "$GITHUB_ENV"

	- name: Download Baseline RPM
	uses: actions/download-artifact@v4
	with:
	name: baseline-rpm
	path: ${{ github.workspace }}/baseline_rpm

	- name: Download Current RPM
	uses: actions/download-artifact@v4
	with:
	name: current-rpm
	path: ${{ github.workspace }}/current_rpm

	- name: Download pg_regress
	uses: actions/download-artifact@v4
	with:
	name: pg-regress-bin
	path: ${{ github.workspace }}/regress_bin

	- name: Install Baseline RPM
	run: \|
	set -eo pipefail
	BASELINE_RPM=$(ls "${GITHUB_WORKSPACE}"/baseline_rpm/*.rpm \| head -1)
	echo "Installing baseline RPM: ${BASELINE_RPM}"
	rpm -ivh "${BASELINE_RPM}" --prefix="${BASELINE_INSTALL_DIR}"
	chown -R gpadmin:gpadmin "${BASELINE_INSTALL_DIR}"
	ls -la "${BASELINE_INSTALL_DIR}"

	- name: Install Current RPM
	run: \|
	set -eo pipefail
	CURRENT_RPM=$(ls "${GITHUB_WORKSPACE}"/current_rpm/*.rpm \| head -1)
	echo "Installing current RPM: ${CURRENT_RPM}"
	rpm -ivh "${CURRENT_RPM}" --prefix="${CURRENT_INSTALL_DIR}"
	chown -R gpadmin:gpadmin "${CURRENT_INSTALL_DIR}"
	ls -la "${CURRENT_INSTALL_DIR}"

	- name: Setup pg_regress
	run: \|
	# The test_binary_swap.sh script expects pg_regress at ../regress/pg_regress
	# We are running from src/test/binary_swap, so it looks for src/test/regress/pg_regress
	mkdir -p "${SRC_DIR}/src/test/regress"
	cp "${GITHUB_WORKSPACE}/regress_bin/pg_regress" "${SRC_DIR}/src/test/regress/"
	cp "${GITHUB_WORKSPACE}/regress_bin/init_file" "${SRC_DIR}/src/test/regress/" \|\| true
	chmod +x "${SRC_DIR}/src/test/regress/pg_regress"
	ls -la "${SRC_DIR}/src/test/regress/"

	- name: Run Binary Swap Tests
	id: binary-swap-test
	run: \|
	set -eo pipefail
	TEST_RESULT=0

	# Initialize cluster required by the script?
	# No, test_binary_swap.sh starts binaries itself, BUT it assumes a data directory exists or is created?
	# Actually, test_binary_swap.sh expects the user to provide -m MDD.
	# It assumes the cluster is INITITALIZED.
	# So we need to initialize a cluster first using Baseline.

	ln -sfn "${BASELINE_INSTALL_DIR}" "${CBDB_INSTALL_DIR}"
	su - gpadmin -c "
	source ${CBDB_INSTALL_DIR}/cloudberry-env.sh
	cd ${GITHUB_WORKSPACE}/gpAux/gpdemo
	make cluster
	gpstop -a
	"

	# Now run the test script
	# The script will:
	# 1. Start baseline (-b)
	# 2. Dump
	# 3. Swap to current (-c)
	# 4. Dump & Verify

	su - gpadmin -c "
	export MASTER_DATA_DIRECTORY=${GITHUB_WORKSPACE}/gpAux/gpdemo/datadirs/qddir/demoDataDir-1
	export PGPORT=15432
	cd ${SRC_DIR}/src/test/binary_swap
	./test_binary_swap.sh -b ${BASELINE_INSTALL_DIR} -c ${CURRENT_INSTALL_DIR} -p 15432
	" 2>&1 \| tee -a "${LOGS_DIR}/details/test_binary_swap.log" \|\| TEST_RESULT=$?

	if [[ $TEST_RESULT -ne 0 ]]; then
	echo "::error::Binary swap test script failed"
	echo "test_passed=false" >> "$GITHUB_OUTPUT"
	exit 1
	fi
	echo "test_passed=true" >> "$GITHUB_OUTPUT"

	- name: Generate Report
	if: always()
	env:
	BASELINE_VERSION: ${{ needs.resolve-baseline.outputs.baseline_version }}
	TEST_PASSED: ${{ steps.binary-swap-test.outputs.test_passed }}
	run: \|
	cat > "${LOGS_DIR}/binary-swap-report.md" <<EOF
	# Binary Swap Compatibility Report

	## Summary

	\| Check \| Result \|
	\|-------\|--------\|
	\| Baseline Version \| ${BASELINE_VERSION} \|
	\| Binary Swap Test \| $([ "${TEST_PASSED}" == "true" ] && echo "✅ Passed" \|\| echo "❌ Failed") \|

	## Result

	EOF

	if [[ "${TEST_PASSED}" == "true" ]]; then
	echo "✅ COMPATIBLE: Binary swap test passed successfully." >> "${LOGS_DIR}/binary-swap-report.md"
	else
	echo "❌ INCOMPATIBLE: Binary swap test failed." >> "${LOGS_DIR}/binary-swap-report.md"
	fi

	cat "${LOGS_DIR}/binary-swap-report.md"

	- name: Upload Logs
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: binary-swap-logs
	retention-days: ${{ env.LOG_RETENTION_DAYS }}
	path: \|
	binary-swap-logs/
	dump_*.sql

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

CI: add binary swap CI for REL_2_STABLE #3

Workflow file

CI: add binary swap CI for REL_2_STABLE #3

Uh oh!

Workflow file for this run