Skip to content
This repository was archived by the owner on Sep 12, 2025. It is now read-only.

Commit 2c8302c

Browse files
edespinoedespino
committed
build: add Apache Cloudberry build automation and test framework
Introduces a comprehensive build automation framework for Apache Cloudberry: - Configure/build/test automation scripts with standardized logging - Unit test and regression test execution framework - Demo cluster creation and management utilities - Test results parsing and reporting tools - Shared utility functions for common operations The framework provides: - Consistent environment setup and teardown - Standardized logging and error handling - Clear exit codes and error reporting - GitHub Actions integration support - RPM package building capabilities This commit establishes the foundation for Apache Cloudberry (incubating) CI/CD pipeline and release automation.
1 parent 6b8f893 commit 2c8302c

File tree

9 files changed

+1064
-0
lines changed

9 files changed

+1064
-0
lines changed

‎build_automation/cloudberry/scripts/build-cloudberry.sh‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/bin/bash
2+
# --------------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed
6+
# with this work for additional information regarding copyright
7+
# ownership. The ASF licenses this file to You under the Apache
8+
# License, Version 2.0 (the "License"); you may not use this file
9+
# except in compliance with the License. You may obtain a copy of the
10+
# License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
17+
# implied. See the License for the specific language governing
18+
# permissions and limitations under the License.
19+
#
20+
# --------------------------------------------------------------------
21+
#
22+
# Script: build-cloudberry.sh
23+
# Description: Builds Apache Cloudberry from source code and installs
24+
# it.
25+
# Performs the following steps:
26+
# 1. Builds main Apache Cloudberry database components
27+
# 2. Builds contrib modules
28+
# 3. Installs both main and contrib components
29+
# Uses parallel compilation based on available CPU cores.
30+
#
31+
# Required Environment Variables:
32+
# SRC_DIR - Root source directory containing Apache Cloudberry
33+
# source code
34+
#
35+
# Optional Environment Variables:
36+
# LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs)
37+
# NPROC - Number of parallel jobs (defaults to all available cores)
38+
#
39+
# Usage:
40+
# Export required variables:
41+
# export SRC_DIR=/path/to/cloudberry/source
42+
# Then run:
43+
# ./build-cloudberry.sh
44+
#
45+
# Prerequisites:
46+
# - configure-cloudberry.sh must be run first
47+
# - Required build dependencies must be installed
48+
# - /usr/local/cloudberry-db/lib must exist and be writable
49+
#
50+
# Exit Codes:
51+
# 0 - Build and installation completed successfully
52+
# 1 - Environment setup failed (missing SRC_DIR, LOG_DIR creation failed)
53+
# 2 - Main component build failed
54+
# 3 - Contrib build failed
55+
# 4 - Installation failed
56+
#
57+
# --------------------------------------------------------------------
58+
59+
set -euo pipefail
60+
61+
# Source common utilities
62+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
63+
source "${SCRIPT_DIR}/cloudberry-utils.sh"
64+
65+
# Define log directory and files
66+
export LOG_DIR="${SRC_DIR}/build-logs"
67+
BUILD_LOG="${LOG_DIR}/build.log"
68+
69+
# Initialize environment
70+
init_environment "Cloudberry Build Script" "${BUILD_LOG}"
71+
72+
# Set environment
73+
log_section "Environment Setup"
74+
export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH
75+
log_section_end "Environment Setup"
76+
77+
# Build process
78+
log_section "Build Process"
79+
execute_cmd make -j$(nproc) --directory ${SRC_DIR} || exit 2
80+
execute_cmd make -j$(nproc) --directory ${SRC_DIR}/contrib || exit 3
81+
log_section_end "Build Process"
82+
83+
# Installation
84+
log_section "Installation"
85+
execute_cmd make install --directory ${SRC_DIR} || exit 4
86+
execute_cmd make install --directory ${SRC_DIR}/contrib || exit 4
87+
log_section_end "Installation"
88+
89+
# Log completion
90+
log_completion "Cloudberry Build Script" "${BUILD_LOG}"
91+
exit 0

‎build_automation/cloudberry/scripts/cloudberry-utils.sh‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
#!/bin/bash
2+
# --------------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed
6+
# with this work for additional information regarding copyright
7+
# ownership. The ASF licenses this file to You under the Apache
8+
# License, Version 2.0 (the "License"); you may not use this file
9+
# except in compliance with the License. You may obtain a copy of the
10+
# License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
17+
# implied. See the License for the specific language governing
18+
# permissions and limitations under the License.
19+
#
20+
# --------------------------------------------------------------------
21+
#
22+
# Library: cloudberry-utils.sh
23+
# Description: Common utility functions for Apache Cloudberry build
24+
# and test scripts
25+
#
26+
# Required Environment Variables:
27+
# SRC_DIR - Root source directory
28+
#
29+
# Optional Environment Variables:
30+
# LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs)
31+
#
32+
# Functions:
33+
# init_environment "Script Name" "Log File"
34+
# - Initialize logging and verify environment
35+
# - Parameters:
36+
# * script_name: Name of the calling script
37+
# * log_file: Path to log file
38+
# - Returns: 0 on success, 1 on failure
39+
#
40+
# execute_cmd command [args...]
41+
# - Execute command with logging
42+
# - Parameters: Command and its arguments
43+
# - Returns: Command's exit code
44+
#
45+
# run_psql_cmd "sql_command"
46+
# - Execute PostgreSQL command with logging
47+
# - Parameters: SQL command string
48+
# - Returns: psql command's exit code
49+
#
50+
# source_cloudberry_env
51+
# - Source Cloudberry environment files
52+
# - Returns: 0 on success
53+
#
54+
# log_section "section_name"
55+
# - Log section start
56+
# - Parameters: Name of the section
57+
#
58+
# log_section_end "section_name"
59+
# - Log section end
60+
# - Parameters: Name of the section
61+
#
62+
# log_completion "script_name" "log_file"
63+
# - Log script completion
64+
# - Parameters:
65+
# * script_name: Name of the calling script
66+
# * log_file: Path to log file
67+
#
68+
# Usage:
69+
# source ./cloudberry-utils.sh
70+
#
71+
# Example:
72+
# source ./cloudberry-utils.sh
73+
# init_environment "My Script" "${LOG_FILE}"
74+
# execute_cmd make clean
75+
# log_section "Build Process"
76+
# execute_cmd make -j$(nproc)
77+
# log_section_end "Build Process"
78+
# log_completion "My Script" "${LOG_FILE}"
79+
#
80+
# --------------------------------------------------------------------
81+
82+
# Initialize logging and environment
83+
init_environment() {
84+
local script_name=$1
85+
local log_file=$2
86+
87+
echo "=== Initializing environment for ${script_name} ==="
88+
echo "${script_name} executed at $(date)" | tee -a "${log_file}"
89+
echo "Whoami: $(whoami)" | tee -a "${log_file}"
90+
echo "Hostname: $(hostname)" | tee -a "${log_file}"
91+
echo "Working directory: $(pwd)" | tee -a "${log_file}"
92+
echo "Source directory: ${SRC_DIR}" | tee -a "${log_file}"
93+
echo "Log directory: ${LOG_DIR}" | tee -a "${log_file}"
94+
95+
if [ -z "${SRC_DIR:-}" ]; then
96+
echo "Error: SRC_DIR environment variable is not set" | tee -a "${log_file}"
97+
exit 1
98+
fi
99+
100+
mkdir -p "${LOG_DIR}"
101+
}
102+
103+
# Function to echo and execute command with logging
104+
execute_cmd() {
105+
local cmd_str="$*"
106+
local timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
107+
echo "Executing at ${timestamp}: $cmd_str" | tee -a "${LOG_DIR}/commands.log"
108+
"$@" 2>&1 | tee -a "${LOG_DIR}/commands.log"
109+
return ${PIPESTATUS[0]}
110+
}
111+
112+
# Function to run psql commands with logging
113+
run_psql_cmd() {
114+
local cmd=$1
115+
local timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
116+
echo "Executing psql at ${timestamp}: $cmd" | tee -a "${LOG_DIR}/psql-commands.log"
117+
psql -P pager=off template1 -c "$cmd" 2>&1 | tee -a "${LOG_DIR}/psql-commands.log"
118+
return ${PIPESTATUS[0]}
119+
}
120+
121+
# Function to source Cloudberry environment
122+
source_cloudberry_env() {
123+
echo "=== Sourcing Cloudberry environment ===" | tee -a "${LOG_DIR}/environment.log"
124+
source /usr/local/cloudberry-db/greenplum_path.sh
125+
source ${SRC_DIR}/../cloudberry/gpAux/gpdemo/gpdemo-env.sh
126+
}
127+
128+
# Function to log section start
129+
log_section() {
130+
local section_name=$1
131+
local timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
132+
echo "=== ${section_name} started at ${timestamp} ===" | tee -a "${LOG_DIR}/sections.log"
133+
}
134+
135+
# Function to log section end
136+
log_section_end() {
137+
local section_name=$1
138+
local timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
139+
echo "=== ${section_name} completed at ${timestamp} ===" | tee -a "${LOG_DIR}/sections.log"
140+
}
141+
142+
# Function to log script completion
143+
log_completion() {
144+
local script_name=$1
145+
local log_file=$2
146+
local timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
147+
echo "${script_name} execution completed successfully at ${timestamp}" | tee -a "${log_file}"
148+
}

‎build_automation/cloudberry/scripts/configure-cloudberry.sh‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#!/bin/bash
2+
# --------------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed
6+
# with this work for additional information regarding copyright
7+
# ownership. The ASF licenses this file to You under the Apache
8+
# License, Version 2.0 (the "License"); you may not use this file
9+
# except in compliance with the License. You may obtain a copy of the
10+
# License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
17+
# implied. See the License for the specific language governing
18+
# permissions and limitations under the License.
19+
#
20+
# --------------------------------------------------------------------
21+
#
22+
# Script: configure-cloudberry.sh
23+
# Description: Configures Apache Cloudberry build environment and runs
24+
# ./configure with optimized settings. Performs the
25+
# following:
26+
# 1. Prepares /usr/local/cloudberry-db directory
27+
# 2. Sets up library dependencies
28+
# 3. Configures build with required features enabled
29+
#
30+
# Configuration Features:
31+
# - Cloud Storage Integration (gpcloud)
32+
# - IC Proxy Support
33+
# - MapReduce Processing
34+
# - Oracle Compatibility (orafce)
35+
# - ORCA Query Optimizer
36+
# - PXF External Table Access
37+
# - Test Automation Support (tap-tests)
38+
#
39+
# System Integration:
40+
# - GSSAPI Authentication
41+
# - LDAP Authentication
42+
# - XML Processing
43+
# - LZ4 Compression
44+
# - OpenSSL Support
45+
# - PAM Authentication
46+
# - Perl Support
47+
# - Python Support
48+
#
49+
# Required Environment Variables:
50+
# SRC_DIR - Root source directory
51+
#
52+
# Optional Environment Variables:
53+
# LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs)
54+
#
55+
# Prerequisites:
56+
# - System dependencies must be installed:
57+
# * xerces-c development files
58+
# * OpenSSL development files
59+
# * Python development files
60+
# * Perl development files
61+
# * LDAP development files
62+
# - /usr/local must be writable
63+
# - User must have sudo privileges
64+
#
65+
# Usage:
66+
# Export required variables:
67+
# export SRC_DIR=/path/to/cloudberry/source
68+
# Then run:
69+
# ./configure-cloudberry.sh
70+
#
71+
# Exit Codes:
72+
# 0 - Configuration completed successfully
73+
# 1 - Environment setup failed
74+
# 2 - Directory preparation failed
75+
# 3 - Library setup failed
76+
# 4 - Configure command failed
77+
#
78+
# --------------------------------------------------------------------
79+
80+
set -euo pipefail
81+
82+
# Source common utilities
83+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
84+
source "${SCRIPT_DIR}/cloudberry-utils.sh"
85+
86+
# Define log directory and files
87+
export LOG_DIR="${SRC_DIR}/build-logs"
88+
CONFIGURE_LOG="${LOG_DIR}/configure.log"
89+
90+
# Initialize environment
91+
init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}"
92+
93+
# Initial setup
94+
log_section "Initial Setup"
95+
execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2
96+
execute_cmd sudo chmod a+w /usr/local || exit 2
97+
execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2
98+
execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \
99+
/usr/local/xerces-c/lib/libxerces-c-3.3.so \
100+
/usr/local/cloudberry-db/lib || exit 3
101+
execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2
102+
log_section_end "Initial Setup"
103+
104+
# Set environment
105+
log_section "Environment Setup"
106+
export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH
107+
log_section_end "Environment Setup"
108+
109+
# Configure build
110+
log_section "Configure"
111+
execute_cmd ./configure --prefix=/usr/local/cloudberry-db \
112+
--disable-external-fts \
113+
--enable-gpcloud \
114+
--enable-ic-proxy \
115+
--enable-mapreduce \
116+
--enable-orafce \
117+
--enable-orca \
118+
--enable-pxf \
119+
--enable-tap-tests \
120+
--with-gssapi \
121+
--with-ldap \
122+
--with-libxml \
123+
--with-lz4 \
124+
--with-openssl \
125+
--with-pam \
126+
--with-perl \
127+
--with-pgport=5432 \
128+
--with-python \
129+
--with-pythonsrc-ext \
130+
--with-ssl=openssl \
131+
--with-openssl \
132+
--with-uuid=e2fs \
133+
--with-includes=/usr/local/xerces-c/include \
134+
--with-libraries=/usr/local/cloudberry-db/lib || exit 4
135+
log_section_end "Configure"
136+
137+
# Capture version information
138+
log_section "Version Information"
139+
execute_cmd ag "GP_VERSION | GP_VERSION_NUM | PG_VERSION | PG_VERSION_NUM | PG_VERSION_STR" src/include/pg_config.h
140+
log_section_end "Version Information"
141+
142+
# Log completion
143+
log_completion "Cloudberry Configure Script" "${CONFIGURE_LOG}"
144+
exit 0

0 commit comments

Comments
 (0)