diff --git a/packaging_qa/Debian.bookworm.dockerfile b/packaging_qa/Debian.bookworm.dockerfile new file mode 100644 index 00000000..f0c21927 --- /dev/null +++ b/packaging_qa/Debian.bookworm.dockerfile @@ -0,0 +1,62 @@ +# Use Debian as the base image +FROM debian:bookworm + +# Set environment variables to avoid interactive prompts +ENV DEBIAN_FRONTEND=noninteractive +ARG VERSION=13.0.1 +ENV CITUS_VERSION=${VERSION}.citus-1 +ENV PG_MAJOR=17 + +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + sudo + +# Add PostgreSQL repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | tee /etc/apt/trusted.gpg.d/postgresql.asc && \ + echo "deb http://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list + +# Install PostgreSQL $PG_MAJOR and required packages +RUN apt-get update && apt-get install -y postgresql-$PG_MAJOR postgresql-contrib + +# Install Citus repository and required extensions +RUN curl -s https://install.citusdata.com/community/deb.sh | bash && \ + apt-get update && apt-get install -y \ + postgresql-$PG_MAJOR-citus-13.0=$CITUS_VERSION \ + postgresql-$PG_MAJOR-hll=2.18.citus-1 \ + postgresql-$PG_MAJOR-topn=2.7.0.citus-1 + +# Ensure PostgreSQL data directory exists and has correct permissions +RUN mkdir -p /var/lib/postgresql/$PG_MAJOR/main && chown -R postgres:postgres /var/lib/postgresql && chmod -R 700 /var/lib/postgresql + +# Switch to postgres user before running PostgreSQL +USER postgres + +# Initialize the database only if it's not already initialized +RUN bash -c '[ ! -f "/var/lib/postgresql/${PG_MAJOR}/main/PG_VERSION" ] && /usr/lib/postgresql/${PG_MAJOR}/bin/initdb -D /var/lib/postgresql/${PG_MAJOR}/main || echo "Database already initialized"' + +# Fix pg_hba.conf to allow connections +RUN echo "local all postgres trust" > /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "local all all md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "host all all 127.0.0.1/32 md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "host all all ::1/128 md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf + +# Add Citus to PostgreSQL config +RUN echo "shared_preload_libraries = 'citus'" >> /var/lib/postgresql/$PG_MAJOR/main/postgresql.conf + +# Create an entrypoint script to start PostgreSQL properly +USER root +COPY docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +# Switch back to postgres user +USER postgres + +# Use custom entrypoint to start PostgreSQL and create the Citus extension +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +# Expose PostgreSQL port +EXPOSE 5432 diff --git a/packaging_qa/Debian.bullseye.dockerfile b/packaging_qa/Debian.bullseye.dockerfile new file mode 100644 index 00000000..571da83b --- /dev/null +++ b/packaging_qa/Debian.bullseye.dockerfile @@ -0,0 +1,62 @@ +# Use Debian as the base image +FROM debian:bullseye + +# Set environment variables to avoid interactive prompts +ENV DEBIAN_FRONTEND=noninteractive +ARG VERSION=13.0.1 +ENV CITUS_VERSION=${VERSION}.citus-1 +ENV PG_MAJOR=15 + +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + sudo + +# Add PostgreSQL repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | tee /etc/apt/trusted.gpg.d/postgresql.asc && \ + echo "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list + +# Install PostgreSQL $PG_MAJOR and required packages +RUN apt-get update && apt-get install -y postgresql-$PG_MAJOR postgresql-contrib + +# Install Citus repository and required extensions +RUN curl -s https://install.citusdata.com/community/deb.sh | bash && \ + apt-get update && apt-get install -y \ + postgresql-$PG_MAJOR-citus-13.0=$CITUS_VERSION \ + postgresql-$PG_MAJOR-hll=2.18.citus-1 \ + postgresql-$PG_MAJOR-topn=2.7.0.citus-1 + +# Ensure PostgreSQL data directory exists and has correct permissions +RUN mkdir -p /var/lib/postgresql/$PG_MAJOR/main && chown -R postgres:postgres /var/lib/postgresql && chmod -R 700 /var/lib/postgresql + +# Switch to postgres user before running PostgreSQL +USER postgres + +# Initialize the database only if it's not already initialized +RUN bash -c '[ ! -f "/var/lib/postgresql/${PG_MAJOR}/main/PG_VERSION" ] && /usr/lib/postgresql/${PG_MAJOR}/bin/initdb -D /var/lib/postgresql/${PG_MAJOR}/main || echo "Database already initialized"' + +# Fix pg_hba.conf to allow connections +RUN echo "local all postgres trust" > /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "local all all md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "host all all 127.0.0.1/32 md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "host all all ::1/128 md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf + +# Add Citus to PostgreSQL config +RUN echo "shared_preload_libraries = 'citus'" >> /var/lib/postgresql/$PG_MAJOR/main/postgresql.conf + +# Create an entrypoint script to start PostgreSQL properly +USER root +COPY docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +# Switch back to postgres user +USER postgres + +# Use custom entrypoint to start PostgreSQL and create the Citus extension +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +# Expose PostgreSQL port +EXPOSE 5432 diff --git a/packaging_qa/Redhat.dockerfile b/packaging_qa/Redhat.dockerfile new file mode 100644 index 00000000..66b902f0 --- /dev/null +++ b/packaging_qa/Redhat.dockerfile @@ -0,0 +1,78 @@ +# RHEL 8-based EL8 rockylinux:8 / ubi8 +# RHEL 9-based EL9 rockylinux:9 / ubi9 +# Oracle Linux 8 OL8 oraclelinux:8 # may need to run docker pull oraclelinux:8 +# Oracle Linux 9 OL9 oraclelinux:9 +# Use Red Hat or Oracle Linux as the base image +ARG OS_TYPE=oraclelinux +ARG VERSION_ID=8 +ARG BASE_IMAGE=${OS_TYPE}:${VERSION_ID} +FROM $BASE_IMAGE + +# Set environment variables to avoid interactive prompts +ARG CITUS_VERSION_FULL=13.0.1 +ARG CITUS_VERSION_COMPACT=130 +ARG PG_MAJOR=17 + +# Install dependencies +RUN dnf install -y --allowerasing \ + epel-release \ + dnf-utils \ + sudo \ + curl \ + gnupg2 \ + which \ + libffi-devel \ + readline-devel \ + gcc \ + make \ + which \ + tar + +# Add PostgreSQL repository based on EL version +RUN export EL_VERSION=$(source /etc/os-release && echo ${VERSION_ID%%.*}) && \ + dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-${EL_VERSION}-x86_64/pgdg-redhat-repo-latest.noarch.rpm + +# Disable built-in PostgreSQL module +RUN dnf -qy module disable postgresql + +# Install PostgreSQL and required packages +RUN dnf install -y postgresql${PG_MAJOR} postgresql${PG_MAJOR}-server postgresql${PG_MAJOR}-contrib + +# Install Citus repository and extensions +RUN export VERSION_ID=$(source /etc/os-release && echo ${VERSION_ID%%.*}) && \ + curl -s https://packagecloud.io/install/repositories/citusdata/community/script.rpm.sh | sudo bash && \ + yum install -y citus${CITUS_VERSION_COMPACT}_${PG_MAJOR}-${CITUS_VERSION_FULL}.citus-1.el${VERSION_ID}.x86_64 \ + topn_${PG_MAJOR}-2.7.0.citus-1.el${VERSION_ID}.x86_64 \ + hll_${PG_MAJOR}-2.18.citus-1.el${VERSION_ID}.x86_64 + +# Ensure PostgreSQL data directory exists and has correct permissions +RUN mkdir -p /var/lib/pgsql/${PG_MAJOR}/data && chown -R postgres:postgres /var/lib/pgsql && chmod -R 700 /var/lib/pgsql + +# Switch to postgres user before running PostgreSQL +USER postgres + +# Initialize the database only if it's not already initialized +RUN [ ! -f "/var/lib/pgsql/${PG_MAJOR}/data/PG_VERSION" ] && /usr/pgsql-${PG_MAJOR}/bin/initdb -D /var/lib/pgsql/${PG_MAJOR}/data || echo "Database already initialized" + +# Fix pg_hba.conf to allow connections +RUN echo "local all postgres trust" > /var/lib/pgsql/${PG_MAJOR}/data/pg_hba.conf && \ + echo "local all all md5" >> /var/lib/pgsql/${PG_MAJOR}/data/pg_hba.conf && \ + echo "host all all 127.0.0.1/32 md5" >> /var/lib/pgsql/${PG_MAJOR}/data/pg_hba.conf && \ + echo "host all all ::1/128 md5" >> /var/lib/pgsql/${PG_MAJOR}/data/pg_hba.conf + +# Add Citus to PostgreSQL config +RUN echo "shared_preload_libraries = 'citus'" >> /var/lib/pgsql/${PG_MAJOR}/data/postgresql.conf + +# Create an entrypoint script to start PostgreSQL properly +USER root +COPY docker-entrypoint-rhel.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint-rhel.sh + +# Switch back to postgres user +USER postgres + +# Use custom entrypoint to start PostgreSQL and create the Citus extension +ENTRYPOINT ["/usr/local/bin/docker-entrypoint-rhel.sh"] + +# Expose PostgreSQL port +EXPOSE 5432 diff --git a/packaging_qa/Ubuntu.dockerfile b/packaging_qa/Ubuntu.dockerfile new file mode 100644 index 00000000..b1ade86a --- /dev/null +++ b/packaging_qa/Ubuntu.dockerfile @@ -0,0 +1,66 @@ +# Use Ubuntu 22.04 Jammy as the base image +# FROM ubuntu:22.04 + +# Use Ubuntu 20.04 Focal as the base image +FROM ubuntu:20.04 + + +# Set environment variables to avoid interactive prompts +ENV DEBIAN_FRONTEND=noninteractive +ARG VERSION=13.0.1 +ENV CITUS_VERSION=${VERSION}.citus-1 +ENV PG_MAJOR=17 + +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + sudo + +# Add PostgreSQL repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | tee /etc/apt/trusted.gpg.d/postgresql.asc && \ + echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list + +# Install PostgreSQL $PG_MAJOR and required packages +RUN apt-get update && apt-get install -y postgresql-$PG_MAJOR postgresql-contrib + +# Install Citus repository and required extensions +RUN curl -s https://install.citusdata.com/community/deb.sh | bash && \ + apt-get update && apt-get install -y \ + postgresql-$PG_MAJOR-citus-13.0=$CITUS_VERSION \ + postgresql-$PG_MAJOR-hll=2.18.citus-1 \ + postgresql-$PG_MAJOR-topn=2.7.0.citus-1 + +# Ensure PostgreSQL data directory exists and has correct permissions +RUN mkdir -p /var/lib/postgresql/$PG_MAJOR/main && chown -R postgres:postgres /var/lib/postgresql && chmod -R 700 /var/lib/postgresql + +# Switch to postgres user before running PostgreSQL +USER postgres + +# Initialize the database only if it's not already initialized +RUN bash -c '[ ! -f "/var/lib/postgresql/${PG_MAJOR}/main/PG_VERSION" ] && /usr/lib/postgresql/${PG_MAJOR}/bin/initdb -D /var/lib/postgresql/${PG_MAJOR}/main || echo "Database already initialized"' + +# Fix pg_hba.conf to allow connections +RUN echo "local all postgres trust" > /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "local all all md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "host all all 127.0.0.1/32 md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf && \ + echo "host all all ::1/128 md5" >> /var/lib/postgresql/$PG_MAJOR/main/pg_hba.conf + +# Add Citus to PostgreSQL config +RUN echo "shared_preload_libraries = 'citus'" >> /var/lib/postgresql/$PG_MAJOR/main/postgresql.conf + +# Create an entrypoint script to start PostgreSQL properly +USER root +COPY docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +# Switch back to postgres user +USER postgres + +# Use custom entrypoint to start PostgreSQL and create the Citus extension +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + +# Expose PostgreSQL port +EXPOSE 5432 diff --git a/packaging_qa/build_run b/packaging_qa/build_run new file mode 100644 index 00000000..e849ef40 --- /dev/null +++ b/packaging_qa/build_run @@ -0,0 +1,14 @@ +docker build -t citus-test -f Ubuntu.dockerfile . +docker build -t citus-test -f Debian.bookworm.dockerfile . + +docker rm -f citus-container +docker run -d --name citus-container citus-test +docker exec -it citus-container psql -U postgres + +docker exec -i citus-container psql -U postgres < test.sql + + +docker logs -f citus-container + +docker exec -it citus-container cat /etc/os-release + diff --git a/packaging_qa/docker-entrypoint-rhel.sh b/packaging_qa/docker-entrypoint-rhel.sh new file mode 100644 index 00000000..62ff08ee --- /dev/null +++ b/packaging_qa/docker-entrypoint-rhel.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +# Set PG_MAJOR to 15 by default if not provided +if command -v psql &> /dev/null; then + PG_MAJOR=$(psql -V | awk '{print $3}' | cut -d. -f1) +else + PG_MAJOR=15 +fi + +# Define PostgreSQL paths dynamically +PG_CTL="/usr/pgsql-${PG_MAJOR}/bin/pg_ctl" +PG_DATA_DIR="/var/lib/pgsql/${PG_MAJOR}/data" +PG_LOG="/var/lib/pgsql/logfile" +PG_BIN="/usr/pgsql-${PG_MAJOR}/bin/postgres" +PG_ISREADY="/usr/pgsql-${PG_MAJOR}/bin/pg_isready" + +# Ensure PostgreSQL data directory exists +if [ ! -d "$PG_DATA_DIR" ]; then + echo "Initializing PostgreSQL data directory: $PG_DATA_DIR" + mkdir -p "$PG_DATA_DIR" + chown -R postgres:postgres "$PG_DATA_DIR" + chmod 700 "$PG_DATA_DIR" + /usr/pgsql-${PG_MAJOR}/bin/initdb -D "$PG_DATA_DIR" +fi + +# Start PostgreSQL in the background +$PG_CTL -D "$PG_DATA_DIR" -l "$PG_LOG" start + +# Wait for PostgreSQL to be ready +until $PG_ISREADY -q -d postgres; do + sleep 1 +done + +# Ensure Citus extension is installed +psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS citus;" +psql -U postgres -c "SELECT * FROM pg_extension WHERE extname = 'citus';" + +# Stop PostgreSQL before running the container foreground process +$PG_CTL -D "$PG_DATA_DIR" stop + +# Start PostgreSQL in the foreground +exec $PG_BIN -D "$PG_DATA_DIR" diff --git a/packaging_qa/docker-entrypoint.sh b/packaging_qa/docker-entrypoint.sh new file mode 100644 index 00000000..9a1365a7 --- /dev/null +++ b/packaging_qa/docker-entrypoint.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e + +# Set PG_MAJOR to 16 by default if not provided +PG_MAJOR=${PG_MAJOR:-16} + +# Define PostgreSQL paths dynamically +PG_CTL="/usr/lib/postgresql/${PG_MAJOR}/bin/pg_ctl" +PG_DATA_DIR="/var/lib/postgresql/${PG_MAJOR}/main" +PG_LOG="/var/lib/postgresql/logfile" +PG_BIN="/usr/lib/postgresql/${PG_MAJOR}/bin/postgres" + +# Ensure PostgreSQL data directory exists +if [ ! -d "$PG_DATA_DIR" ]; then + echo "Error: PostgreSQL data directory $PG_DATA_DIR does not exist!" + exit 1 +fi + +# Start PostgreSQL in the background +$PG_CTL -D "$PG_DATA_DIR" -l "$PG_LOG" start + +# Wait for PostgreSQL to be ready +until pg_isready -q -d postgres; do + sleep 1 +done + +# Ensure Citus extension is installed +psql -U postgres -c "CREATE EXTENSION IF NOT EXISTS citus;" +psql -U postgres -c "SELECT * FROM pg_extension WHERE extname = 'citus';" + +# Stop PostgreSQL before running the container foreground process +$PG_CTL -D "$PG_DATA_DIR" stop + +# Start PostgreSQL in the foreground +exec $PG_BIN -D "$PG_DATA_DIR" diff --git a/packaging_qa/test.sql b/packaging_qa/test.sql new file mode 100644 index 00000000..3f8988c6 --- /dev/null +++ b/packaging_qa/test.sql @@ -0,0 +1,45 @@ +SELECT pg_read_file('/etc/os-release', 0, 1000); +select version(); +\dx + +CREATE TABLE events ( + device_id bigint, + event_id bigserial, + event_time timestamptz default now(), + data jsonb not null, + PRIMARY KEY (device_id, event_id) +); + +-- distribute the events table across shards placed locally or on the worker nodes +SELECT create_distributed_table('events', 'device_id'); + +INSERT INTO events (device_id, data) +SELECT s % 100, ('{"measurement":'||random()||'}')::jsonb FROM generate_series(1,1000000) s; + +-- get the last 3 events for device 1, routed to a single node +SELECT * FROM events WHERE device_id = 1 ORDER BY event_time DESC, event_id DESC LIMIT 3; + + +CREATE TABLE devices ( + device_id bigint primary key, + device_name text, + device_type_id int +); +CREATE INDEX ON devices (device_type_id); + +-- co-locate the devices table with the events table +SELECT create_distributed_table('devices', 'device_id', colocate_with := 'events'); + +-- insert device metadata +INSERT INTO devices (device_id, device_name, device_type_id) +SELECT s, 'device-'||s, 55 FROM generate_series(0, 99) s; + +-- optionally: make sure the application can only insert events for a known device +ALTER TABLE events ADD CONSTRAINT device_id_fk +FOREIGN KEY (device_id) REFERENCES devices (device_id); + +-- get the average measurement across all devices of type 55, parallelized across shards +SELECT avg((data->>'measurement')::double precision) +FROM events JOIN devices USING (device_id) +WHERE device_type_id = 55; +