Travis fixes (fix test_ptrack_vacuum_full test) (#14)

* [travis] fix test_ptrack_vacuum_full
Remove unused PG_VERSION (PG_MAJOR) variable
Reworking working directories (helps to run tap test)
Remove unused amcheck install

Author: kulaginm

.travis.yml

@@ -21,26 +21,26 @@ notifications:
 
 # keep in sync with codecov.yml number of builds
 env:
-    - PG_VERSION=15 PG_BRANCH=master TEST_CASE=tap
-    - PG_VERSION=15 PG_BRANCH=master TEST_CASE=tap MODE=legacy
-    - PG_VERSION=15 PG_BRANCH=master TEST_CASE=all
-    - PG_VERSION=15 PG_BRANCH=master TEST_CASE=all MODE=paranoia
-    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=tap
-    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy
-    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=all
-    - PG_VERSION=14 PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia
-    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap
-    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy
-    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all
-    - PG_VERSION=13 PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia
-    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=tap
-    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy
-    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all
-    - PG_VERSION=12 PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia
-    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap
-    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy
-    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all
-    - PG_VERSION=11 PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia
+    - PG_BRANCH=master TEST_CASE=tap
+    - PG_BRANCH=master TEST_CASE=tap MODE=legacy
+#    - PG_BRANCH=master TEST_CASE=all
+    - PG_BRANCH=master TEST_CASE=all MODE=paranoia
+    - PG_BRANCH=REL_14_STABLE TEST_CASE=tap
+    - PG_BRANCH=REL_14_STABLE TEST_CASE=tap MODE=legacy
+#    - PG_BRANCH=REL_14_STABLE TEST_CASE=all
+    - PG_BRANCH=REL_14_STABLE TEST_CASE=all MODE=paranoia
+    - PG_BRANCH=REL_13_STABLE TEST_CASE=tap
+    - PG_BRANCH=REL_13_STABLE TEST_CASE=tap MODE=legacy
+#    - PG_BRANCH=REL_13_STABLE TEST_CASE=all
+    - PG_BRANCH=REL_13_STABLE TEST_CASE=all MODE=paranoia
+    - PG_BRANCH=REL_12_STABLE TEST_CASE=tap
+    - PG_BRANCH=REL_12_STABLE TEST_CASE=tap MODE=legacy
+#    - PG_BRANCH=REL_12_STABLE TEST_CASE=all
+    - PG_BRANCH=REL_12_STABLE TEST_CASE=all MODE=paranoia
+    - PG_BRANCH=REL_11_STABLE TEST_CASE=tap
+    - PG_BRANCH=REL_11_STABLE TEST_CASE=tap MODE=legacy
+#    - PG_BRANCH=REL_11_STABLE TEST_CASE=all
+    - PG_BRANCH=REL_11_STABLE TEST_CASE=all MODE=paranoia
 
 jobs:
     allow_failures:

Dockerfile.in

@@ -5,21 +5,21 @@ RUN apt-get update
 RUN apt-get -yq install python python-pip python-virtualenv
 
 # Environment
-ENV PG_MAJOR=${PG_VERSION} PG_BRANCH=${PG_BRANCH}
-ENV LANG=C.UTF-8 PGHOME=/pg/testdir/pgbin
+ENV PG_BRANCH=${PG_BRANCH}
+ENV LANG=C.UTF-8 PGHOME=/testdir/pgbin
 ENV MODE=${MODE} TEST_CASE=${TEST_CASE} TEST_REPEATS=${TEST_REPEATS}
 
 # Make directories
-RUN	mkdir -p /pg/testdir
+RUN	mkdir -p /testdir
 
 COPY run_tests.sh /run.sh
 RUN chmod 755 /run.sh
 
-COPY . /pg/testdir
-WORKDIR /pg/testdir
+COPY . /testdir/ptrack
+WORKDIR /testdir
 
 # Grant privileges
-RUN	chown -R postgres:postgres /pg/testdir
+RUN	chown -R postgres:postgres /testdir
 
 USER postgres
 ENTRYPOINT /run.sh

README.md

@@ -151,7 +151,6 @@ Feel free to [send pull requests](https://github.com/postgrespro/ptrack/compare)
 Everything is tested automatically with [travis-ci.com](https://travis-ci.com/postgrespro/ptrack) and [codecov.io](https://codecov.io/gh/postgrespro/ptrack), but you can also run tests locally via `Docker`:
 
 ```sh
-export PG_VERSION=14
 export PG_BRANCH=REL_14_STABLE
 export TEST_CASE=all
 export MODE=paranoia

docker-compose.yml

@@ -1,3 +1,17 @@
-tests:
-    privileged: true
-    build: .
+version: "3.7"
+services:
+    tests:
+        build:
+            context: .
+
+        cap_add:
+            - SYS_PTRACE
+
+        security_opt:
+            - seccomp=unconfined
+
+        # don't work
+        #sysctls:
+        #    kernel.yama.ptrace_scope: 0
+        privileged: true
+

make_dockerfile.sh

@@ -1,10 +1,5 @@
 #!/usr/bin/env sh
 
-if [ -z ${PG_VERSION+x} ]; then
-	echo PG_VERSION is not set!
-	exit 1
-fi
-
 if [ -z ${PG_BRANCH+x} ]; then
 	echo PG_BRANCH is not set!
 	exit 1
@@ -28,11 +23,9 @@ else
 	echo TEST_REPEATS=${TEST_REPEATS}
 fi
 
-echo PG_VERSION=${PG_VERSION}
 echo PG_BRANCH=${PG_BRANCH}
 
 sed \
-	-e 's/${PG_VERSION}/'${PG_VERSION}/g \
 	-e 's/${PG_BRANCH}/'${PG_BRANCH}/g \
 	-e 's/${MODE}/'${MODE}/g \
 	-e 's/${TEST_CASE}/'${TEST_CASE}/g \

run_tests.sh

@@ -1,55 +1,47 @@
 #!/usr/bin/env bash
 
 #
-# Copyright (c) 2019-2020, Postgres Professional
+# Copyright (c) 2019-2021, Postgres Professional
 #
 
-PG_SRC=$PWD/postgres
+PTRACK_SRC=${PWD}/ptrack
+PG_SRC=${PWD}/postgres
+PBK_SRC=${PWD}/pg_probackup
 status=0
 
-# curl "https://ftp.postgresql.org/pub/source/v$PG_VERSION/postgresql-$PG_VERSION.tar.bz2" -o postgresql.tar.bz2
-# echo "$PG_SHA256 *postgresql.tar.bz2" | sha256sum -c -
-
-# mkdir $PG_SRC
-
-# tar \
-# 	--extract \
-# 	--file postgresql.tar.bz2 \
-# 	--directory $PG_SRC \
-# 	--strip-components 1
-
+#########################################################
 # Clone Postgres
 echo "############### Getting Postgres sources"
-git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1
+git clone https://github.com/postgres/postgres.git --depth=1 --branch=${PG_BRANCH} ${PG_SRC}
 
 # Clone pg_probackup
 echo "############### Getting pg_probackup sources"
-git clone https://github.com/postgrespro/pg_probackup.git --depth=1 -b master
-# git clone https://github.com/ololobus/pg_probackup.git --depth=1 -b ptrack-tests
+git clone https://github.com/postgrespro/pg_probackup.git --depth=1 --branch=master ${PBK_SRC}
 
+#########################################################
 # Compile and install Postgres
-cd postgres # Go to postgres dir
+cd ${PG_SRC} # Go to postgres dir
 
 echo "############### Applying ptrack patch"
-git apply -v -3 ../patches/$PG_BRANCH-ptrack-core.diff
+git apply --verbose --3way ${PTRACK_SRC}/patches/${PG_BRANCH}-ptrack-core.diff
 
-if [ "$MODE" = "paranoia" ]; then
+if [ "${MODE}" = "paranoia" ]; then
     echo "############### Paranoia mode: applying turn-off-hint-bits.diff"
-    git apply -v -3 ../patches/turn-off-hint-bits.diff
+    git apply --verbose --3way ${PTRACK_SRC}/patches/turn-off-hint-bits.diff
 fi
 
 echo "############### Compiling Postgres"
-if [ "$TEST_CASE" = "tap" ] && [ "$MODE" = "legacy" ]; then
-    ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests
+if [ "${TEST_CASE}" = "tap" ] && [ "${MODE}" = "legacy" ]; then
+    ./configure CFLAGS='-DEXEC_BACKEND' --disable-atomics --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet
 else
-    ./configure --prefix=$PGHOME --enable-debug --enable-cassert --enable-depend --enable-tap-tests
+    ./configure --prefix=${PGHOME} --enable-debug --enable-cassert --enable-depend --enable-tap-tests --quiet
 fi
-make -s -j$(nproc) install
-make -s -j$(nproc) -C contrib/ install
+make --quiet --jobs=$(nproc) install
+make --quiet --jobs=$(nproc) --directory=contrib/ install
 
 # Override default Postgres instance
-export PATH=$PGHOME/bin:$PATH
-export LD_LIBRARY_PATH=$PGHOME/lib
+export PATH=${PGHOME}/bin:${PATH}
+export LD_LIBRARY_PATH=${PGHOME}/lib
 export PG_CONFIG=$(which pg_config)
 
 # Show pg_config path (just in case)
@@ -60,48 +52,37 @@ which pg_config
 echo "############### pg_config"
 pg_config
 
-# Get amcheck if missing
-if [ ! -d "contrib/amcheck" ]; then
-    echo "############### Getting missing amcheck"
-    git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck
-    make USE_PGXS=1 -C contrib/amcheck install
-fi
-
-# Get back to testdir
-cd ..
-
+#########################################################
 # Build and install ptrack extension
 echo "############### Compiling and installing ptrack extension"
+cp --recursive ${PTRACK_SRC} ${PG_SRC}/contrib/ptrack
+make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" --directory=${PG_SRC}/contrib/ptrack/ install
 
-# XXX: Hackish way to make possible to run tap tests
-mkdir $PG_SRC/contrib/ptrack
-cp * $PG_SRC/contrib/ptrack/
-cp -R t $PG_SRC/contrib/ptrack/
-
-make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" -C $PG_SRC/contrib/ptrack/ install
-
-if [ "$TEST_CASE" = "tap" ]; then
+if [ "${TEST_CASE}" = "tap" ]; then
 
     # Run tap tests
     echo "############### Running tap tests"
-    if [ "$MODE" = "legacy" ]; then
+    if [ "${MODE}" = "legacy" ]; then
         # There is a known issue with attaching shared memory segment using the same
         # address each time, when EXEC_BACKEND mechanism is turned on.  It happens due
         # to the ASLR address space randomization, so we are trying to attach a segment
         # to the already occupied location.  That way we simply turning off ASLR here.
         #
         # Postgres comment: https://github.com/postgres/postgres/blob/5cbfce562f7cd2aab0cdc4694ce298ec3567930e/src/backend/postmaster/postmaster.c#L4929
-        setarch x86_64 --addr-no-randomize make -C postgres/contrib/ptrack check || status=$?
+        setarch x86_64 --addr-no-randomize make --directory=${PG_SRC}/contrib/ptrack check || status=$?
     else
-        make -C postgres/contrib/ptrack check || status=$?
+        make --directory=${PG_SRC}/contrib/ptrack check || status=$?
     fi
 
 else
+    # Set kernel params (used for debugging -- probackup tests)
+    echo "############### setting kernel params"
+    sudo sh -c 'echo 0 > /proc/sys/kernel/yama/ptrace_scope'
 
     # Build and install pg_probackup
     echo "############### Compiling and installing pg_probackup"
-    cd pg_probackup # Go to pg_probackup dir
-    make USE_PGXS=1 top_srcdir=$PG_SRC install
+    cd ${PBK_SRC} # Go to pg_probackup dir
+    make USE_PGXS=1 top_srcdir=${PG_SRC} install
 
     # Setup python environment
     echo "############### Setting up python env"
@@ -110,35 +91,36 @@ else
     pip install testgres==1.8.2
 
     echo "############### Testing"
-    if [ "$MODE" = "basic" ]; then
+    export PG_PROBACKUP_PTRACK=ON
+    if [ "${MODE}" = "basic" ]; then
         export PG_PROBACKUP_TEST_BASIC=ON
-    elif [ "$MODE" = "paranoia" ]; then
+    elif [ "${MODE}" = "paranoia" ]; then
         export PG_PROBACKUP_PARANOIA=ON
     fi
 
-    export PG_PROBACKUP_PTRACK=ON
-    if [ "$TEST_CASE" = "all" ]; then
+    if [ "${TEST_CASE}" = "all" ]; then
         # Run all pg_probackup ptrack tests
-        python3 -m unittest -v tests.ptrack || status=$?
+        PBK_TEST_CASE=tests.ptrack
     else
-        for i in `seq $TEST_REPEATS`; do
-            python3 -m unittest -v tests.ptrack.PtrackTest.$TEST_CASE || status=$?
-        done
+        PBK_TEST_CASE=tests.ptrack.PtrackTest.${TEST_CASE}
     fi
+    for i in `seq ${TEST_REPEATS}`; do
+        python3 -m unittest -v ${PBK_TEST_CASE} || status=$?
+    done
 
     # Exit virtualenv
     deactivate
-
-    # Get back to testdir
-    cd ..
-
 fi
 
+#########################################################
+# codecov
+echo "############### Codecov"
+cd ${PTRACK_SRC}
 # Generate *.gcov files
-gcov $PG_SRC/contrib/ptrack/*.c $PG_SRC/contrib/ptrack/*.h
+gcov ${PG_SRC}/contrib/ptrack/*.c ${PG_SRC}/contrib/ptrack/*.h
 
 # Send coverage stats to Codecov
 bash <(curl -s https://codecov.io/bash)
 
 # Something went wrong, exit with code 1
-if [ $status -ne 0 ]; then exit 1; fi
+if [ ${status} -ne 0 ]; then exit 1; fi