Merge pull request #892 from openstreetmap/use-pg-virtualenv

 Tests: enable running tests inside pg_virtualenv

Author: lonvia

.travis.yml

@@ -56,22 +56,22 @@ matrix:
   # ---- Linux + CLANG ---------------------------
     - os: linux
       compiler: "clang-3.8"
-      env: T="clang38_pg92_dbtest" RUNTEST="#Dbtest" LUAJIT_OPTION="OFF"
+      env: T="clang38_pg92_dbtest" LUAJIT_OPTION="OFF"
            CXXFLAGS="-pedantic -Werror"
            CC=clang-3.8 CXX=clang++-3.8
       addons: *clang38_pg92
 
     - os: linux
       compiler: "clang-7"
-      env: T="clang7_pg96_dbtest_luajit" RUNTEST="#Dbtest" LUAJIT_OPTION="ON"
+      env: T="clang7_pg96_dbtest_luajit" LUAJIT_OPTION="ON"
            CXXFLAGS="-pedantic -Werror"
            CC=clang-7 CXX=clang++-7
       addons: *clang7_pg96
 
   # ---- OSX + CLANG ---------------------------
     - os: osx
       compiler: clang
-      env: T="osx_clang_NoDB" RUNTEST="-L NoDB" LUAJIT_OPTION="OFF"
+      env: T="osx_clang_NoDB" LUAJIT_OPTION="OFF" TEST_NODB=1
            CXXFLAGS="-pedantic -Werror"
       before_install:
         - brew install lua;
@@ -83,26 +83,23 @@ matrix:
   # ---- Linux + GCC ---------------------------
     - os: linux
       compiler: "gcc-4.8"
-      env: T="gcc48_pg96_dbtest"  RUNTEST="#Dbtest" LUAJIT_OPTION="OFF"
+      env: T="gcc48_pg96_dbtest" LUAJIT_OPTION="OFF"
            CXXFLAGS="-pedantic -Werror"
            CC=gcc-4.8 CXX=g++-4.8
       addons: *gcc48_pg96
 
     - os: linux
       compiler: gcc-8
-      env: T="gcc8_pg96_dbtest_luajit" RUNTEST="#Dbtest" LUAJIT_OPTION="ON"
+      env: T="gcc8_pg96_dbtest_luajit" LUAJIT_OPTION="ON"
            CXXFLAGS="-pedantic -Werror"
            CC=gcc-8 CXX=g++-8
       addons: *gcc8_pg96
 
 
 before_install:
-  - sudo mkdir -p /extra/pg/tablespacetest   # for the database test
-  - sudo chown postgres:postgres /extra/pg/tablespacetest
   - dpkg -l | grep -E 'lua|proj|xml|bz2|postgis|zlib|boost|expat'  # checking available versions
 before_script:
   - psql -U postgres -c "SELECT version()"
-  - psql -U postgres -c "CREATE TABLESPACE tablespacetest LOCATION '/extra/pg/tablespacetest'"
   - psql -U postgres -c "CREATE EXTENSION postgis"
   - psql -U postgres -c "CREATE EXTENSION hstore"
   - psql -U postgres -c "SELECT PostGIS_Full_Version()"
@@ -115,9 +112,14 @@ script:
   - cmake .. -DBUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DWITH_LUAJIT=$LUAJIT_OPTION
   - make -j2
   - echo "Running tests ... "
-  - if [[ $RUNTEST ]]; then ctest -VV $RUNTEST; fi
+  - if [[ $TEST_NODB ]]; then
+      ctest -VV -L NoDB;
+    else
+      PG_VERSION=`psql -U postgres -t -c "SELECT version()" | head -n 1 |  cut -d ' ' -f 3 | cut -d . -f 1-2`;
+      pg_virtualenv -v $PG_VERSION ctest -VV;
+    fi
 after_failure:
   - # rerun make, but verbosely
     make VERBOSE=1
 
-# end of .travis
+# end of .travis

CONTRIBUTING.md

@@ -77,8 +77,20 @@ sudo apt-get install python-psycopg2
 ```
 
 Most of these tests depend on being able to set up a database and run osm2pgsql
-against it. You need to ensure that PostgreSQL is running and that your user is
-a superuser of that system. To do that, run:
+against it. This is most easily done using ``pg_virtualenv``. Just run
+
+```sh
+pg_virtualenv ctest
+```
+
+``pg_virtualenv`` creates a separate postgres server instance. The test databases
+are created in this instance and the complete server is destroyed after the
+tests are finished. ctest also calls appropriate fixtures that create the
+separate tablespace required for some tests.
+
+When running without ``pg_virtualenv``, you need to ensure that PostgreSQL is
+running and that your user is a superuser of that system. You also need to
+create an appropriate test tablespace manually. To do that, run:
 
 ```sh
 sudo -u postgres createuser -s $USER
@@ -94,9 +106,6 @@ to be a bug, please check to see if it is a known issue at
 https://github.com/openstreetmap/osm2pgsql/issues and, if it's not
 already known, report it there.
 
-If running the tests in a virtual machine, allocate sufficient disk space for a
-20GB flat nodes file.
-
 ### Performance Testing
 
 If performance testing with a full planet import is required, indicate what

tests/CMakeLists.txt

@@ -63,6 +63,23 @@ endforeach()
 set_property(TEST test-middle-flat PROPERTY LABELS FlatNodes)
 set_property(TEST test-output-pgsql PROPERTY LABELS FlatNodes)
 
+# Fixture for creating test tablespace under a pg_virtualenv
+if (NOT WIN32)
+  message(STATUS "Added tablespace fixture...")
+  add_test(NAME FixtureTablespaceSetup
+      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/fixture-tablespace-setup
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+  add_test(NAME FixtureTablespaceCleanup
+      COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/fixture-tablespace-cleanup
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+  set_tests_properties(FixtureTablespaceSetup PROPERTIES FIXTURES_SETUP Tablespace)
+  set_tests_properties(FixtureTablespaceCleanup PROPERTIES FIXTURES_CLEANUP Tablespace)
+endif()
+
+set_tests_properties(test-output-pgsql-tablespace
+                     PROPERTIES FIXTURES_REQUIRED Tablespace)
+
 if (NOT HAVE_LUA)
   # these tests require LUA support
   set_tests_properties(test-output-multi-poly-trivial PROPERTIES WILL_FAIL on)
@@ -77,6 +94,8 @@ if (PYTHONINTERP_FOUND)
   set_tests_properties(regression-test-pbf PROPERTIES TIMEOUT ${TESTING_TIMEOUT})
   set_tests_properties(regression-test-pbf
       PROPERTIES ENVIRONMENT "HAVE_LUA=${LUA_FOUND}")
+  set_tests_properties(regression-test-pbf
+                       PROPERTIES FIXTURES_REQUIRED Tablespace)
   message(STATUS "Added test: regression-test-pbf (needs Python with psycopg2 module)")
 else()
   message(WARNING "Can not find python, regression test disabled")

tests/fixture-tablespace-cleanup

@@ -0,0 +1,11 @@
+#!/bin/sh
+#
+# Remove any custom tablespaces we might have created.
+
+BASEDIR=`pwd`
+
+TBLDIR=${BASEDIR}/osm2pgsql-test-tablespace
+
+if find $TBLDIR -maxdepth 1 -mindepth 1 -type d -name 'PG*'; then
+  rm -r $TBLDIR/PG*
+fi

tests/fixture-tablespace-setup

@@ -0,0 +1,25 @@
+#!/bin/sh
+#
+# Creates the required tablespace for tests.
+#
+# Only do this when we are running under pg_virtualenv.
+# Then the tablespace can be owned by the current user.
+
+if [ "x$PG_CLUSTER_CONF_ROOT" != "x" ]; then
+  BASEDIR=`pwd`
+
+  TBLDIR=${BASEDIR}/osm2pgsql-test-tablespace
+
+  if [ -d "$TBLDIR" ]; then
+    echo "Tablespace directory already exists. Cleaning up."
+
+    if find $TBLDIR -maxdepth 1 -mindepth 1 -type d -name 'PG*'; then
+        rm -r $TBLDIR/PG*
+    fi
+  else
+    mkdir $TBLDIR
+    chmod 777 $TBLDIR
+  fi
+
+  psql -c "CREATE TABLESPACE tablespacetest LOCATION '$TBLDIR'" postgres
+fi