JDBC Tests With PG AUDIT Enable #684
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: JDBC Tests With PG AUDIT Enable | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # runs every midnight | |
| jobs: | |
| run-babelfish-jdbc-tests-with-pg-audit: | |
| strategy: | |
| matrix: | |
| extension_branch: [BABEL_4_X_DEV,BABEL_5_X_DEV] | |
| env: | |
| INSTALL_DIR: psql | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| id: checkout | |
| - name: Install Dependencies | |
| id: install-dependencies | |
| if: always() | |
| uses: ./.github/composite-actions/install-dependencies | |
| - name: Determine Engine Branch | |
| id: determine-engine-branch | |
| run: | | |
| if [[ "${{ matrix.extension_branch }}" == "BABEL_4_X_DEV" ]]; then | |
| echo "ENGINE_BRANCH=BABEL_4_X_DEV__PG_16_X" >> $GITHUB_OUTPUT | |
| elif [[ "${{ matrix.extension_branch }}" == "BABEL_5_X_DEV" ]]; then | |
| echo "ENGINE_BRANCH=BABEL_5_X_DEV__PG_17_X" >> $GITHUB_OUTPUT | |
| elif [[ "${{ matrix.extension_branch }}" == "latest" ]]; then | |
| echo "ENGINE_BRANCH=BABEL_5_X_DEV__PG_17_X" >> $GITHUB_OUTPUT | |
| else | |
| echo "ENGINE_BRANCH=main" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Build Modified Postgres | |
| id: build-modified-postgres | |
| if: always() && steps.install-dependencies.outcome == 'success' | |
| uses: ./.github/composite-actions/build-modified-postgres | |
| with: | |
| install_dir: 'psql' | |
| engine_branch: ${{ steps.determine-engine-branch.outputs.ENGINE_BRANCH }} | |
| - name: Compile ANTLR | |
| id: compile-antlr | |
| if: always() && steps.build-modified-postgres.outcome == 'success' | |
| uses: ./.github/composite-actions/compile-antlr | |
| - name: Build Extensions | |
| id: build-extensions | |
| if: always() && steps.compile-antlr.outcome == 'success' | |
| uses: ./.github/composite-actions/build-extensions | |
| with: | |
| extension_branch: ${{ matrix.extension_branch }} | |
| - name: Build tds_fdw Extension | |
| id: build-tds_fdw-extension | |
| if: always() && steps.build-extensions.outcome == 'success' | |
| uses: ./.github/composite-actions/build-tds_fdw-extension | |
| - name: Build vector Extension | |
| id: build-vector-extension | |
| if: always() && steps.build-tds_fdw-extension.outcome == 'success' | |
| uses: ./.github/composite-actions/build-vector-extension | |
| - name: Build PostGIS Extension | |
| id: build-postgis-extension | |
| if: always() && steps.build-vector-extension.outcome == 'success' | |
| uses: ./.github/composite-actions/build-postgis-extension | |
| - name: Install Extensions | |
| id: install-extensions | |
| if: always() && steps.build-postgis-extension.outcome == 'success' | |
| uses: ./.github/composite-actions/install-extensions | |
| with: | |
| wal_level: logical | |
| - name: Adding PgAudit Patch | |
| run: | | |
| cat > ~/pgaudit_executor_hook_null_check.patch << 'EOF' | |
| --- a/pgaudit.c | |
| +++ b/pgaudit.c | |
| @@ -1529,7 +1529,7 @@ pgaudit_ExecutorCheckPerms_hook(List *rangeTabls, | |
| auditEventStack->auditEvent.permInfos = permInfos; | |
| } | |
| } | |
| - else | |
| + else if (auditEventStack != NULL) | |
| { | |
| STACK_NOT_EMPTY(); | |
| log_select_dml(auditOid, rangeTabls, permInfos); | |
| EOF | |
| shell: bash | |
| - name: Build and Install PgAudit | |
| run: | | |
| cd ~ | |
| rm -rf pgaudit | |
| # Determine PgAudit version based on Babelfish branch | |
| if [[ "${{ matrix.extension_branch }}" == 'BABEL_5_X_DEV' ]]; then | |
| git clone --depth 1 --branch REL_17_STABLE https://github.com/pgaudit/pgaudit.git | |
| cd pgaudit | |
| # Apply patch to fix BABEL-4600 | |
| patch -p1 < ~/pgaudit_executor_hook_null_check.patch | |
| make install USE_PGXS=1 PG_CONFIG=~/psql/bin/pg_config | |
| elif [[ "${{ matrix.extension_branch }}" == 'BABEL_4_X_DEV' ]]; then | |
| git clone --depth 1 --branch REL_16_STABLE https://github.com/pgaudit/pgaudit.git | |
| cd pgaudit | |
| # Apply patch to fix BABEL-4600 | |
| patch -p1 < ~/pgaudit_executor_hook_null_check.patch | |
| make install USE_PGXS=1 PG_CONFIG=~/psql/bin/pg_config | |
| else | |
| echo "Unsupported branch: ${{ matrix.extension_branch }}" | |
| exit 1 | |
| fi | |
| cd ~/${{ env.INSTALL_DIR }} | |
| sudo sed -i 's/^shared_preload_libraries .*$/shared_preload_libraries = '\''babelfishpg_tds, pg_stat_statements,pgaudit'\''/g' data/postgresql.conf | |
| echo "everything set" | |
| ~/${{ env.INSTALL_DIR }}/bin/pg_ctl -c -D ~/psql/data/ -l logfile restart | |
| sudo PGPASSWORD=12345678 ~/${{ env.INSTALL_DIR }}/bin/psql -v ON_ERROR_STOP=1 -h localhost -d babelfish_db -U jdbc_user -c "CREATE EXTENSION pgaudit;" | |
| sudo PGPASSWORD=12345678 ~/${{ env.INSTALL_DIR }}/bin/psql -v ON_ERROR_STOP=1 -h localhost -d babelfish_db -U jdbc_user -c "ALTER SYSTEM SET pgaudit.log = 'all';" | |
| shell: bash | |
| - name: Ignore some JDBC files | |
| run: | | |
| export PATH=~/${{env.INSTALL_DIR}}/bin:$PATH | |
| export PG_SRC=~/work/babelfish_extensions/postgresql_modified_for_babelfish | |
| export inputFilesPath=${{inputs.input_dir}} | |
| cd test | |
| # ignoring BABEL-2843 test file BABEL-2843,as in this file we are enabling the babelfish_statistics profile On for DDL, which is not giving any error but just adding the query under Query Text which is being passed from pg_audit and hence getting a diff file | |
| sudo sed -i '$a\ignore#!#BABEL-2843' JDBC/jdbc_schedule | |
| # ignoring below files as we will fix this issue in BABEL-5272 | |
| sudo sed -i '$a\ignore#!#babel_datatype_sqlvariant' JDBC/jdbc_schedule | |
| sudo sed -i '$a\ignore#!#babel_datatype_sqlvariant-vu-prepare' JDBC/jdbc_schedule | |
| sudo sed -i '$a\ignore#!#babel_datatype_sqlvariant-vu-verify' JDBC/jdbc_schedule | |
| sudo sed -i '$a\ignore#!#babel_datatype_sqlvariant-vu-cleanup' JDBC/jdbc_schedule | |
| - name: Run JDBC Tests | |
| id: jdbc | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| timeout-minutes: 60 | |
| uses: ./.github/composite-actions/run-jdbc-tests | |
| - name: Drop and re-create Babelfish database with pgaudit installed | |
| id: re-install-extensions | |
| if: always() && steps.install-extensions.outcome == 'success' | |
| run: | | |
| sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -f .github/scripts/cleanup_babelfish_database.sql | |
| sudo ~/psql/bin/psql -v ON_ERROR_STOP=1 -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -v migration_mode="multi-db" -v tsql_port=1433 -v parallel_query_mode=false -f .github/scripts/create_extension.sql | |
| sudo PGPASSWORD=12345678 ~/${{ env.INSTALL_DIR }}/bin/psql -v ON_ERROR_STOP=1 -h localhost -d babelfish_db -U jdbc_user -c "CREATE EXTENSION pgaudit;" | |
| sqlcmd -S localhost -U "jdbc_user" -P 12345678 -Q "SELECT @@version GO" | |
| - name: Run Dotnet Tests | |
| id: run-dotnet-tests | |
| if: always() && steps.re-install-extensions.outcome == 'success' | |
| uses: ./.github/composite-actions/install-and-run-dotnet | |
| - name: Start secondary server | |
| id: start-secondary | |
| if: always() && steps.jdbc.outcome == 'success' | |
| uses: ./.github/composite-actions/install-extensions | |
| with: | |
| psql_port: 5433 | |
| tsql_port: 8199 | |
| wal_level: logical | |
| - name: Setup Publication and Subscription | |
| id: setup-pub-sub | |
| if: always() && steps.start-secondary.outcome == 'success' | |
| run: | | |
| ~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d babelfish_db -U runner -c "CREATE PUBLICATION my_pub;" | |
| ~/${{env.INSTALL_DIR}}/bin/psql -v ON_ERROR_STOP=1 -d babelfish_db -U runner -p 5433 -c "CREATE SUBSCRIPTION my_sub CONNECTION 'host=localhost port=5432 user=jdbc_user dbname=babelfish_db password=12345678' PUBLICATION my_pub;" | |
| - name: Run Replication Tests | |
| id: replication | |
| if: always() && steps.setup-pub-sub.outcome == 'success' | |
| timeout-minutes: 60 | |
| uses: ./.github/composite-actions/run-jdbc-tests | |
| with: | |
| input_dir: 'replication' | |
| - name: Cleanup babelfish database | |
| id: cleanup | |
| if: always() && steps.replication.outcome == 'success' | |
| run: | | |
| sudo ~/psql/bin/psql -d postgres -U runner -v user="jdbc_user" -v db="babelfish_db" -f .github/scripts/cleanup_babelfish_database.sql | |
| - name: Upload Log | |
| if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: postgres-log-jdbc-${{ matrix.extension_branch }} | |
| path: | | |
| ~/psql/data/logfile | |
| ~/psql/data_5433/logfile | |
| # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions | |
| - name: Rename Test Summary Files | |
| id: test-file-rename | |
| if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') | |
| run: | | |
| cd test/JDBC/Info | |
| timestamp=`ls -Art | tail -n 1` | |
| cd $timestamp | |
| mv $timestamp.diff ../output-diff.diff | |
| mv "$timestamp"_runSummary.log ../run-summary.log | |
| cd .. | |
| # get the replication output diff as well if it is present | |
| dir_count=`ls | wc -l` | |
| if [[ $dir_count -eq 2 ]];then | |
| timestamp=`ls -rt | tail -n 2 | sort -r | tail -n 1` | |
| cd $timestamp | |
| mv $timestamp.diff ../replication-output-diff.diff | |
| mv "$timestamp"_runSummary.log ../replication-run-summary.log | |
| - name: Upload Run Summary-${{ matrix.extension_branch }} | |
| if: always() && steps.test-file-rename.outcome == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: run-summary.log | |
| path: | | |
| test/JDBC/Info/run-summary.log | |
| test/JDBC/Info/replication-run-summary.log | |
| - name: Upload Output Diff-${{ matrix.extension_branch }} | |
| if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: jdbc-output-diff.diff | |
| path: | | |
| test/JDBC/Info/output-diff.diff | |
| test/JDBC/Info/replication-output-diff.diff | |
| - name: Check and upload coredumps-${{ matrix.extension_branch }} | |
| if: always() && (steps.jdbc.outcome == 'failure' || steps.replication.outcome == 'failure') | |
| uses: ./.github/composite-actions/upload-coredump | |