diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index d500af738..08de3dca9 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -4,6 +4,7 @@ on: push: branches: - develop + - release/* paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' @@ -54,7 +55,7 @@ jobs: run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version diff --git a/.github/workflows/dockerhub-release-15-6.yml b/.github/workflows/dockerhub-release-15-6.yml index 253648bd4..4c7b5b998 100644 --- a/.github/workflows/dockerhub-release-15-6.yml +++ b/.github/workflows/dockerhub-release-15-6.yml @@ -4,9 +4,11 @@ on: push: branches: - develop + - release/* paths: - ".github/workflows/dockerhub-release-15-6.yml" - "common-nix.vars*" + workflow_dispatch: jobs: settings: diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 6513e21c0..a3f95f95f 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -4,7 +4,9 @@ on: push: branches: - develop + - release/* pull_request: + workflow_dispatch: permissions: contents: read @@ -56,4 +58,4 @@ jobs: -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \ base_nix bash -c "./workspace/docker/nix/build_nix.sh" name: build psql bundle on ${{ matrix.arch }} - \ No newline at end of file + diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml new file mode 100644 index 000000000..5b985f4be --- /dev/null +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -0,0 +1,101 @@ +name: Publish nix pg_upgrade_bin flake version + +on: + workflow_dispatch: + inputs: + postgresVersion: + description: 'Optional. Postgres version to publish against, i.e. 15.1.1.78' + required: false + +permissions: + id-token: write + +jobs: + publish-staging: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Grab release version + id: process_release_version + run: | + VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" + + - name: Create a tarball containing the latest nix flake version + working-directory: /tmp/ + run: | + mkdir -p ${{ steps.process_release_version.outputs.major_version }} + echo $GITHUB_SHA > ${{ steps.process_release_version.outputs.major_version }}/nix_flake_version + tar -czvf pg_upgrade_bin.tar.gz ${{ steps.process_release_version.outputs.major_version }} + + - name: configure aws credentials - staging + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: "us-east-1" + + - name: Upload pg_upgrade scripts to s3 staging + run: | + aws s3 cp /tmp/pg_upgrade_bin.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + + - name: Slack Notification on Failure + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: 'gha-failures-notifier' + SLACK_COLOR: 'danger' + SLACK_MESSAGE: 'Publishing pg_upgrade binaries flake version failed' + SLACK_FOOTER: '' + + publish-prod: + runs-on: ubuntu-latest + if: github.ref_name == 'develop' || contains( github.ref, 'release' ) + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Grab release version + id: process_release_version + run: | + VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" + + - name: Create a tarball containing the latest nix flake version + working-directory: /tmp/ + run: | + mkdir -p ${{ steps.process_release_version.outputs.major_version }} + echo $GITHUB_SHA > ${{ steps.process_release_version.outputs.major_version }}/nix_flake_version + tar -czvf pg_upgrade_bin.tar.gz ${{ steps.process_release_version.outputs.major_version }} + + - name: configure aws credentials - prod + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PROD_AWS_ROLE }} + aws-region: "us-east-1" + + - name: Upload pg_upgrade scripts to s3 prod + run: | + aws s3 cp /tmp/pg_upgrade_bin.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + + - name: Slack Notification on Failure + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: 'gha-failures-notifier' + SLACK_COLOR: 'danger' + SLACK_MESSAGE: 'Publishing pg_upgrade binaries flake version failed' + SLACK_FOOTER: '' diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 8ce9de40f..5d373ad8f 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -4,9 +4,9 @@ on: push: branches: - develop - - sam/nix-and-conventional-ami + - release/* paths: - - '.github/workflows/publish-pgupgrade-scripts.yml' + - '.github/workflows/publish-nix-pgupgrade-scripts.yml' - 'common-nix.vars.pkr.hcl' workflow_dispatch: inputs: @@ -62,7 +62,7 @@ jobs: publish-prod: runs-on: ubuntu-latest - if: github.ref_name == 'develop' + if: github.ref_name == 'develop' || contains( github.ref, 'release' ) steps: - name: Checkout Repo diff --git a/ansible/files/admin_api_scripts/grow_fs.sh b/ansible/files/admin_api_scripts/grow_fs.sh index 5c3dc73d4..1bca017b7 100644 --- a/ansible/files/admin_api_scripts/grow_fs.sh +++ b/ansible/files/admin_api_scripts/grow_fs.sh @@ -4,6 +4,11 @@ set -euo pipefail VOLUME_TYPE=${1:-data} +if pgrep resizefs; then + echo "resize2fs is already running" + exit 1 +fi + if [ -b /dev/nvme1n1 ] ; then if [[ "${VOLUME_TYPE}" == "data" ]]; then resize2fs /dev/nvme1n1 diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 879b63246..f9c35ed16 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -171,6 +171,58 @@ EOF done } +function patch_wrappers { + local IS_NIX_UPGRADE=$1 + + WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');") + if [ "$WRAPPERS_ENABLED" = "f" ]; then + echo "Wrappers extension not enabled. Skipping." + return + fi + + # This is a workaround for older versions of wrappers which don't have the expected + # naming scheme, containing the version in their library's file name + # e.g. wrappers-0.1.16.so, rather than wrappers.so + # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's + # library directory, so we're making sure the new version has the expected (old version's) + # file name. + # After the upgrade completes, the new version's library file is used. + # i.e. + # - old version: wrappers-0.1.16.so + # - new version: wrappers-0.1.18.so + # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so + if [ "$IS_NIX_UPGRADE" = "true" ]; then + if [ -d "$PGLIBOLD" ]; then + OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) + OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") + + find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do + if [ -f "$WRAPPERS_LIB_PATH" ]; then + WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") + if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then + echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" + fi + fi + done + fi + else + if [ -d "$PGLIBOLD" ]; then + WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) + if [ -f "$WRAPPERS_LIB_PATH" ]; then + OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) + if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then + LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") + if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then + echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" + fi + fi + fi + fi + fi +} + function initiate_upgrade { mkdir -p "$MOUNT_POINT" SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") @@ -324,30 +376,7 @@ function initiate_upgrade { export LD_LIBRARY_PATH="${PGLIBNEW}" fi - # This is a workaround for older versions of wrappers which don't have the expected - # naming scheme, containing the version in their library's file name - # e.g. wrappers-0.1.16.so, rather than wrappers.so - # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's - # library directory, so we're making sure the new version has the expected (old version's) - # file name. - # After the upgrade completes, the new version's library file is used. - # i.e. - # - old version: wrappers-0.1.16.so - # - new version: wrappers-0.1.18.so - # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so - if [ -d "$PGLIBOLD" ]; then - WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) - if [ -f "$WRAPPERS_LIB_PATH" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then - LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then - echo "Copying $OLD_WRAPPER_LIB_PATH to $WRAPPERS_LIB_PATH" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" - fi - fi - fi - fi + patch_wrappers "$IS_NIX_UPGRADE" echo "9. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 7938aa7b4..84802e469 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -206,3 +206,9 @@ shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove osquery" when: stage2_nix + + - name: nix collect garbage + become: yes + shell: | + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d" + when: stage2_nix diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index 57b76e1ee..a98d1990f 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -1,6 +1,20 @@ - name: PostgREST - system user user: name=postgrest +- name: PostgREST - add Postgres PPA gpg key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: present + +- name: PostgREST - add Postgres PPA + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}" + state: present + +- name: PostgREST - update apt cache + apt: + update_cache: yes + # libpq is a C library that enables user programs to communicate with # the PostgreSQL database server. - name: PostgREST - system dependencies @@ -9,9 +23,20 @@ - libpq5 - libnuma-dev +- name: PostgREST - remove Postgres PPA gpg key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: absent + +- name: PostgREST - remove Postgres PPA + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg {{ postgresql_major }}" + state: absent + - name: postgis - ensure dependencies do not get autoremoved shell: | set -e + apt-mark manual libpq5* apt-mark manual libnuma* apt-mark auto libnuma*-dev diff --git a/ansible/vars.yml b/ansible/vars.yml index 816dcc2df..38cdbb227 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -50,7 +50,7 @@ postgres_exporter_release_checksum: arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 -adminapi_release: 0.64.2 +adminapi_release: 0.66.1 adminmgr_release: 0.22.1 # Postgres Extensions diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 04eb9d52f..17f58af83 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.104" +postgres-version = "15.6.1.114" diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix index 7bc737d1c..33eb7ff6c 100644 --- a/nix/ext/pg_jsonschema.nix +++ b/nix/ext/pg_jsonschema.nix @@ -14,7 +14,10 @@ buildPgrxExtension_0_11_3 rec { nativeBuildInputs = [ cargo ]; buildInputs = [ postgresql ]; - + # update the following array when the pg_jsonschema version is updated + # required to ensure that extensions update scripts from previous versions are generated + + previousVersions = ["0.3.0" "0.2.0" "0.1.4" "0.1.4" "0.1.2" "0.1.1" "0.1.0"]; CARGO="${cargo}/bin/cargo"; env = lib.optionalAttrs stdenv.isDarwin { POSTGRES_LIB = "${postgresql}/lib"; @@ -26,6 +29,31 @@ buildPgrxExtension_0_11_3 rec { # to fix this a bit later. doCheck = false; + preBuild = '' + echo "Processing git tags..." + echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt + ''; + + postInstall = '' + echo "Creating SQL files for previous versions..." + current_version="${version}" + sql_file="$out/share/postgresql/extension/pg_jsonschema--$current_version.sql" + + if [ -f "$sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/pg_jsonschema--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$sql_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $sql_file not found" + fi + rm git_tags.txt + ''; + + meta = with lib; { description = "JSON Schema Validation for PostgreSQL"; homepage = "https://github.com/supabase/${pname}"; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index aca14dc9a..db1e972f1 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -7,43 +7,38 @@ , buildPgrxExtension_0_11_3 , cargo , darwin +, jq }: buildPgrxExtension_0_11_3 rec { pname = "supabase-wrappers"; version = "0.4.1"; + # update the following array when the wrappers version is updated + # required to ensure that extensions update scripts from previous versions are generated + previousVersions = ["0.4.0" "0.3.1" "0.3.0" "0.2.0" "0.1.19" "0.1.18" "0.1.17" "0.1.16" "0.1.15" "0.1.14" "0.1.12" "0.1.11" "0.1.10" "0.1.9" "0.1.8" "0.1.7" "0.1.6" "0.1.5" "0.1.4" "0.1.1" "0.1.0"]; inherit postgresql; - src = fetchFromGitHub { owner = "supabase"; repo = "wrappers"; rev = "v${version}"; hash = "sha256-AU9Y43qEMcIBVBThu+Aor1HCtfFIg+CdkzK9IxVdkzM="; }; - nativeBuildInputs = [ pkg-config cargo ]; - - buildInputs = [ openssl ] ++ lib.optionals (stdenv.isDarwin) [ + buildInputs = [ openssl ] ++ lib.optionals (stdenv.isDarwin) [ darwin.apple_sdk.frameworks.CoreFoundation darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.SystemConfiguration ]; - - # Needed to get openssl-sys to use pkg-config. OPENSSL_NO_VENDOR = 1; CARGO="${cargo}/bin/cargo"; - cargoLock = { - #TODO when we move to newer versions this lockfile will need to be sourced - # from ${src}/Cargo.lock lockFile = "${src}/Cargo.lock"; outputHashes = { "clickhouse-rs-1.0.0-alpha.1" = "sha256-0zmoUo/GLyCKDLkpBsnLAyGs1xz6cubJhn+eVqMEMaw="; }; }; postPatch = "cp ${cargoLock.lockFile} Cargo.lock"; - buildAndTestSubdir = "wrappers"; buildFeatures = [ "helloworld_fdw" @@ -60,12 +55,32 @@ buildPgrxExtension_0_11_3 rec { "cognito_fdw" "wasm_fdw" ]; - - # FIXME (aseipp): disable the tests since they try to install .control - # files into the wrong spot, aside from that the one main test seems - # to work, though doCheck = false; + preBuild = '' + echo "Processing git tags..." + echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt + ''; + + postInstall = '' + echo "Creating SQL files for previous versions..." + current_version="${version}" + sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + + if [ -f "$sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$sql_file" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $sql_file not found" + fi + rm git_tags.txt + ''; + meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; homepage = "https://github.com/supabase/wrappers"; @@ -73,4 +88,4 @@ buildPgrxExtension_0_11_3 rec { platforms = postgresql.meta.platforms; license = licenses.postgresql; }; -} \ No newline at end of file +}